Merge branch 'develop' into feature

netbox/utilities/constants.py

@@ -93,3 +93,7 @@ HTML_ALLOWED_ATTRIBUTES = {
     "td": {"align"},
     "th": {"align"},
 }
+
+HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS = ['socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h']
+HTTP_PROXY_SOCK_RDNS_SCHEMAS = ['socks4h', 'socks4a', 'socks5h', 'socks5a']
+HTTP_PROXY_SUPPORTED_SCHEMAS = ['http', 'https', 'socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h']

netbox/utilities/forms/fields/csv.py

@@ -1,7 +1,7 @@
 from django import forms
 from django.utils.translation import gettext_lazy as _
 from django.contrib.contenttypes.models import ContentType
-from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
+from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist, FieldError
 from django.db.models import Q
 
 from utilities.choices import unpack_grouped_choices
@@ -64,6 +64,10 @@ class CSVModelChoiceField(forms.ModelChoiceField):
             raise forms.ValidationError(
                 _('"{value}" is not a unique value for this field; multiple objects were found').format(value=value)
             )
+        except FieldError:
+            raise forms.ValidationError(
+                _('"{field_name}" is an invalid accessor field name.').format(field_name=self.to_field_name)
+            )
 
 
 class CSVModelMultipleChoiceField(forms.ModelMultipleChoiceField):

netbox/utilities/forms/fields/dynamic.py

@@ -142,7 +142,7 @@ class DynamicModelChoiceMixin:
 
         if data:
             # When the field is multiple choice pass the data as a list if it's not already
-            if isinstance(bound_field.field, DynamicModelMultipleChoiceField) and not type(data) is list:
+            if isinstance(bound_field.field, DynamicModelMultipleChoiceField) and type(data) is not list:
                 data = [data]
 
             field_name = getattr(self, 'to_field_name') or 'pk'

netbox/utilities/html.py

@@ -59,7 +59,7 @@ def highlight(value, highlight, trim_pre=None, trim_post=None, trim_placeholder=
         else:
             highlight = re.escape(highlight)
             pre, match, post = re.split(fr'({highlight})', value, maxsplit=1, flags=re.IGNORECASE)
-    except ValueError as e:
+    except ValueError:
         # Match not found
         return escape(value)
 

netbox/utilities/management/commands/calculate_cached_counts.py

@@ -1,7 +1,6 @@
 from collections import defaultdict
 
 from django.core.management.base import BaseCommand
-from django.db.models import Count, OuterRef, Subquery
 
 from netbox.registry import registry
 from utilities.counters import update_counts

netbox/utilities/markdown.py

@@ -14,7 +14,7 @@ class StrikethroughExtension(markdown.Extension):
     """
     def extendMarkdown(self, md):
         md.inlinePatterns.register(
-            markdown.inlinepatterns.SimpleTagPattern(STRIKE_RE, 'del'),
+            SimpleTagPattern(STRIKE_RE, 'del'),
             'strikethrough',
             200
         )

netbox/utilities/release.py

@@ -1,9 +1,9 @@
 import datetime
 import os
-import yaml
 from dataclasses import asdict, dataclass, field
-from typing import List, Union
+from typing import Union
 
+import yaml
 from django.core.exceptions import ImproperlyConfigured
 
 from utilities.datetime import datetime_from_timestamp

netbox/utilities/serializers/json.py

@@ -1,5 +1,5 @@
 from django.contrib.postgres.fields import ArrayField
-from django.core.serializers.json import Deserializer, Serializer as Serializer_  # noqa
+from django.core.serializers.json import Deserializer, Serializer as Serializer_  # noqa: F401
 from django.utils.encoding import is_protected_type
 
 # NOTE: Module must contain both Serializer and Deserializer

netbox/utilities/socks.py

@@ -0,0 +1,101 @@
+import logging
+
+from urllib.parse import urlparse
+from urllib3 import PoolManager, HTTPConnectionPool, HTTPSConnectionPool
+from urllib3.connection import HTTPConnection, HTTPSConnection
+from .constants import HTTP_PROXY_SOCK_RDNS_SCHEMAS
+
+
+logger = logging.getLogger('netbox.utilities')
+
+
+class ProxyHTTPConnection(HTTPConnection):
+    """
+    A Proxy connection class that uses a SOCK proxy - used to create
+    a urllib3 PoolManager that routes connections via the proxy.
+    This is for an HTTP (not HTTPS) connection
+    """
+    use_rdns = False
+
+    def __init__(self, *args, **kwargs):
+        socks_options = kwargs.pop('_socks_options')
+        self._proxy_url = socks_options['proxy_url']
+        super().__init__(*args, **kwargs)
+
+    def _new_conn(self):
+        try:
+            from python_socks.sync import Proxy
+        except ModuleNotFoundError as e:
+            logger.info("Configuring an HTTP proxy using SOCKS requires the python_socks library. Check that it has been installed.")
+            raise e
+
+        proxy = Proxy.from_url(self._proxy_url, rdns=self.use_rdns)
+        return proxy.connect(
+            dest_host=self.host,
+            dest_port=self.port,
+            timeout=self.timeout
+        )
+
+
+class ProxyHTTPSConnection(ProxyHTTPConnection, HTTPSConnection):
+    """
+    A Proxy connection class for an HTTPS (not HTTP) connection.
+    """
+    pass
+
+
+class RdnsProxyHTTPConnection(ProxyHTTPConnection):
+    """
+    A Proxy connection class for an HTTP remote-dns connection.
+    I.E. socks4a, socks4h, socks5a, socks5h
+    """
+    use_rdns = True
+
+
+class RdnsProxyHTTPSConnection(ProxyHTTPSConnection):
+    """
+    A Proxy connection class for an HTTPS remote-dns connection.
+    I.E. socks4a, socks4h, socks5a, socks5h
+    """
+    use_rdns = True
+
+
+class ProxyHTTPConnectionPool(HTTPConnectionPool):
+    ConnectionCls = ProxyHTTPConnection
+
+
+class ProxyHTTPSConnectionPool(HTTPSConnectionPool):
+    ConnectionCls = ProxyHTTPSConnection
+
+
+class RdnsProxyHTTPConnectionPool(HTTPConnectionPool):
+    ConnectionCls = RdnsProxyHTTPConnection
+
+
+class RdnsProxyHTTPSConnectionPool(HTTPSConnectionPool):
+    ConnectionCls = RdnsProxyHTTPSConnection
+
+
+class ProxyPoolManager(PoolManager):
+    def __init__(self, proxy_url, timeout=5, num_pools=10, headers=None, **connection_pool_kw):
+        # python_socks uses rdns param to denote remote DNS parsing and
+        # doesn't accept the 'h' or 'a' in the proxy URL
+        if use_rdns := urlparse(proxy_url).scheme in HTTP_PROXY_SOCK_RDNS_SCHEMAS:
+            proxy_url = proxy_url.replace('socks5h:', 'socks5:').replace('socks5a:', 'socks5:')
+            proxy_url = proxy_url.replace('socks4h:', 'socks4:').replace('socks4a:', 'socks4:')
+
+        connection_pool_kw['_socks_options'] = {'proxy_url': proxy_url}
+        connection_pool_kw['timeout'] = timeout
+
+        super().__init__(num_pools, headers, **connection_pool_kw)
+
+        if use_rdns:
+            self.pool_classes_by_scheme = {
+                'http': RdnsProxyHTTPConnectionPool,
+                'https': RdnsProxyHTTPSConnectionPool,
+            }
+        else:
+            self.pool_classes_by_scheme = {
+                'http': ProxyHTTPConnectionPool,
+                'https': ProxyHTTPSConnectionPool,
+            }

netbox/utilities/templatetags/form_helpers.py

@@ -1,8 +1,6 @@
-import warnings
-
 from django import template
 
-from utilities.forms.rendering import FieldSet, InlineFields, ObjectAttribute, TabbedGroups
+from utilities.forms.rendering import InlineFields, ObjectAttribute, TabbedGroups
 
 __all__ = (
     'getfield',

netbox/utilities/tests/test_api.py

@@ -149,7 +149,7 @@ class APIPaginationTestCase(APITestCase):
 
         self.assertHttpStatus(response, status.HTTP_200_OK)
         self.assertEqual(response.data['count'], 100)
-        self.assertTrue(response.data['next'].endswith(f'?limit=10&offset=10'))
+        self.assertTrue(response.data['next'].endswith('?limit=10&offset=10'))
         self.assertIsNone(response.data['previous'])
         self.assertEqual(len(response.data['results']), 10)
 
@@ -159,7 +159,7 @@ class APIPaginationTestCase(APITestCase):
 
         self.assertHttpStatus(response, status.HTTP_200_OK)
         self.assertEqual(response.data['count'], 100)
-        self.assertTrue(response.data['next'].endswith(f'?limit=20&offset=20'))
+        self.assertTrue(response.data['next'].endswith('?limit=20&offset=20'))
         self.assertIsNone(response.data['previous'])
         self.assertEqual(len(response.data['results']), 20)
 

netbox/utilities/tests/test_counters.py

@@ -85,7 +85,7 @@ class CountersTest(TestCase):
     def test_mptt_child_delete(self):
         device1, device2 = Device.objects.all()
         inventory_item1 = InventoryItem.objects.create(device=device1, name='Inventory Item 1')
-        inventory_item2 = InventoryItem.objects.create(device=device1, name='Inventory Item 2', parent=inventory_item1)
+        InventoryItem.objects.create(device=device1, name='Inventory Item 2', parent=inventory_item1)
         device1.refresh_from_db()
         self.assertEqual(device1.inventory_item_count, 2)
 

netbox/utilities/tests/test_filters.py

@@ -7,7 +7,7 @@ from taggit.managers import TaggableManager
 
 from dcim.choices import *
 from dcim.fields import MACAddressField
-from dcim.filtersets import DeviceFilterSet, SiteFilterSet
+from dcim.filtersets import DeviceFilterSet, SiteFilterSet, InterfaceFilterSet
 from dcim.models import (
     Device, DeviceRole, DeviceType, Interface, Manufacturer, Platform, Rack, Region, Site
 )
@@ -16,6 +16,7 @@ from extras.models import TaggedItem
 from ipam.filtersets import ASNFilterSet
 from ipam.models import RIR, ASN
 from netbox.filtersets import BaseFilterSet
+from wireless.choices import WirelessRoleChoices
 from utilities.filters import (
     MultiValueCharFilter, MultiValueDateFilter, MultiValueDateTimeFilter, MultiValueMACAddressFilter,
     MultiValueNumberFilter, MultiValueTimeFilter, TreeNodeMultipleChoiceFilter,
@@ -408,9 +409,9 @@ class DynamicFilterLookupExpressionTest(TestCase):
             region.save()
 
         sites = (
-            Site(name='Site 1', slug='abc-site-1', region=regions[0]),
-            Site(name='Site 2', slug='def-site-2', region=regions[1]),
-            Site(name='Site 3', slug='ghi-site-3', region=regions[2]),
+            Site(name='Site 1', slug='abc-site-1', region=regions[0], status=SiteStatusChoices.STATUS_ACTIVE),
+            Site(name='Site 2', slug='def-site-2', region=regions[1], status=SiteStatusChoices.STATUS_ACTIVE),
+            Site(name='Site 3', slug='ghi-site-3', region=regions[2], status=SiteStatusChoices.STATUS_PLANNED),
         )
         Site.objects.bulk_create(sites)
 
@@ -438,7 +439,7 @@ class DynamicFilterLookupExpressionTest(TestCase):
             Interface(device=devices[1], name='Interface 3', mac_address='00-00-00-00-00-02'),
             Interface(device=devices[1], name='Interface 4', mac_address='bb-00-00-00-00-02'),
             Interface(device=devices[2], name='Interface 5', mac_address='00-00-00-00-00-03'),
-            Interface(device=devices[2], name='Interface 6', mac_address='cc-00-00-00-00-03'),
+            Interface(device=devices[2], name='Interface 6', mac_address='cc-00-00-00-00-03', rf_role=WirelessRoleChoices.ROLE_AP),
         )
         Interface.objects.bulk_create(interfaces)
 
@@ -446,6 +447,14 @@ class DynamicFilterLookupExpressionTest(TestCase):
         params = {'name__n': ['Site 1']}
         self.assertEqual(SiteFilterSet(params, Site.objects.all()).qs.count(), 2)
 
+    def test_site_status_icontains(self):
+        params = {'status__ic': [SiteStatusChoices.STATUS_ACTIVE]}
+        self.assertEqual(SiteFilterSet(params, Site.objects.all()).qs.count(), 2)
+
+    def test_site_status_icontains_negation(self):
+        params = {'status__nic': [SiteStatusChoices.STATUS_ACTIVE]}
+        self.assertEqual(SiteFilterSet(params, Site.objects.all()).qs.count(), 1)
+
     def test_site_slug_icontains(self):
         params = {'slug__ic': ['-1']}
         self.assertEqual(SiteFilterSet(params, Site.objects.all()).qs.count(), 1)
@@ -553,3 +562,9 @@ class DynamicFilterLookupExpressionTest(TestCase):
     def test_device_mac_address_icontains_negation(self):
         params = {'mac_address__nic': ['aa:', 'bb']}
         self.assertEqual(DeviceFilterSet(params, Device.objects.all()).qs.count(), 1)
+
+    def test_interface_rf_role_empty(self):
+        params = {'rf_role__empty': 'true'}
+        self.assertEqual(InterfaceFilterSet(params, Interface.objects.all()).qs.count(), 5)
+        params = {'rf_role__empty': 'false'}
+        self.assertEqual(InterfaceFilterSet(params, Interface.objects.all()).qs.count(), 1)