Merge branch 'develop' into 4164-object-list-template

base_requirements.txt

@@ -22,6 +22,10 @@ django-filter
 # https://github.com/django-mptt/django-mptt
 django-mptt
 
+# Context managers for PostgreSQL advisory locks
+# https://github.com/Xof/django-pglocks
+django-pglocks
+
 # Prometheus metrics library for Django
 # https://github.com/korfuri/django-prometheus
 django-prometheus

docs/configuration/required-settings.md

@@ -80,14 +80,56 @@ REDIS = {
 }
 ```
 
-!!! note:
+!!! note
     If you are upgrading from a version prior to v2.7, please note that the Redis connection configuration settings have
     changed. Manual modification to bring the `REDIS` section inline with the above specification is necessary
 
-!!! warning:
+!!! note
     It is highly recommended to keep the webhook and cache databases separate. Using the same database number on the
     same Redis instance for both may result in webhook processing data being lost during cache flushing events.
 
+### Using Redis Sentinel
+
+If you are using [Redis Sentinel](https://redis.io/topics/sentinel) for high-availability purposes, there is minimal 
+configuration necessary to convert NetBox to recognize it. It requires the removal of the `HOST` and `PORT` keys from 
+above and the addition of two new keys.
+
+* `SENTINELS`: List of tuples or tuple of tuples with each inner tuple containing the name or IP address 
+of the Redis server and port for each sentinel instance to connect to
+* `SENTINEL_SERVICE`: Name of the master / service to connect to
+
+Example:
+
+```python
+REDIS = {
+    'webhooks': {
+        'SENTINELS': [('mysentinel.redis.example.com', 6379)],
+        'SENTINEL_SERVICE': 'netbox',
+        'PASSWORD': '',
+        'DATABASE': 0,
+        'DEFAULT_TIMEOUT': 300,
+        'SSL': False,
+    },
+    'caching': {
+        'SENTINELS': [
+            ('mysentinel.redis.example.com', 6379),
+            ('othersentinel.redis.example.com', 6379)
+        ],
+        'SENTINEL_SERVICE': 'netbox',
+        'PASSWORD': '',
+        'DATABASE': 1,
+        'DEFAULT_TIMEOUT': 300,
+        'SSL': False,
+    }
+}
+```
+
+!!! note
+    It is possible to have only one or the other Redis configurations to use Sentinel functionality. It is possible
+    for example to have the webhook use sentinel via `HOST`/`PORT` and for caching to use Sentinel via 
+    `SENTINELS`/`SENTINEL_SERVICE`.
+
+
 ---
 
 ## SECRET_KEY

docs/release-notes/version-2.7.md

@@ -1,11 +1,36 @@
-# v2.7.5 (FUTURE)
+# v2.7.7 (FUTURE)
+
+## Enhancements
+
+* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Enhance search function when selecting VLANs for interface assignment
+* [#4170](https://github.com/netbox-community/netbox/issues/4170) - Improve color contrast in rack elevation drawings
+
+## Bug Fixes
+
+* [#2519](https://github.com/netbox-community/netbox/issues/2519) - Avoid race condition when provisioning "next available" IPs/prefixes via the API
+* [#4168](https://github.com/netbox-community/netbox/issues/4168) - Role is not required when creating a virtual machine
+
+---
+
+# v2.7.6 (2020-02-13)
+
+## Bug Fixes
+
+* [#4166](https://github.com/netbox-community/netbox/issues/4166) - Fix schema migrations to enforce maximum character length for naturalized fields
+
+---
+
+# v2.7.5 (2020-02-13)
+
+**Note:** This release includes several database schema migrations that calculate and store copies of names for certain objects to improve natural ordering performance (see [#3799](https://github.com/netbox-community/netbox/issues/3799)). These migrations may take a few minutes to run if you have a very large number of objects defined in NetBox.
 
 ## Enhancements
 
 * [#3766](https://github.com/netbox-community/netbox/issues/3766) - Allow custom script authors to specify the form widget for each variable
 * [#3799](https://github.com/netbox-community/netbox/issues/3799) - Greatly improve performance when ordering device components
-* [#3986](https://github.com/netbox-community/netbox/issues/3986) - Include position numbers in SVG image when rendering rack elevation
+* [#3984](https://github.com/netbox-community/netbox/issues/3984) - Add support for Redis Sentinel
-* [#4093](https://github.com/netbox-community/netbox/issues/4093) - Add multiple status choices for VMs
+* [#3986](https://github.com/netbox-community/netbox/issues/3986) - Include position numbers in SVG image when rendering rack elevations
+* [#4093](https://github.com/netbox-community/netbox/issues/4093) - Add more status choices for virtual machines
 * [#4100](https://github.com/netbox-community/netbox/issues/4100) - Add device filter to component list views
 * [#4113](https://github.com/netbox-community/netbox/issues/4113) - Add bulk edit functionality for device type components
 * [#4116](https://github.com/netbox-community/netbox/issues/4116) - Enable bulk edit and delete functions for device component list views
@@ -13,8 +38,8 @@
 
 ## Bug Fixes
 
-* [#3507](https://github.com/netbox-community/netbox/issues/3507) - Fix filtering IPaddress by multiple devices
+* [#3507](https://github.com/netbox-community/netbox/issues/3507) - Fix filtering IP addresses by multiple devices
-* [#3995](https://github.com/netbox-community/netbox/issues/3995) - Make dropdown menus in the navigation bar scrollable
+* [#3995](https://github.com/netbox-community/netbox/issues/3995) - Make dropdown menus in the navigation bar scrollable on small screens
 * [#4083](https://github.com/netbox-community/netbox/issues/4083) - Permit nullifying applicable choice fields via API requests
 * [#4089](https://github.com/netbox-community/netbox/issues/4089) - Selection of power outlet type during bulk update is optional
 * [#4090](https://github.com/netbox-community/netbox/issues/4090) - Render URL custom fields as links under object view
@@ -23,7 +48,7 @@
 * [#4108](https://github.com/netbox-community/netbox/issues/4108) - Avoid extraneous database queries when rendering search forms
 * [#4134](https://github.com/netbox-community/netbox/issues/4134) - Device power ports and outlets should inherit type from the parent device type
 * [#4137](https://github.com/netbox-community/netbox/issues/4137) - Disable occupied terminations when connecting a cable to a circuit
-* [#4138](https://github.com/netbox-community/netbox/issues/4138) - Include device bay counts in rack elevation diagrams
+* [#4138](https://github.com/netbox-community/netbox/issues/4138) - Restore device bay counts in rack elevation diagrams
 * [#4146](https://github.com/netbox-community/netbox/issues/4146) - Fix enforcement of secret role assignment for secret decryption
 * [#4150](https://github.com/netbox-community/netbox/issues/4150) - Correct YAML rendering of config contexts
 * [#4159](https://github.com/netbox-community/netbox/issues/4159) - Fix implementation of Redis caching configuration

netbox/dcim/forms.py

@@ -2832,7 +2832,10 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm):
         widget=APISelect(
             api_url="/api/ipam/vlans/",
             display_field='display_name',
-            full=True
+            full=True,
+            additional_query_params={
+                'site_id': 'null',
+            },
         )
     )
     tagged_vlans = DynamicModelMultipleChoiceField(
@@ -2842,7 +2845,10 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm):
         widget=APISelectMultiple(
             api_url="/api/ipam/vlans/",
             display_field='display_name',
-            full=True
+            full=True,
+            additional_query_params={
+                'site_id': 'null',
+            },
         )
     )
     tags = TagField(
@@ -2871,18 +2877,20 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        # Limit LAG choices to interfaces belonging to this device (or VC master)
         if self.is_bound:
             device = Device.objects.get(pk=self.data['device'])
-            self.fields['lag'].queryset = Interface.objects.filter(
-                device__in=[device, device.get_vc_master()],
-                type=InterfaceTypeChoices.TYPE_LAG
-            )
         else:
-            self.fields['lag'].queryset = Interface.objects.filter(
+            device = self.instance.device
-                device__in=[self.instance.device, self.instance.device.get_vc_master()],
+
-                type=InterfaceTypeChoices.TYPE_LAG
+        # Limit LAG choices to interfaces belonging to this device (or VC master)
-            )
+        self.fields['lag'].queryset = Interface.objects.filter(
+            device__in=[device, device.get_vc_master()],
+            type=InterfaceTypeChoices.TYPE_LAG
+        )
+
+        # Add current site to VLANs query params
+        self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', device.site.pk)
+        self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', device.site.pk)
 
 
 class InterfaceCreateForm(BootstrapMixin, InterfaceCommonForm, forms.Form):
@@ -2942,7 +2950,10 @@ class InterfaceCreateForm(BootstrapMixin, InterfaceCommonForm, forms.Form):
         widget=APISelect(
             api_url="/api/ipam/vlans/",
             display_field='display_name',
-            full=True
+            full=True,
+            additional_query_params={
+                'site_id': 'null',
+            },
         )
     )
     tagged_vlans = DynamicModelMultipleChoiceField(
@@ -2951,7 +2962,10 @@ class InterfaceCreateForm(BootstrapMixin, InterfaceCommonForm, forms.Form):
         widget=APISelectMultiple(
             api_url="/api/ipam/vlans/",
             display_field='display_name',
-            full=True
+            full=True,
+            additional_query_params={
+                'site_id': 'null',
+            },
         )
     )
 
@@ -2967,6 +2981,10 @@ class InterfaceCreateForm(BootstrapMixin, InterfaceCommonForm, forms.Form):
             type=InterfaceTypeChoices.TYPE_LAG
         )
 
+        # Add current site to VLANs query params
+        self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', device.site.pk)
+        self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', device.site.pk)
+
 
 class InterfaceCSVForm(forms.ModelForm):
     device = FlexibleModelChoiceField(
@@ -3090,7 +3108,10 @@ class InterfaceBulkEditForm(BootstrapMixin, AddRemoveTagsForm, BulkEditForm):
         widget=APISelect(
             api_url="/api/ipam/vlans/",
             display_field='display_name',
-            full=True
+            full=True,
+            additional_query_params={
+                'site_id': 'null',
+            },
         )
     )
     tagged_vlans = DynamicModelMultipleChoiceField(
@@ -3099,7 +3120,10 @@ class InterfaceBulkEditForm(BootstrapMixin, AddRemoveTagsForm, BulkEditForm):
         widget=APISelectMultiple(
             api_url="/api/ipam/vlans/",
             display_field='display_name',
-            full=True
+            full=True,
+            additional_query_params={
+                'site_id': 'null',
+            },
         )
     )
 
@@ -3118,6 +3142,10 @@ class InterfaceBulkEditForm(BootstrapMixin, AddRemoveTagsForm, BulkEditForm):
                 device__in=[device, device.get_vc_master()],
                 type=InterfaceTypeChoices.TYPE_LAG
             )
+
+            # Add current site to VLANs query params
+            self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', device.site.pk)
+            self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', device.site.pk)
         else:
             self.fields['lag'].choices = ()
             self.fields['lag'].widget.attrs['disabled'] = True

netbox/dcim/migrations/0093_device_component_ordering.py

@@ -6,7 +6,7 @@ import utilities.ordering
 def _update_model_names(model):
     # Update each unique field value in bulk
     for name in model.objects.values_list('name', flat=True).order_by('name').distinct():
-        model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name))
+        model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name, max_length=100))
 
 
 def naturalize_consoleports(apps, schema_editor):

netbox/dcim/migrations/0094_device_component_template_ordering.py

@@ -6,7 +6,7 @@ import utilities.ordering
 def _update_model_names(model):
     # Update each unique field value in bulk
     for name in model.objects.values_list('name', flat=True).order_by('name').distinct():
-        model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name))
+        model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name, max_length=100))
 
 
 def naturalize_consoleporttemplates(apps, schema_editor):

netbox/dcim/migrations/0095_primary_model_ordering.py

@@ -6,7 +6,7 @@ import utilities.ordering
 def _update_model_names(model):
     # Update each unique field value in bulk
     for name in model.objects.values_list('name', flat=True).order_by('name').distinct():
-        model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name))
+        model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name, max_length=100))
 
 
 def naturalize_sites(apps, schema_editor):

netbox/dcim/migrations/0096_interface_ordering.py

@@ -6,7 +6,7 @@ import utilities.ordering
 def _update_model_names(model):
     # Update each unique field value in bulk
     for name in model.objects.values_list('name', flat=True).order_by('name').distinct():
-        model.objects.filter(name=name).update(_name=utilities.ordering.naturalize_interface(name))
+        model.objects.filter(name=name).update(_name=utilities.ordering.naturalize_interface(name, max_length=100))
 
 
 def naturalize_interfacetemplates(apps, schema_editor):

netbox/dcim/models/__init__.py

@@ -382,8 +382,8 @@ class RackElevationHelperMixin:
 
         # add gradients
         RackElevationHelperMixin._add_gradient(drawing, 'reserved', '#c7c7ff')
-        RackElevationHelperMixin._add_gradient(drawing, 'occupied', '#f0f0f0')
+        RackElevationHelperMixin._add_gradient(drawing, 'occupied', '#d7d7d7')
-        RackElevationHelperMixin._add_gradient(drawing, 'blocked', '#ffc7c7')
+        RackElevationHelperMixin._add_gradient(drawing, 'blocked', '#ffc0c0')
 
         return drawing
 

netbox/extras/apps.py

@@ -1,28 +1,8 @@
 from django.apps import AppConfig
-from django.conf import settings
-from django.core.exceptions import ImproperlyConfigured
-import redis
 
 
 class ExtrasConfig(AppConfig):
     name = "extras"
 
     def ready(self):
-
         import extras.signals
-
-        # Check that we can connect to the configured Redis database.
-        try:
-            rs = redis.Redis(
-                host=settings.WEBHOOKS_REDIS_HOST,
-                port=settings.WEBHOOKS_REDIS_PORT,
-                db=settings.WEBHOOKS_REDIS_DATABASE,
-                password=settings.WEBHOOKS_REDIS_PASSWORD or None,
-                ssl=settings.WEBHOOKS_REDIS_SSL,
-            )
-            rs.ping()
-        except redis.exceptions.ConnectionError:
-            raise ImproperlyConfigured(
-                "Unable to connect to the Redis database. Check that the Redis configuration has been defined in "
-                "configuration.py."
-            )

netbox/extras/management/commands/renaturalize.py

@@ -86,7 +86,7 @@ class Command(BaseCommand):
                 # Find all unique values for the field
                 queryset = model.objects.values_list(target_field, flat=True).order_by(target_field).distinct()
                 for value in queryset:
-                    naturalized_value = naturalize(value)
+                    naturalized_value = naturalize(value, max_length=field.max_length)
 
                     if options['verbosity'] >= 2:
                         self.stdout.write("  {} -> {}".format(value, naturalized_value), ending='')

netbox/ipam/api/views.py

@@ -1,6 +1,7 @@
 from django.conf import settings
 from django.db.models import Count
 from django.shortcuts import get_object_or_404
+from django_pglocks import advisory_lock
 from rest_framework import status
 from rest_framework.decorators import action
 from rest_framework.exceptions import PermissionDenied
@@ -10,6 +11,7 @@ from extras.api.views import CustomFieldModelViewSet
 from ipam import filters
 from ipam.models import Aggregate, IPAddress, Prefix, RIR, Role, Service, VLAN, VLANGroup, VRF
 from utilities.api import FieldChoicesViewSet, ModelViewSet
+from utilities.constants import ADVISORY_LOCK_KEYS
 from utilities.utils import get_subquery
 from . import serializers
 
@@ -86,9 +88,13 @@ class PrefixViewSet(CustomFieldModelViewSet):
     filterset_class = filters.PrefixFilterSet
 
     @action(detail=True, url_path='available-prefixes', methods=['get', 'post'])
+    @advisory_lock(ADVISORY_LOCK_KEYS['available-prefixes'])
     def available_prefixes(self, request, pk=None):
         """
         A convenience method for returning available child prefixes within a parent.
+
+        The advisory lock decorator uses a PostgreSQL advisory lock to prevent this API from being
+        invoked in parallel, which results in a race condition where multiple insertions can occur.
         """
         prefix = get_object_or_404(Prefix, pk=pk)
         available_prefixes = prefix.get_available_prefixes()
@@ -180,11 +186,15 @@ class PrefixViewSet(CustomFieldModelViewSet):
             return Response(serializer.data)
 
     @action(detail=True, url_path='available-ips', methods=['get', 'post'])
+    @advisory_lock(ADVISORY_LOCK_KEYS['available-ips'])
     def available_ips(self, request, pk=None):
         """
         A convenience method for returning available IP addresses within a prefix. By default, the number of IPs
         returned will be equivalent to PAGINATE_COUNT. An arbitrary limit (up to MAX_PAGE_SIZE, if set) may be passed,
         however results will not be paginated.
+
+        The advisory lock decorator uses a PostgreSQL advisory lock to prevent this API from being
+        invoked in parallel, which results in a race condition where multiple insertions can occur.
         """
         prefix = get_object_or_404(Prefix, pk=pk)
 

netbox/netbox/configuration.example.py

@@ -28,6 +28,9 @@ REDIS = {
     'webhooks': {
         'HOST': 'localhost',
         'PORT': 6379,
+        # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel
+        # 'SENTINELS': [('mysentinel.redis.example.com', 6379)],
+        # 'SENTINEL_SERVICE': 'netbox',
         'PASSWORD': '',
         'DATABASE': 0,
         'DEFAULT_TIMEOUT': 300,
@@ -36,6 +39,9 @@ REDIS = {
     'caching': {
         'HOST': 'localhost',
         'PORT': 6379,
+        # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel
+        # 'SENTINELS': [('mysentinel.redis.example.com', 6379)],
+        # 'SENTINEL_SERVICE': 'netbox',
         'PASSWORD': '',
         'DATABASE': 1,
         'DEFAULT_TIMEOUT': 300,

netbox/netbox/settings.py

@@ -12,7 +12,7 @@ from django.core.exceptions import ImproperlyConfigured
 # Environment setup
 #
 
-VERSION = '2.7.5-dev'
+VERSION = '2.7.7-dev'
 
 # Hostname
 HOSTNAME = platform.node()
@@ -170,14 +170,27 @@ if 'caching' not in REDIS:
 WEBHOOKS_REDIS = REDIS.get('webhooks', {})
 WEBHOOKS_REDIS_HOST = WEBHOOKS_REDIS.get('HOST', 'localhost')
 WEBHOOKS_REDIS_PORT = WEBHOOKS_REDIS.get('PORT', 6379)
+WEBHOOKS_REDIS_SENTINELS = WEBHOOKS_REDIS.get('SENTINELS', [])
+WEBHOOKS_REDIS_USING_SENTINEL = all([
+    isinstance(WEBHOOKS_REDIS_SENTINELS, (list, tuple)),
+    len(WEBHOOKS_REDIS_SENTINELS) > 0
+])
+WEBHOOKS_REDIS_SENTINEL_SERVICE = WEBHOOKS_REDIS.get('SENTINEL_SERVICE', 'default')
 WEBHOOKS_REDIS_PASSWORD = WEBHOOKS_REDIS.get('PASSWORD', '')
 WEBHOOKS_REDIS_DATABASE = WEBHOOKS_REDIS.get('DATABASE', 0)
 WEBHOOKS_REDIS_DEFAULT_TIMEOUT = WEBHOOKS_REDIS.get('DEFAULT_TIMEOUT', 300)
 WEBHOOKS_REDIS_SSL = WEBHOOKS_REDIS.get('SSL', False)
 
+
 CACHING_REDIS = REDIS.get('caching', {})
 CACHING_REDIS_HOST = CACHING_REDIS.get('HOST', 'localhost')
 CACHING_REDIS_PORT = CACHING_REDIS.get('PORT', 6379)
+CACHING_REDIS_SENTINELS = CACHING_REDIS.get('SENTINELS', [])
+CACHING_REDIS_USING_SENTINEL = all([
+    isinstance(CACHING_REDIS_SENTINELS, (list, tuple)),
+    len(CACHING_REDIS_SENTINELS) > 0
+])
+CACHING_REDIS_SENTINEL_SERVICE = CACHING_REDIS.get('SENTINEL_SERVICE', 'default')
 CACHING_REDIS_PASSWORD = CACHING_REDIS.get('PASSWORD', '')
 CACHING_REDIS_DATABASE = CACHING_REDIS.get('DATABASE', 0)
 CACHING_REDIS_DEFAULT_TIMEOUT = CACHING_REDIS.get('DEFAULT_TIMEOUT', 300)
@@ -394,28 +407,35 @@ if LDAP_CONFIG is not None:
 #
 # Caching
 #
-
+if CACHING_REDIS_USING_SENTINEL:
-if CACHING_REDIS_SSL:
+    CACHEOPS_SENTINEL = {
-    REDIS_CACHE_CON_STRING = 'rediss://'
+        'locations': CACHING_REDIS_SENTINELS,
+        'service_name': CACHING_REDIS_SENTINEL_SERVICE,
+        'db': CACHING_REDIS_DATABASE,
+    }
 else:
-    REDIS_CACHE_CON_STRING = 'redis://'
+    if CACHING_REDIS_SSL:
+        REDIS_CACHE_CON_STRING = 'rediss://'
+    else:
+        REDIS_CACHE_CON_STRING = 'redis://'
 
-if CACHING_REDIS_PASSWORD:
+    if CACHING_REDIS_PASSWORD:
-    REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, CACHING_REDIS_PASSWORD)
+        REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, CACHING_REDIS_PASSWORD)
 
-REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format(
+    REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format(
-    REDIS_CACHE_CON_STRING,
+        REDIS_CACHE_CON_STRING,
-    CACHING_REDIS_HOST,
+        CACHING_REDIS_HOST,
-    CACHING_REDIS_PORT,
+        CACHING_REDIS_PORT,
-    CACHING_REDIS_DATABASE
+        CACHING_REDIS_DATABASE
-)
+    )
+    CACHEOPS_REDIS = REDIS_CACHE_CON_STRING
 
 if not CACHE_TIMEOUT:
     CACHEOPS_ENABLED = False
 else:
     CACHEOPS_ENABLED = True
 
-CACHEOPS_REDIS = REDIS_CACHE_CON_STRING
+
 CACHEOPS_DEFAULTS = {
     'timeout': CACHE_TIMEOUT
 }
@@ -534,6 +554,15 @@ RQ_QUEUES = {
         'PASSWORD': WEBHOOKS_REDIS_PASSWORD,
         'DEFAULT_TIMEOUT': WEBHOOKS_REDIS_DEFAULT_TIMEOUT,
         'SSL': WEBHOOKS_REDIS_SSL,
+    } if not WEBHOOKS_REDIS_USING_SENTINEL else {
+        'SENTINELS': WEBHOOKS_REDIS_SENTINELS,
+        'MASTER_NAME': WEBHOOKS_REDIS_SENTINEL_SERVICE,
+        'DB': WEBHOOKS_REDIS_DATABASE,
+        'PASSWORD': WEBHOOKS_REDIS_PASSWORD,
+        'SOCKET_TIMEOUT': None,
+        'CONNECTION_KWARGS': {
+            'socket_connect_timeout': WEBHOOKS_REDIS_DEFAULT_TIMEOUT
+        },
     }
 }
 

netbox/project-static/js/forms.js

@@ -190,15 +190,18 @@ $(document).ready(function() {
                 $.each(element.attributes, function(index, attr){
                     if (attr.name.includes("data-additional-query-param-")){
                         var param_name = attr.name.split("data-additional-query-param-")[1];
-                        if (param_name in parameters) {
+
-                            if (Array.isArray(parameters[param_name])) {
+                        $.each($.parseJSON(attr.value), function(index, value) {
-                                parameters[param_name].push(attr.value)
+                            if (param_name in parameters) {
+                                if (Array.isArray(parameters[param_name])) {
+                                    parameters[param_name].push(value);
+                                } else {
+                                    parameters[param_name] = [parameters[param_name], value];
+                                }
                             } else {
-                                parameters[param_name] = [parameters[param_name], attr.value]
+                                parameters[param_name] = value;
                             }
-                        } else {
+                        });
-                            parameters[param_name] = attr.value;
-                        }
                     }
                 });
 

netbox/utilities/constants.py

@@ -27,3 +27,14 @@ COLOR_CHOICES = (
     ('111111', 'Black'),
     ('ffffff', 'White'),
 )
+
+# Keys for PostgreSQL advisory locks. These are arbitrary bigints used by
+# the advisory_lock contextmanager. When a lock is acquired,
+# one of these keys will be used to identify said lock.
+#
+# When adding a new key, pick something arbitrary and unique so
+# that it is easily searchable in query logs.
+ADVISORY_LOCK_KEYS = {
+    'available-prefixes': 100100,
+    'available-ips': 100200,
+}

netbox/utilities/forms.py

@@ -309,12 +309,17 @@ class APISelect(SelectWithDisabled):
 
     def add_additional_query_param(self, name, value):
         """
-        Add details for an additional query param in the form of a data-* attribute.
+        Add details for an additional query param in the form of a data-* JSON-encoded list attribute.
 
         :param name: The name of the query param
         :param value: The value of the query param
         """
-        self.attrs['data-additional-query-param-{}'.format(name)] = value
+        key = 'data-additional-query-param-{}'.format(name)
+
+        values = json.loads(self.attrs.get(key, '[]'))
+        values.append(value)
+
+        self.attrs[key] = json.dumps(values)
 
     def add_conditional_query_param(self, condition, value):
         """

netbox/utilities/ordering.py

@@ -10,7 +10,7 @@ INTERFACE_NAME_REGEX = r'(^(?P<type>[^\d\.:]+)?)' \
                        r'(.(?P<vc>\d+)$)?'
 
 
-def naturalize(value, max_length=None, integer_places=8):
+def naturalize(value, max_length, integer_places=8):
     """
     Take an alphanumeric string and prepend all integers to `integer_places` places to ensure the strings
     are ordered naturally. For example:
@@ -39,10 +39,10 @@ def naturalize(value, max_length=None, integer_places=8):
             output.append(segment)
     ret = ''.join(output)
 
-    return ret[:max_length] if max_length else ret
+    return ret[:max_length]
 
 
-def naturalize_interface(value, max_length=None):
+def naturalize_interface(value, max_length):
     """
     Similar in nature to naturalize(), but takes into account a particular naming format adapted from the old
     InterfaceManager.
@@ -68,7 +68,7 @@ def naturalize_interface(value, max_length=None):
     if match.group('type') is not None:
         output.append(match.group('type'))
 
-    # Finally, append any remaining fields, left-padding to eight digits each.
+    # Finally, append any remaining fields, left-padding to six digits each.
     for part_name in ('id', 'channel', 'vc'):
         part = match.group(part_name)
         if part is not None:
@@ -77,4 +77,4 @@ def naturalize_interface(value, max_length=None):
             output.append('000000')
 
     ret = ''.join(output)
-    return ret[:max_length] if max_length else ret
+    return ret[:max_length]

netbox/utilities/tests/test_ordering.py

@@ -0,0 +1,49 @@
+from django.test import TestCase
+
+from utilities.ordering import naturalize, naturalize_interface
+
+
+class NaturalizationTestCase(TestCase):
+    """
+    Validate the operation of the functions which generate values suitable for natural ordering.
+    """
+    def test_naturalize(self):
+
+        data = (
+            # Original, naturalized
+            ('abc', 'abc'),
+            ('123', '00000123'),
+            ('abc123', 'abc00000123'),
+            ('123abc', '00000123abc'),
+            ('123abc456', '00000123abc00000456'),
+            ('abc123def', 'abc00000123def'),
+            ('abc123def456', 'abc00000123def00000456'),
+        )
+
+        for origin, naturalized in data:
+            self.assertEqual(naturalize(origin, max_length=50), naturalized)
+
+    def test_naturalize_max_length(self):
+        self.assertEqual(naturalize('abc123def456', max_length=10), 'abc0000012')
+
+    def test_naturalize_interface(self):
+
+        data = (
+            # Original, naturalized
+            ('Gi', '9999999999999999Gi000000000000000000'),
+            ('Gi1', '9999999999999999Gi000001000000000000'),
+            ('Gi1/2', '0001999999999999Gi000002000000000000'),
+            ('Gi1/2/3', '0001000299999999Gi000003000000000000'),
+            ('Gi1/2/3/4', '0001000200039999Gi000004000000000000'),
+            ('Gi1/2/3/4/5', '0001000200030004Gi000005000000000000'),
+            ('Gi1/2/3/4/5:6', '0001000200030004Gi000005000006000000'),
+            ('Gi1/2/3/4/5:6.7', '0001000200030004Gi000005000006000007'),
+            ('Gi1:2', '9999999999999999Gi000001000002000000'),
+            ('Gi1:2.3', '9999999999999999Gi000001000002000003'),
+        )
+
+        for origin, naturalized in data:
+            self.assertEqual(naturalize_interface(origin, max_length=50), naturalized)
+
+    def test_naturalize_interface_max_length(self):
+        self.assertEqual(naturalize_interface('Gi1/2/3', max_length=20), '0001000299999999Gi00')

netbox/virtualization/forms.py

@@ -351,6 +351,7 @@ class VirtualMachineForm(BootstrapMixin, TenancyForm, CustomFieldModelForm):
     )
     role = DynamicModelChoiceField(
         queryset=DeviceRole.objects.all(),
+        required=False,
         widget=APISelect(
             api_url="/api/dcim/device-roles/",
             additional_query_params={
@@ -658,7 +659,10 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm):
         widget=APISelect(
             api_url="/api/ipam/vlans/",
             display_field='display_name',
-            full=True
+            full=True,
+            additional_query_params={
+                'site_id': 'null',
+            },
         )
     )
     tagged_vlans = DynamicModelMultipleChoiceField(
@@ -667,7 +671,10 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm):
         widget=APISelectMultiple(
             api_url="/api/ipam/vlans/",
             display_field='display_name',
-            full=True
+            full=True,
+            additional_query_params={
+                'site_id': 'null',
+            },
         )
     )
     tags = TagField(
@@ -695,35 +702,12 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        # Limit VLan choices to those in: global vlans, global groups, the current site's group, the current site
+        # Add current site to VLANs query params
-        vlan_choices = []
-        global_vlans = VLAN.objects.filter(site=None, group=None)
-        vlan_choices.append(
-            ('Global', [(vlan.pk, vlan) for vlan in global_vlans])
-        )
-        for group in VLANGroup.objects.filter(site=None):
-            global_group_vlans = VLAN.objects.filter(group=group)
-            vlan_choices.append(
-                (group.name, [(vlan.pk, vlan) for vlan in global_group_vlans])
-            )
-
         site = getattr(self.instance.parent, 'site', None)
         if site is not None:
-
+            # Add current site to VLANs query params
-            # Add non-grouped site VLANs
+            self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk)
-            site_vlans = VLAN.objects.filter(site=site, group=None)
+            self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk)
-            vlan_choices.append((site.name, [(vlan.pk, vlan) for vlan in site_vlans]))
-
-            # Add grouped site VLANs
-            for group in VLANGroup.objects.filter(site=site):
-                site_group_vlans = VLAN.objects.filter(group=group)
-                vlan_choices.append((
-                    '{} / {}'.format(group.site.name, group.name),
-                    [(vlan.pk, vlan) for vlan in site_group_vlans]
-                ))
-
-        self.fields['untagged_vlan'].choices = [(None, '---------')] + vlan_choices
-        self.fields['tagged_vlans'].choices = vlan_choices
 
     def clean(self):
         super().clean()
@@ -784,7 +768,10 @@ class InterfaceCreateForm(BootstrapMixin, forms.Form):
         widget=APISelect(
             api_url="/api/ipam/vlans/",
             display_field='display_name',
-            full=True
+            full=True,
+            additional_query_params={
+                'site_id': 'null',
+            },
         )
     )
     tagged_vlans = DynamicModelMultipleChoiceField(
@@ -793,7 +780,10 @@ class InterfaceCreateForm(BootstrapMixin, forms.Form):
         widget=APISelectMultiple(
             api_url="/api/ipam/vlans/",
             display_field='display_name',
-            full=True
+            full=True,
+            additional_query_params={
+                'site_id': 'null',
+            },
         )
     )
     tags = TagField(
@@ -807,35 +797,11 @@ class InterfaceCreateForm(BootstrapMixin, forms.Form):
             pk=self.initial.get('virtual_machine') or self.data.get('virtual_machine')
         )
 
-        # Limit VLAN choices to those in: global vlans, global groups, the current site's group, the current site
-        vlan_choices = []
-        global_vlans = VLAN.objects.filter(site=None, group=None)
-        vlan_choices.append(
-            ('Global', [(vlan.pk, vlan) for vlan in global_vlans])
-        )
-        for group in VLANGroup.objects.filter(site=None):
-            global_group_vlans = VLAN.objects.filter(group=group)
-            vlan_choices.append(
-                (group.name, [(vlan.pk, vlan) for vlan in global_group_vlans])
-            )
-
         site = getattr(virtual_machine.cluster, 'site', None)
         if site is not None:
-
+            # Add current site to VLANs query params
-            # Add non-grouped site VLANs
+            self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk)
-            site_vlans = VLAN.objects.filter(site=site, group=None)
+            self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk)
-            vlan_choices.append((site.name, [(vlan.pk, vlan) for vlan in site_vlans]))
-
-            # Add grouped site VLANs
-            for group in VLANGroup.objects.filter(site=site):
-                site_group_vlans = VLAN.objects.filter(group=group)
-                vlan_choices.append((
-                    '{} / {}'.format(group.site.name, group.name),
-                    [(vlan.pk, vlan) for vlan in site_group_vlans]
-                ))
-
-        self.fields['untagged_vlan'].choices = [(None, '---------')] + vlan_choices
-        self.fields['tagged_vlans'].choices = vlan_choices
 
 
 class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm):
@@ -872,7 +838,10 @@ class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm):
         widget=APISelect(
             api_url="/api/ipam/vlans/",
             display_field='display_name',
-            full=True
+            full=True,
+            additional_query_params={
+                'site_id': 'null',
+            },
         )
     )
     tagged_vlans = DynamicModelMultipleChoiceField(
@@ -881,7 +850,10 @@ class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm):
         widget=APISelectMultiple(
             api_url="/api/ipam/vlans/",
             display_field='display_name',
-            full=True
+            full=True,
+            additional_query_params={
+                'site_id': 'null',
+            },
         )
     )
 
@@ -897,35 +869,11 @@ class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm):
         if 'virtual_machine' in self.initial:
             parent_obj = VirtualMachine.objects.filter(pk=self.initial['virtual_machine']).first()
 
-            # Limit VLAN choices to global VLANs, VLANs in global groups, the current site's group, the current site
+            site = getattr(parent_obj.cluster, 'site', None)
-            vlan_choices = []
+            if site is not None:
-            global_vlans = VLAN.objects.filter(site=None, group=None)
+                # Add current site to VLANs query params
-            vlan_choices.append(
+                self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk)
-                ('Global', [(vlan.pk, vlan) for vlan in global_vlans])
+                self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk)
-            )
-            for group in VLANGroup.objects.filter(site=None):
-                global_group_vlans = VLAN.objects.filter(group=group)
-                vlan_choices.append(
-                    (group.name, [(vlan.pk, vlan) for vlan in global_group_vlans])
-                )
-            if parent_obj.cluster is not None:
-                site = getattr(parent_obj.cluster, 'site', None)
-                if site is not None:
-
-                    # Add non-grouped site VLANs
-                    site_vlans = VLAN.objects.filter(site=site, group=None)
-                    vlan_choices.append((site.name, [(vlan.pk, vlan) for vlan in site_vlans]))
-
-                    # Add grouped site VLANs
-                    for group in VLANGroup.objects.filter(site=site):
-                        site_group_vlans = VLAN.objects.filter(group=group)
-                        vlan_choices.append((
-                            '{} / {}'.format(group.site.name, group.name),
-                            [(vlan.pk, vlan) for vlan in site_group_vlans]
-                        ))
-
-            self.fields['untagged_vlan'].choices = [(None, '---------')] + vlan_choices
-            self.fields['tagged_vlans'].choices = vlan_choices
 
 
 #

requirements.txt

@@ -4,6 +4,7 @@ django-cors-headers==3.2.1
 django-debug-toolbar==2.1
 django-filter==2.2.0
 django-mptt==0.9.1
+django-pglocks==1.0.4
 django-prometheus==1.1.0
 django-rq==2.2.0
 django-tables2==2.2.1