diff --git a/base_requirements.txt b/base_requirements.txt index 8b42c835d..ed42b6c08 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -22,6 +22,10 @@ django-filter # https://github.com/django-mptt/django-mptt django-mptt +# Context managers for PostgreSQL advisory locks +# https://github.com/Xof/django-pglocks +django-pglocks + # Prometheus metrics library for Django # https://github.com/korfuri/django-prometheus django-prometheus diff --git a/docs/configuration/required-settings.md b/docs/configuration/required-settings.md index 70c6de276..e86b2810a 100644 --- a/docs/configuration/required-settings.md +++ b/docs/configuration/required-settings.md @@ -80,14 +80,56 @@ REDIS = { } ``` -!!! note: +!!! note If you are upgrading from a version prior to v2.7, please note that the Redis connection configuration settings have changed. Manual modification to bring the `REDIS` section inline with the above specification is necessary -!!! warning: +!!! note It is highly recommended to keep the webhook and cache databases separate. Using the same database number on the same Redis instance for both may result in webhook processing data being lost during cache flushing events. +### Using Redis Sentinel + +If you are using [Redis Sentinel](https://redis.io/topics/sentinel) for high-availability purposes, there is minimal +configuration necessary to convert NetBox to recognize it. It requires the removal of the `HOST` and `PORT` keys from +above and the addition of two new keys. + +* `SENTINELS`: List of tuples or tuple of tuples with each inner tuple containing the name or IP address +of the Redis server and port for each sentinel instance to connect to +* `SENTINEL_SERVICE`: Name of the master / service to connect to + +Example: + +```python +REDIS = { + 'webhooks': { + 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'SENTINELS': [ + ('mysentinel.redis.example.com', 6379), + ('othersentinel.redis.example.com', 6379) + ], + 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } +} +``` + +!!! note + It is possible to have only one or the other Redis configurations to use Sentinel functionality. It is possible + for example to have the webhook use sentinel via `HOST`/`PORT` and for caching to use Sentinel via + `SENTINELS`/`SENTINEL_SERVICE`. + + --- ## SECRET_KEY diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 63745bbab..d3b889513 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,11 +1,36 @@ -# v2.7.5 (FUTURE) +# v2.7.7 (FUTURE) + +## Enhancements + +* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Enhance search function when selecting VLANs for interface assignment +* [#4170](https://github.com/netbox-community/netbox/issues/4170) - Improve color contrast in rack elevation drawings + +## Bug Fixes + +* [#2519](https://github.com/netbox-community/netbox/issues/2519) - Avoid race condition when provisioning "next available" IPs/prefixes via the API +* [#4168](https://github.com/netbox-community/netbox/issues/4168) - Role is not required when creating a virtual machine + +--- + +# v2.7.6 (2020-02-13) + +## Bug Fixes + +* [#4166](https://github.com/netbox-community/netbox/issues/4166) - Fix schema migrations to enforce maximum character length for naturalized fields + +--- + +# v2.7.5 (2020-02-13) + +**Note:** This release includes several database schema migrations that calculate and store copies of names for certain objects to improve natural ordering performance (see [#3799](https://github.com/netbox-community/netbox/issues/3799)). These migrations may take a few minutes to run if you have a very large number of objects defined in NetBox. ## Enhancements * [#3766](https://github.com/netbox-community/netbox/issues/3766) - Allow custom script authors to specify the form widget for each variable * [#3799](https://github.com/netbox-community/netbox/issues/3799) - Greatly improve performance when ordering device components -* [#3986](https://github.com/netbox-community/netbox/issues/3986) - Include position numbers in SVG image when rendering rack elevation -* [#4093](https://github.com/netbox-community/netbox/issues/4093) - Add multiple status choices for VMs +* [#3984](https://github.com/netbox-community/netbox/issues/3984) - Add support for Redis Sentinel +* [#3986](https://github.com/netbox-community/netbox/issues/3986) - Include position numbers in SVG image when rendering rack elevations +* [#4093](https://github.com/netbox-community/netbox/issues/4093) - Add more status choices for virtual machines * [#4100](https://github.com/netbox-community/netbox/issues/4100) - Add device filter to component list views * [#4113](https://github.com/netbox-community/netbox/issues/4113) - Add bulk edit functionality for device type components * [#4116](https://github.com/netbox-community/netbox/issues/4116) - Enable bulk edit and delete functions for device component list views @@ -13,8 +38,8 @@ ## Bug Fixes -* [#3507](https://github.com/netbox-community/netbox/issues/3507) - Fix filtering IPaddress by multiple devices -* [#3995](https://github.com/netbox-community/netbox/issues/3995) - Make dropdown menus in the navigation bar scrollable +* [#3507](https://github.com/netbox-community/netbox/issues/3507) - Fix filtering IP addresses by multiple devices +* [#3995](https://github.com/netbox-community/netbox/issues/3995) - Make dropdown menus in the navigation bar scrollable on small screens * [#4083](https://github.com/netbox-community/netbox/issues/4083) - Permit nullifying applicable choice fields via API requests * [#4089](https://github.com/netbox-community/netbox/issues/4089) - Selection of power outlet type during bulk update is optional * [#4090](https://github.com/netbox-community/netbox/issues/4090) - Render URL custom fields as links under object view @@ -23,7 +48,7 @@ * [#4108](https://github.com/netbox-community/netbox/issues/4108) - Avoid extraneous database queries when rendering search forms * [#4134](https://github.com/netbox-community/netbox/issues/4134) - Device power ports and outlets should inherit type from the parent device type * [#4137](https://github.com/netbox-community/netbox/issues/4137) - Disable occupied terminations when connecting a cable to a circuit -* [#4138](https://github.com/netbox-community/netbox/issues/4138) - Include device bay counts in rack elevation diagrams +* [#4138](https://github.com/netbox-community/netbox/issues/4138) - Restore device bay counts in rack elevation diagrams * [#4146](https://github.com/netbox-community/netbox/issues/4146) - Fix enforcement of secret role assignment for secret decryption * [#4150](https://github.com/netbox-community/netbox/issues/4150) - Correct YAML rendering of config contexts * [#4159](https://github.com/netbox-community/netbox/issues/4159) - Fix implementation of Redis caching configuration diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 8f035ccbb..4c8a0821f 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -2832,7 +2832,10 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = DynamicModelMultipleChoiceField( @@ -2842,7 +2845,10 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tags = TagField( @@ -2871,18 +2877,20 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # Limit LAG choices to interfaces belonging to this device (or VC master) if self.is_bound: device = Device.objects.get(pk=self.data['device']) - self.fields['lag'].queryset = Interface.objects.filter( - device__in=[device, device.get_vc_master()], - type=InterfaceTypeChoices.TYPE_LAG - ) else: - self.fields['lag'].queryset = Interface.objects.filter( - device__in=[self.instance.device, self.instance.device.get_vc_master()], - type=InterfaceTypeChoices.TYPE_LAG - ) + device = self.instance.device + + # Limit LAG choices to interfaces belonging to this device (or VC master) + self.fields['lag'].queryset = Interface.objects.filter( + device__in=[device, device.get_vc_master()], + type=InterfaceTypeChoices.TYPE_LAG + ) + + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', device.site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', device.site.pk) class InterfaceCreateForm(BootstrapMixin, InterfaceCommonForm, forms.Form): @@ -2942,7 +2950,10 @@ class InterfaceCreateForm(BootstrapMixin, InterfaceCommonForm, forms.Form): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = DynamicModelMultipleChoiceField( @@ -2951,7 +2962,10 @@ class InterfaceCreateForm(BootstrapMixin, InterfaceCommonForm, forms.Form): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) @@ -2967,6 +2981,10 @@ class InterfaceCreateForm(BootstrapMixin, InterfaceCommonForm, forms.Form): type=InterfaceTypeChoices.TYPE_LAG ) + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', device.site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', device.site.pk) + class InterfaceCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( @@ -3090,7 +3108,10 @@ class InterfaceBulkEditForm(BootstrapMixin, AddRemoveTagsForm, BulkEditForm): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = DynamicModelMultipleChoiceField( @@ -3099,7 +3120,10 @@ class InterfaceBulkEditForm(BootstrapMixin, AddRemoveTagsForm, BulkEditForm): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) @@ -3118,6 +3142,10 @@ class InterfaceBulkEditForm(BootstrapMixin, AddRemoveTagsForm, BulkEditForm): device__in=[device, device.get_vc_master()], type=InterfaceTypeChoices.TYPE_LAG ) + + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', device.site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', device.site.pk) else: self.fields['lag'].choices = () self.fields['lag'].widget.attrs['disabled'] = True diff --git a/netbox/dcim/migrations/0093_device_component_ordering.py b/netbox/dcim/migrations/0093_device_component_ordering.py index 017241c8b..4e3c941a1 100644 --- a/netbox/dcim/migrations/0093_device_component_ordering.py +++ b/netbox/dcim/migrations/0093_device_component_ordering.py @@ -6,7 +6,7 @@ import utilities.ordering def _update_model_names(model): # Update each unique field value in bulk for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): - model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name, max_length=100)) def naturalize_consoleports(apps, schema_editor): diff --git a/netbox/dcim/migrations/0094_device_component_template_ordering.py b/netbox/dcim/migrations/0094_device_component_template_ordering.py index fc39f76b2..24fe98e94 100644 --- a/netbox/dcim/migrations/0094_device_component_template_ordering.py +++ b/netbox/dcim/migrations/0094_device_component_template_ordering.py @@ -6,7 +6,7 @@ import utilities.ordering def _update_model_names(model): # Update each unique field value in bulk for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): - model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name, max_length=100)) def naturalize_consoleporttemplates(apps, schema_editor): diff --git a/netbox/dcim/migrations/0095_primary_model_ordering.py b/netbox/dcim/migrations/0095_primary_model_ordering.py index 9cef0a581..3bc780161 100644 --- a/netbox/dcim/migrations/0095_primary_model_ordering.py +++ b/netbox/dcim/migrations/0095_primary_model_ordering.py @@ -6,7 +6,7 @@ import utilities.ordering def _update_model_names(model): # Update each unique field value in bulk for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): - model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name, max_length=100)) def naturalize_sites(apps, schema_editor): diff --git a/netbox/dcim/migrations/0096_interface_ordering.py b/netbox/dcim/migrations/0096_interface_ordering.py index 284066462..f1622f504 100644 --- a/netbox/dcim/migrations/0096_interface_ordering.py +++ b/netbox/dcim/migrations/0096_interface_ordering.py @@ -6,7 +6,7 @@ import utilities.ordering def _update_model_names(model): # Update each unique field value in bulk for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): - model.objects.filter(name=name).update(_name=utilities.ordering.naturalize_interface(name)) + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize_interface(name, max_length=100)) def naturalize_interfacetemplates(apps, schema_editor): diff --git a/netbox/dcim/models/__init__.py b/netbox/dcim/models/__init__.py index f291fc825..29afef1f1 100644 --- a/netbox/dcim/models/__init__.py +++ b/netbox/dcim/models/__init__.py @@ -382,8 +382,8 @@ class RackElevationHelperMixin: # add gradients RackElevationHelperMixin._add_gradient(drawing, 'reserved', '#c7c7ff') - RackElevationHelperMixin._add_gradient(drawing, 'occupied', '#f0f0f0') - RackElevationHelperMixin._add_gradient(drawing, 'blocked', '#ffc7c7') + RackElevationHelperMixin._add_gradient(drawing, 'occupied', '#d7d7d7') + RackElevationHelperMixin._add_gradient(drawing, 'blocked', '#ffc0c0') return drawing diff --git a/netbox/extras/apps.py b/netbox/extras/apps.py index f8c5a98e6..3201c3bb2 100644 --- a/netbox/extras/apps.py +++ b/netbox/extras/apps.py @@ -1,28 +1,8 @@ from django.apps import AppConfig -from django.conf import settings -from django.core.exceptions import ImproperlyConfigured -import redis class ExtrasConfig(AppConfig): name = "extras" def ready(self): - import extras.signals - - # Check that we can connect to the configured Redis database. - try: - rs = redis.Redis( - host=settings.WEBHOOKS_REDIS_HOST, - port=settings.WEBHOOKS_REDIS_PORT, - db=settings.WEBHOOKS_REDIS_DATABASE, - password=settings.WEBHOOKS_REDIS_PASSWORD or None, - ssl=settings.WEBHOOKS_REDIS_SSL, - ) - rs.ping() - except redis.exceptions.ConnectionError: - raise ImproperlyConfigured( - "Unable to connect to the Redis database. Check that the Redis configuration has been defined in " - "configuration.py." - ) diff --git a/netbox/extras/management/commands/renaturalize.py b/netbox/extras/management/commands/renaturalize.py index 70f57c1ba..cfd037910 100644 --- a/netbox/extras/management/commands/renaturalize.py +++ b/netbox/extras/management/commands/renaturalize.py @@ -86,7 +86,7 @@ class Command(BaseCommand): # Find all unique values for the field queryset = model.objects.values_list(target_field, flat=True).order_by(target_field).distinct() for value in queryset: - naturalized_value = naturalize(value) + naturalized_value = naturalize(value, max_length=field.max_length) if options['verbosity'] >= 2: self.stdout.write(" {} -> {}".format(value, naturalized_value), ending='') diff --git a/netbox/ipam/api/views.py b/netbox/ipam/api/views.py index 08e21367c..262ca7908 100644 --- a/netbox/ipam/api/views.py +++ b/netbox/ipam/api/views.py @@ -1,6 +1,7 @@ from django.conf import settings from django.db.models import Count from django.shortcuts import get_object_or_404 +from django_pglocks import advisory_lock from rest_framework import status from rest_framework.decorators import action from rest_framework.exceptions import PermissionDenied @@ -10,6 +11,7 @@ from extras.api.views import CustomFieldModelViewSet from ipam import filters from ipam.models import Aggregate, IPAddress, Prefix, RIR, Role, Service, VLAN, VLANGroup, VRF from utilities.api import FieldChoicesViewSet, ModelViewSet +from utilities.constants import ADVISORY_LOCK_KEYS from utilities.utils import get_subquery from . import serializers @@ -86,9 +88,13 @@ class PrefixViewSet(CustomFieldModelViewSet): filterset_class = filters.PrefixFilterSet @action(detail=True, url_path='available-prefixes', methods=['get', 'post']) + @advisory_lock(ADVISORY_LOCK_KEYS['available-prefixes']) def available_prefixes(self, request, pk=None): """ A convenience method for returning available child prefixes within a parent. + + The advisory lock decorator uses a PostgreSQL advisory lock to prevent this API from being + invoked in parallel, which results in a race condition where multiple insertions can occur. """ prefix = get_object_or_404(Prefix, pk=pk) available_prefixes = prefix.get_available_prefixes() @@ -180,11 +186,15 @@ class PrefixViewSet(CustomFieldModelViewSet): return Response(serializer.data) @action(detail=True, url_path='available-ips', methods=['get', 'post']) + @advisory_lock(ADVISORY_LOCK_KEYS['available-ips']) def available_ips(self, request, pk=None): """ A convenience method for returning available IP addresses within a prefix. By default, the number of IPs returned will be equivalent to PAGINATE_COUNT. An arbitrary limit (up to MAX_PAGE_SIZE, if set) may be passed, however results will not be paginated. + + The advisory lock decorator uses a PostgreSQL advisory lock to prevent this API from being + invoked in parallel, which results in a race condition where multiple insertions can occur. """ prefix = get_object_or_404(Prefix, pk=pk) diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index 885c9e0ba..7002def9b 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -28,6 +28,9 @@ REDIS = { 'webhooks': { 'HOST': 'localhost', 'PORT': 6379, + # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '', 'DATABASE': 0, 'DEFAULT_TIMEOUT': 300, @@ -36,6 +39,9 @@ REDIS = { 'caching': { 'HOST': 'localhost', 'PORT': 6379, + # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '', 'DATABASE': 1, 'DEFAULT_TIMEOUT': 300, diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 305f7efa8..249ee9e53 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -12,7 +12,7 @@ from django.core.exceptions import ImproperlyConfigured # Environment setup # -VERSION = '2.7.5-dev' +VERSION = '2.7.7-dev' # Hostname HOSTNAME = platform.node() @@ -170,14 +170,27 @@ if 'caching' not in REDIS: WEBHOOKS_REDIS = REDIS.get('webhooks', {}) WEBHOOKS_REDIS_HOST = WEBHOOKS_REDIS.get('HOST', 'localhost') WEBHOOKS_REDIS_PORT = WEBHOOKS_REDIS.get('PORT', 6379) +WEBHOOKS_REDIS_SENTINELS = WEBHOOKS_REDIS.get('SENTINELS', []) +WEBHOOKS_REDIS_USING_SENTINEL = all([ + isinstance(WEBHOOKS_REDIS_SENTINELS, (list, tuple)), + len(WEBHOOKS_REDIS_SENTINELS) > 0 +]) +WEBHOOKS_REDIS_SENTINEL_SERVICE = WEBHOOKS_REDIS.get('SENTINEL_SERVICE', 'default') WEBHOOKS_REDIS_PASSWORD = WEBHOOKS_REDIS.get('PASSWORD', '') WEBHOOKS_REDIS_DATABASE = WEBHOOKS_REDIS.get('DATABASE', 0) WEBHOOKS_REDIS_DEFAULT_TIMEOUT = WEBHOOKS_REDIS.get('DEFAULT_TIMEOUT', 300) WEBHOOKS_REDIS_SSL = WEBHOOKS_REDIS.get('SSL', False) + CACHING_REDIS = REDIS.get('caching', {}) CACHING_REDIS_HOST = CACHING_REDIS.get('HOST', 'localhost') CACHING_REDIS_PORT = CACHING_REDIS.get('PORT', 6379) +CACHING_REDIS_SENTINELS = CACHING_REDIS.get('SENTINELS', []) +CACHING_REDIS_USING_SENTINEL = all([ + isinstance(CACHING_REDIS_SENTINELS, (list, tuple)), + len(CACHING_REDIS_SENTINELS) > 0 +]) +CACHING_REDIS_SENTINEL_SERVICE = CACHING_REDIS.get('SENTINEL_SERVICE', 'default') CACHING_REDIS_PASSWORD = CACHING_REDIS.get('PASSWORD', '') CACHING_REDIS_DATABASE = CACHING_REDIS.get('DATABASE', 0) CACHING_REDIS_DEFAULT_TIMEOUT = CACHING_REDIS.get('DEFAULT_TIMEOUT', 300) @@ -394,28 +407,35 @@ if LDAP_CONFIG is not None: # # Caching # - -if CACHING_REDIS_SSL: - REDIS_CACHE_CON_STRING = 'rediss://' +if CACHING_REDIS_USING_SENTINEL: + CACHEOPS_SENTINEL = { + 'locations': CACHING_REDIS_SENTINELS, + 'service_name': CACHING_REDIS_SENTINEL_SERVICE, + 'db': CACHING_REDIS_DATABASE, + } else: - REDIS_CACHE_CON_STRING = 'redis://' + if CACHING_REDIS_SSL: + REDIS_CACHE_CON_STRING = 'rediss://' + else: + REDIS_CACHE_CON_STRING = 'redis://' -if CACHING_REDIS_PASSWORD: - REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, CACHING_REDIS_PASSWORD) + if CACHING_REDIS_PASSWORD: + REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, CACHING_REDIS_PASSWORD) -REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format( - REDIS_CACHE_CON_STRING, - CACHING_REDIS_HOST, - CACHING_REDIS_PORT, - CACHING_REDIS_DATABASE -) + REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format( + REDIS_CACHE_CON_STRING, + CACHING_REDIS_HOST, + CACHING_REDIS_PORT, + CACHING_REDIS_DATABASE + ) + CACHEOPS_REDIS = REDIS_CACHE_CON_STRING if not CACHE_TIMEOUT: CACHEOPS_ENABLED = False else: CACHEOPS_ENABLED = True -CACHEOPS_REDIS = REDIS_CACHE_CON_STRING + CACHEOPS_DEFAULTS = { 'timeout': CACHE_TIMEOUT } @@ -534,6 +554,15 @@ RQ_QUEUES = { 'PASSWORD': WEBHOOKS_REDIS_PASSWORD, 'DEFAULT_TIMEOUT': WEBHOOKS_REDIS_DEFAULT_TIMEOUT, 'SSL': WEBHOOKS_REDIS_SSL, + } if not WEBHOOKS_REDIS_USING_SENTINEL else { + 'SENTINELS': WEBHOOKS_REDIS_SENTINELS, + 'MASTER_NAME': WEBHOOKS_REDIS_SENTINEL_SERVICE, + 'DB': WEBHOOKS_REDIS_DATABASE, + 'PASSWORD': WEBHOOKS_REDIS_PASSWORD, + 'SOCKET_TIMEOUT': None, + 'CONNECTION_KWARGS': { + 'socket_connect_timeout': WEBHOOKS_REDIS_DEFAULT_TIMEOUT + }, } } diff --git a/netbox/project-static/js/forms.js b/netbox/project-static/js/forms.js index 4e1c9b0cc..802d1b4e9 100644 --- a/netbox/project-static/js/forms.js +++ b/netbox/project-static/js/forms.js @@ -190,15 +190,18 @@ $(document).ready(function() { $.each(element.attributes, function(index, attr){ if (attr.name.includes("data-additional-query-param-")){ var param_name = attr.name.split("data-additional-query-param-")[1]; - if (param_name in parameters) { - if (Array.isArray(parameters[param_name])) { - parameters[param_name].push(attr.value) + + $.each($.parseJSON(attr.value), function(index, value) { + if (param_name in parameters) { + if (Array.isArray(parameters[param_name])) { + parameters[param_name].push(value); + } else { + parameters[param_name] = [parameters[param_name], value]; + } } else { - parameters[param_name] = [parameters[param_name], attr.value] + parameters[param_name] = value; } - } else { - parameters[param_name] = attr.value; - } + }); } }); diff --git a/netbox/utilities/constants.py b/netbox/utilities/constants.py index ad6e8fd90..2cec2b532 100644 --- a/netbox/utilities/constants.py +++ b/netbox/utilities/constants.py @@ -27,3 +27,14 @@ COLOR_CHOICES = ( ('111111', 'Black'), ('ffffff', 'White'), ) + +# Keys for PostgreSQL advisory locks. These are arbitrary bigints used by +# the advisory_lock contextmanager. When a lock is acquired, +# one of these keys will be used to identify said lock. +# +# When adding a new key, pick something arbitrary and unique so +# that it is easily searchable in query logs. +ADVISORY_LOCK_KEYS = { + 'available-prefixes': 100100, + 'available-ips': 100200, +} diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index c9a857ad0..c629d70a2 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -309,12 +309,17 @@ class APISelect(SelectWithDisabled): def add_additional_query_param(self, name, value): """ - Add details for an additional query param in the form of a data-* attribute. + Add details for an additional query param in the form of a data-* JSON-encoded list attribute. :param name: The name of the query param :param value: The value of the query param """ - self.attrs['data-additional-query-param-{}'.format(name)] = value + key = 'data-additional-query-param-{}'.format(name) + + values = json.loads(self.attrs.get(key, '[]')) + values.append(value) + + self.attrs[key] = json.dumps(values) def add_conditional_query_param(self, condition, value): """ diff --git a/netbox/utilities/ordering.py b/netbox/utilities/ordering.py index d459e6f6c..39e21733a 100644 --- a/netbox/utilities/ordering.py +++ b/netbox/utilities/ordering.py @@ -10,7 +10,7 @@ INTERFACE_NAME_REGEX = r'(^(?P[^\d\.:]+)?)' \ r'(.(?P\d+)$)?' -def naturalize(value, max_length=None, integer_places=8): +def naturalize(value, max_length, integer_places=8): """ Take an alphanumeric string and prepend all integers to `integer_places` places to ensure the strings are ordered naturally. For example: @@ -39,10 +39,10 @@ def naturalize(value, max_length=None, integer_places=8): output.append(segment) ret = ''.join(output) - return ret[:max_length] if max_length else ret + return ret[:max_length] -def naturalize_interface(value, max_length=None): +def naturalize_interface(value, max_length): """ Similar in nature to naturalize(), but takes into account a particular naming format adapted from the old InterfaceManager. @@ -68,7 +68,7 @@ def naturalize_interface(value, max_length=None): if match.group('type') is not None: output.append(match.group('type')) - # Finally, append any remaining fields, left-padding to eight digits each. + # Finally, append any remaining fields, left-padding to six digits each. for part_name in ('id', 'channel', 'vc'): part = match.group(part_name) if part is not None: @@ -77,4 +77,4 @@ def naturalize_interface(value, max_length=None): output.append('000000') ret = ''.join(output) - return ret[:max_length] if max_length else ret + return ret[:max_length] diff --git a/netbox/utilities/tests/test_ordering.py b/netbox/utilities/tests/test_ordering.py new file mode 100644 index 000000000..958224c94 --- /dev/null +++ b/netbox/utilities/tests/test_ordering.py @@ -0,0 +1,49 @@ +from django.test import TestCase + +from utilities.ordering import naturalize, naturalize_interface + + +class NaturalizationTestCase(TestCase): + """ + Validate the operation of the functions which generate values suitable for natural ordering. + """ + def test_naturalize(self): + + data = ( + # Original, naturalized + ('abc', 'abc'), + ('123', '00000123'), + ('abc123', 'abc00000123'), + ('123abc', '00000123abc'), + ('123abc456', '00000123abc00000456'), + ('abc123def', 'abc00000123def'), + ('abc123def456', 'abc00000123def00000456'), + ) + + for origin, naturalized in data: + self.assertEqual(naturalize(origin, max_length=50), naturalized) + + def test_naturalize_max_length(self): + self.assertEqual(naturalize('abc123def456', max_length=10), 'abc0000012') + + def test_naturalize_interface(self): + + data = ( + # Original, naturalized + ('Gi', '9999999999999999Gi000000000000000000'), + ('Gi1', '9999999999999999Gi000001000000000000'), + ('Gi1/2', '0001999999999999Gi000002000000000000'), + ('Gi1/2/3', '0001000299999999Gi000003000000000000'), + ('Gi1/2/3/4', '0001000200039999Gi000004000000000000'), + ('Gi1/2/3/4/5', '0001000200030004Gi000005000000000000'), + ('Gi1/2/3/4/5:6', '0001000200030004Gi000005000006000000'), + ('Gi1/2/3/4/5:6.7', '0001000200030004Gi000005000006000007'), + ('Gi1:2', '9999999999999999Gi000001000002000000'), + ('Gi1:2.3', '9999999999999999Gi000001000002000003'), + ) + + for origin, naturalized in data: + self.assertEqual(naturalize_interface(origin, max_length=50), naturalized) + + def test_naturalize_interface_max_length(self): + self.assertEqual(naturalize_interface('Gi1/2/3', max_length=20), '0001000299999999Gi00') diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index 12393d400..0dbe38324 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -351,6 +351,7 @@ class VirtualMachineForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): ) role = DynamicModelChoiceField( queryset=DeviceRole.objects.all(), + required=False, widget=APISelect( api_url="/api/dcim/device-roles/", additional_query_params={ @@ -658,7 +659,10 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = DynamicModelMultipleChoiceField( @@ -667,7 +671,10 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tags = TagField( @@ -695,35 +702,12 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # Limit VLan choices to those in: global vlans, global groups, the current site's group, the current site - vlan_choices = [] - global_vlans = VLAN.objects.filter(site=None, group=None) - vlan_choices.append( - ('Global', [(vlan.pk, vlan) for vlan in global_vlans]) - ) - for group in VLANGroup.objects.filter(site=None): - global_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append( - (group.name, [(vlan.pk, vlan) for vlan in global_group_vlans]) - ) - + # Add current site to VLANs query params site = getattr(self.instance.parent, 'site', None) if site is not None: - - # Add non-grouped site VLANs - site_vlans = VLAN.objects.filter(site=site, group=None) - vlan_choices.append((site.name, [(vlan.pk, vlan) for vlan in site_vlans])) - - # Add grouped site VLANs - for group in VLANGroup.objects.filter(site=site): - site_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append(( - '{} / {}'.format(group.site.name, group.name), - [(vlan.pk, vlan) for vlan in site_group_vlans] - )) - - self.fields['untagged_vlan'].choices = [(None, '---------')] + vlan_choices - self.fields['tagged_vlans'].choices = vlan_choices + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk) def clean(self): super().clean() @@ -784,7 +768,10 @@ class InterfaceCreateForm(BootstrapMixin, forms.Form): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = DynamicModelMultipleChoiceField( @@ -793,7 +780,10 @@ class InterfaceCreateForm(BootstrapMixin, forms.Form): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tags = TagField( @@ -807,35 +797,11 @@ class InterfaceCreateForm(BootstrapMixin, forms.Form): pk=self.initial.get('virtual_machine') or self.data.get('virtual_machine') ) - # Limit VLAN choices to those in: global vlans, global groups, the current site's group, the current site - vlan_choices = [] - global_vlans = VLAN.objects.filter(site=None, group=None) - vlan_choices.append( - ('Global', [(vlan.pk, vlan) for vlan in global_vlans]) - ) - for group in VLANGroup.objects.filter(site=None): - global_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append( - (group.name, [(vlan.pk, vlan) for vlan in global_group_vlans]) - ) - site = getattr(virtual_machine.cluster, 'site', None) if site is not None: - - # Add non-grouped site VLANs - site_vlans = VLAN.objects.filter(site=site, group=None) - vlan_choices.append((site.name, [(vlan.pk, vlan) for vlan in site_vlans])) - - # Add grouped site VLANs - for group in VLANGroup.objects.filter(site=site): - site_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append(( - '{} / {}'.format(group.site.name, group.name), - [(vlan.pk, vlan) for vlan in site_group_vlans] - )) - - self.fields['untagged_vlan'].choices = [(None, '---------')] + vlan_choices - self.fields['tagged_vlans'].choices = vlan_choices + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk) class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm): @@ -872,7 +838,10 @@ class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = DynamicModelMultipleChoiceField( @@ -881,7 +850,10 @@ class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) @@ -897,35 +869,11 @@ class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm): if 'virtual_machine' in self.initial: parent_obj = VirtualMachine.objects.filter(pk=self.initial['virtual_machine']).first() - # Limit VLAN choices to global VLANs, VLANs in global groups, the current site's group, the current site - vlan_choices = [] - global_vlans = VLAN.objects.filter(site=None, group=None) - vlan_choices.append( - ('Global', [(vlan.pk, vlan) for vlan in global_vlans]) - ) - for group in VLANGroup.objects.filter(site=None): - global_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append( - (group.name, [(vlan.pk, vlan) for vlan in global_group_vlans]) - ) - if parent_obj.cluster is not None: - site = getattr(parent_obj.cluster, 'site', None) - if site is not None: - - # Add non-grouped site VLANs - site_vlans = VLAN.objects.filter(site=site, group=None) - vlan_choices.append((site.name, [(vlan.pk, vlan) for vlan in site_vlans])) - - # Add grouped site VLANs - for group in VLANGroup.objects.filter(site=site): - site_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append(( - '{} / {}'.format(group.site.name, group.name), - [(vlan.pk, vlan) for vlan in site_group_vlans] - )) - - self.fields['untagged_vlan'].choices = [(None, '---------')] + vlan_choices - self.fields['tagged_vlans'].choices = vlan_choices + site = getattr(parent_obj.cluster, 'site', None) + if site is not None: + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk) # diff --git a/requirements.txt b/requirements.txt index 9c5f624f9..b0b1b971d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ django-cors-headers==3.2.1 django-debug-toolbar==2.1 django-filter==2.2.0 django-mptt==0.9.1 +django-pglocks==1.0.4 django-prometheus==1.1.0 django-rq==2.2.0 django-tables2==2.2.1