From 7ed023a8fbb81a71a1334d91df1364c57f5f16af Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Wed, 15 Jan 2020 12:23:34 +0000 Subject: [PATCH 01/28] Fixes #3840: Only show valid interface VLAN choices --- docs/release-notes/version-2.6.md | 3 ++- netbox/dcim/forms.py | 42 ++++++++++++++++++++++++++----- netbox/project-static/js/forms.js | 17 +++++++------ netbox/utilities/forms.py | 9 +++++-- 4 files changed, 55 insertions(+), 16 deletions(-) diff --git a/docs/release-notes/version-2.6.md b/docs/release-notes/version-2.6.md index 792e8990a..b31e769a3 100644 --- a/docs/release-notes/version-2.6.md +++ b/docs/release-notes/version-2.6.md @@ -3,6 +3,7 @@ ## Enhancements * [#3525](https://github.com/netbox-community/netbox/issues/3525) - Enable IP address filtering with multiple address terms +* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Only show the valid list of interface VLAN choices ## Bug Fixes @@ -42,7 +43,7 @@ * [#3872](https://github.com/netbox-community/netbox/issues/3872) - Paginate related IPs on the IP address view * [#3876](https://github.com/netbox-community/netbox/issues/3876) - Fix minimum/maximum value rendering for site ASN field * [#3882](https://github.com/netbox-community/netbox/issues/3882) - Fix filtering of devices by rack group -* [#3898](https://github.com/netbox-community/netbox/issues/3898) - Fix references to deleted cables without a label +* [#3898](https://github.com/netbox-community/netbox/issues/3898) - Fix references to deleted cables without a label * [#3905](https://github.com/netbox-community/netbox/issues/3905) - Fix divide-by-zero on power feeds with low power values --- diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index cd356cc09..4b5dd33cf 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -2238,7 +2238,10 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = forms.ModelMultipleChoiceField( @@ -2247,7 +2250,10 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) @@ -2289,6 +2295,10 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm): device__in=[self.instance.device, self.instance.device.get_vc_master()], type=IFACE_TYPE_LAG ) + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', device.site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', device.site.pk) + class InterfaceCreateForm(InterfaceCommonForm, ComponentForm, forms.Form): name_pattern = ExpandableNameField( @@ -2340,7 +2350,10 @@ class InterfaceCreateForm(InterfaceCommonForm, ComponentForm, forms.Form): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = forms.ModelMultipleChoiceField( @@ -2349,7 +2362,10 @@ class InterfaceCreateForm(InterfaceCommonForm, ComponentForm, forms.Form): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) @@ -2366,6 +2382,10 @@ class InterfaceCreateForm(InterfaceCommonForm, ComponentForm, forms.Form): self.fields['lag'].queryset = Interface.objects.filter( device__in=[self.parent, self.parent.get_vc_master()], type=IFACE_TYPE_LAG ) + + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', self.parent.site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', self.parent.site.pk) else: self.fields['lag'].queryset = Interface.objects.none() @@ -2420,7 +2440,10 @@ class InterfaceBulkEditForm(InterfaceCommonForm, BootstrapMixin, AddRemoveTagsFo widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = forms.ModelMultipleChoiceField( @@ -2429,7 +2452,10 @@ class InterfaceBulkEditForm(InterfaceCommonForm, BootstrapMixin, AddRemoveTagsFo widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) @@ -2448,6 +2474,10 @@ class InterfaceBulkEditForm(InterfaceCommonForm, BootstrapMixin, AddRemoveTagsFo device__in=[device, device.get_vc_master()], type=IFACE_TYPE_LAG ) + + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', device.site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', device.site.pk) else: self.fields['lag'].choices = [] diff --git a/netbox/project-static/js/forms.js b/netbox/project-static/js/forms.js index b7dbb1cfa..1fbd211a7 100644 --- a/netbox/project-static/js/forms.js +++ b/netbox/project-static/js/forms.js @@ -187,15 +187,18 @@ $(document).ready(function() { $.each(element.attributes, function(index, attr){ if (attr.name.includes("data-additional-query-param-")){ var param_name = attr.name.split("data-additional-query-param-")[1]; - if (param_name in parameters) { - if (Array.isArray(parameters[param_name])) { - parameters[param_name].push(attr.value) + + $.each($.parseJSON(attr.value), function(index, value) { + if (param_name in parameters) { + if (Array.isArray(parameters[param_name])) { + parameters[param_name].push(value) + } else { + parameters[param_name] = [parameters[param_name], value] + } } else { - parameters[param_name] = [parameters[param_name], attr.value] + parameters[param_name] = value; } - } else { - parameters[param_name] = attr.value; - } + }) } }); diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index 39422c265..ba16774bb 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -346,12 +346,17 @@ class APISelect(SelectWithDisabled): def add_additional_query_param(self, name, value): """ - Add details for an additional query param in the form of a data-* attribute. + Add details for an additional query param in the form of a data-* JSON-encoded list attribute. :param name: The name of the query param :param value: The value of the query param """ - self.attrs['data-additional-query-param-{}'.format(name)] = value + key = 'data-additional-query-param-{}'.format(name) + + values = json.loads(self.attrs.get(key, '[]')) + values.append(value) + + self.attrs[key] = json.dumps(values) def add_conditional_query_param(self, condition, value): """ From f7e28de1d35817053c507b01b17d70f0dce7005a Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Wed, 15 Jan 2020 12:38:09 +0000 Subject: [PATCH 02/28] Semicolons for completeness --- netbox/project-static/js/forms.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/netbox/project-static/js/forms.js b/netbox/project-static/js/forms.js index 1fbd211a7..60bc32849 100644 --- a/netbox/project-static/js/forms.js +++ b/netbox/project-static/js/forms.js @@ -191,14 +191,14 @@ $(document).ready(function() { $.each($.parseJSON(attr.value), function(index, value) { if (param_name in parameters) { if (Array.isArray(parameters[param_name])) { - parameters[param_name].push(value) + parameters[param_name].push(value); } else { - parameters[param_name] = [parameters[param_name], value] + parameters[param_name] = [parameters[param_name], value]; } } else { parameters[param_name] = value; } - }) + }); } }); From a5425a740b842ec15c4dbe44ee452025e5159bf2 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 16 Jan 2020 15:10:25 +0000 Subject: [PATCH 03/28] Added #3840 changelog --- docs/release-notes/version-2.7.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index ac9d81e2c..5bf9fc314 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -237,6 +237,7 @@ PATCH) to maintain backward compatibility. This behavior will be discontinued be * [#3706](https://github.com/netbox-community/netbox/issues/3706) - Increase `available_power` maximum value on PowerFeed * [#3731](https://github.com/netbox-community/netbox/issues/3731) - Change Graph.type to a ContentType foreign key field * [#3801](https://github.com/netbox-community/netbox/issues/3801) - Use YAML for export of device types +* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Only show the valid list of interface VLAN choices ## Bug Fixes From 9ae8c7e0a67a488cbd738ad1c80b1e156902287d Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 16 Jan 2020 21:51:37 +0000 Subject: [PATCH 04/28] Moved into v2.7.1 --- docs/release-notes/version-2.7.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 5bf9fc314..45223a056 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,3 +1,11 @@ +# v2.7.1 (FUTURE) + +## Enhancements + +* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Only show the valid list of interface VLAN choices + +--- + # v2.7.0 (FUTURE) **Note:** NetBox v2.7 is the last major release that will support Python 3.5. Beginning with NetBox v2.8, Python 3.6 or @@ -237,7 +245,6 @@ PATCH) to maintain backward compatibility. This behavior will be discontinued be * [#3706](https://github.com/netbox-community/netbox/issues/3706) - Increase `available_power` maximum value on PowerFeed * [#3731](https://github.com/netbox-community/netbox/issues/3731) - Change Graph.type to a ContentType foreign key field * [#3801](https://github.com/netbox-community/netbox/issues/3801) - Use YAML for export of device types -* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Only show the valid list of interface VLAN choices ## Bug Fixes From 375925488621a9ddd5be7a82acc7705deb20a81a Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 30 Jan 2020 18:10:39 +0000 Subject: [PATCH 05/28] Corrected linter warning --- netbox/dcim/models/device_components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 68bab8037..e37569f79 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -676,7 +676,7 @@ class Interface(CableTermination, ComponentModel): self.untagged_vlan = None # Only "tagged" interfaces may have tagged VLANs assigned. ("tagged all" implies all VLANs are assigned.) - if self.pk and self.mode is not InterfaceModeChoices.MODE_TAGGED: + if self.pk and self.mode != InterfaceModeChoices.MODE_TAGGED: self.tagged_vlans.clear() return super().save(*args, **kwargs) From 779fde55bb8cd834c9eb8378d8af3634f9927a30 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 30 Jan 2020 18:26:30 +0000 Subject: [PATCH 06/28] Virtualization interfaces VLAN filtering --- netbox/virtualization/forms.py | 143 +++++++++------------------------ 1 file changed, 38 insertions(+), 105 deletions(-) diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index ae516fcb3..018e14e85 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -648,7 +648,10 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = forms.ModelMultipleChoiceField( @@ -657,7 +660,10 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tags = TagField( @@ -685,51 +691,12 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # Limit VLan choices to those in: global vlans, global groups, the current site's group, the current site - vlan_choices = [] - global_vlans = VLAN.objects.filter(site=None, group=None) - vlan_choices.append( - ('Global', [(vlan.pk, vlan) for vlan in global_vlans]) - ) - for group in VLANGroup.objects.filter(site=None): - global_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append( - (group.name, [(vlan.pk, vlan) for vlan in global_group_vlans]) - ) - + # Add current site to VLANs query params site = getattr(self.instance.parent, 'site', None) if site is not None: - - # Add non-grouped site VLANs - site_vlans = VLAN.objects.filter(site=site, group=None) - vlan_choices.append((site.name, [(vlan.pk, vlan) for vlan in site_vlans])) - - # Add grouped site VLANs - for group in VLANGroup.objects.filter(site=site): - site_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append(( - '{} / {}'.format(group.site.name, group.name), - [(vlan.pk, vlan) for vlan in site_group_vlans] - )) - - self.fields['untagged_vlan'].choices = [(None, '---------')] + vlan_choices - self.fields['tagged_vlans'].choices = vlan_choices - - def clean(self): - super().clean() - - # Validate VLAN assignments - tagged_vlans = self.cleaned_data['tagged_vlans'] - - # Untagged interfaces cannot be assigned tagged VLANs - if self.cleaned_data['mode'] == InterfaceModeChoices.MODE_ACCESS and tagged_vlans: - raise forms.ValidationError({ - 'mode': "An access interface cannot have tagged VLANs assigned." - }) - - # Remove all tagged VLAN assignments from "tagged all" interfaces - elif self.cleaned_data['mode'] == InterfaceModeChoices.MODE_TAGGED_ALL: - self.cleaned_data['tagged_vlans'] = [] + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk) class InterfaceCreateForm(ComponentForm): @@ -769,7 +736,10 @@ class InterfaceCreateForm(ComponentForm): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = forms.ModelMultipleChoiceField( @@ -778,7 +748,10 @@ class InterfaceCreateForm(ComponentForm): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tags = TagField( @@ -793,35 +766,12 @@ class InterfaceCreateForm(ComponentForm): super().__init__(*args, **kwargs) - # Limit VLan choices to those in: global vlans, global groups, the current site's group, the current site - vlan_choices = [] - global_vlans = VLAN.objects.filter(site=None, group=None) - vlan_choices.append( - ('Global', [(vlan.pk, vlan) for vlan in global_vlans]) - ) - for group in VLANGroup.objects.filter(site=None): - global_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append( - (group.name, [(vlan.pk, vlan) for vlan in global_group_vlans]) - ) - + # Add current site to VLANs query params site = getattr(self.parent.cluster, 'site', None) if site is not None: - - # Add non-grouped site VLANs - site_vlans = VLAN.objects.filter(site=site, group=None) - vlan_choices.append((site.name, [(vlan.pk, vlan) for vlan in site_vlans])) - - # Add grouped site VLANs - for group in VLANGroup.objects.filter(site=site): - site_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append(( - '{} / {}'.format(group.site.name, group.name), - [(vlan.pk, vlan) for vlan in site_group_vlans] - )) - - self.fields['untagged_vlan'].choices = [(None, '---------')] + vlan_choices - self.fields['tagged_vlans'].choices = vlan_choices + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk) class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm): @@ -854,7 +804,10 @@ class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = forms.ModelMultipleChoiceField( @@ -863,7 +816,10 @@ class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) @@ -875,35 +831,12 @@ class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # Limit VLan choices to those in: global vlans, global groups, the current site's group, the current site - vlan_choices = [] - global_vlans = VLAN.objects.filter(site=None, group=None) - vlan_choices.append( - ('Global', [(vlan.pk, vlan) for vlan in global_vlans]) - ) - for group in VLANGroup.objects.filter(site=None): - global_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append( - (group.name, [(vlan.pk, vlan) for vlan in global_group_vlans]) - ) - if self.parent_obj.cluster is not None: - site = getattr(self.parent_obj.cluster, 'site', None) - if site is not None: - - # Add non-grouped site VLANs - site_vlans = VLAN.objects.filter(site=site, group=None) - vlan_choices.append((site.name, [(vlan.pk, vlan) for vlan in site_vlans])) - - # Add grouped site VLANs - for group in VLANGroup.objects.filter(site=site): - site_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append(( - '{} / {}'.format(group.site.name, group.name), - [(vlan.pk, vlan) for vlan in site_group_vlans] - )) - - self.fields['untagged_vlan'].choices = [(None, '---------')] + vlan_choices - self.fields['tagged_vlans'].choices = vlan_choices + # Add current site to VLANs query params + site = getattr(self.parent_obj.cluster, 'site', None) + if site is not None: + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk) # From 1d9b66cc4c287375df18950219a3d31e1bdf2254 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 30 Jan 2020 18:29:08 +0000 Subject: [PATCH 07/28] Removed changelog to avoid merge conflicts --- docs/release-notes/version-2.7.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 8caf5c17b..5c489a96c 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -4,7 +4,6 @@ * [#3310](https://github.com/netbox-community/netbox/issues/3310) - Pre-select site/rack for B side when creating a new cable * [#3509](https://github.com/netbox-community/netbox/issues/3509) - Add IP address variables for custom scripts -* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Only show the valid list of interface VLAN choices * [#4005](https://github.com/netbox-community/netbox/issues/4005) - Include timezone context in webhook timestamps ## Bug Fixes From dccd0dcc9cf73b98b3e2a1034a11789d3956b944 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Sat, 8 Feb 2020 16:18:58 +0000 Subject: [PATCH 08/28] Cleaned duplicate code --- netbox/dcim/forms.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index ed42e9914..52047151b 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -2871,18 +2871,16 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # Limit LAG choices to interfaces belonging to this device (or VC master) if self.is_bound: device = Device.objects.get(pk=self.data['device']) - self.fields['lag'].queryset = Interface.objects.filter( - device__in=[device, device.get_vc_master()], - type=InterfaceTypeChoices.TYPE_LAG - ) else: - self.fields['lag'].queryset = Interface.objects.filter( - device__in=[self.instance.device, self.instance.device.get_vc_master()], - type=InterfaceTypeChoices.TYPE_LAG - ) + device = self.instance.device + + # Limit LAG choices to interfaces belonging to this device (or VC master) + self.fields['lag'].queryset = Interface.objects.filter( + device__in=[device, device.get_vc_master()], + type=InterfaceTypeChoices.TYPE_LAG + ) # Add current site to VLANs query params self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', device.site.pk) From 7acda4fa3761472e972684124d3692c3cc6306ed Mon Sep 17 00:00:00 2001 From: Matt Olenik Date: Fri, 13 Dec 2019 11:52:59 -0800 Subject: [PATCH 09/28] Fix race condition in available-prefix/ip APIs Implement advisory lock to prevent duplicate records being inserted when making simultaneous calls. Fixes #2519 --- base_requirements.txt | 4 ++++ netbox/ipam/api/views.py | 10 ++++++++++ netbox/utilities/constants.py | 11 +++++++++++ requirements.txt | 1 + 4 files changed, 26 insertions(+) diff --git a/base_requirements.txt b/base_requirements.txt index 8b42c835d..ed42b6c08 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -22,6 +22,10 @@ django-filter # https://github.com/django-mptt/django-mptt django-mptt +# Context managers for PostgreSQL advisory locks +# https://github.com/Xof/django-pglocks +django-pglocks + # Prometheus metrics library for Django # https://github.com/korfuri/django-prometheus django-prometheus diff --git a/netbox/ipam/api/views.py b/netbox/ipam/api/views.py index 08e21367c..262ca7908 100644 --- a/netbox/ipam/api/views.py +++ b/netbox/ipam/api/views.py @@ -1,6 +1,7 @@ from django.conf import settings from django.db.models import Count from django.shortcuts import get_object_or_404 +from django_pglocks import advisory_lock from rest_framework import status from rest_framework.decorators import action from rest_framework.exceptions import PermissionDenied @@ -10,6 +11,7 @@ from extras.api.views import CustomFieldModelViewSet from ipam import filters from ipam.models import Aggregate, IPAddress, Prefix, RIR, Role, Service, VLAN, VLANGroup, VRF from utilities.api import FieldChoicesViewSet, ModelViewSet +from utilities.constants import ADVISORY_LOCK_KEYS from utilities.utils import get_subquery from . import serializers @@ -86,9 +88,13 @@ class PrefixViewSet(CustomFieldModelViewSet): filterset_class = filters.PrefixFilterSet @action(detail=True, url_path='available-prefixes', methods=['get', 'post']) + @advisory_lock(ADVISORY_LOCK_KEYS['available-prefixes']) def available_prefixes(self, request, pk=None): """ A convenience method for returning available child prefixes within a parent. + + The advisory lock decorator uses a PostgreSQL advisory lock to prevent this API from being + invoked in parallel, which results in a race condition where multiple insertions can occur. """ prefix = get_object_or_404(Prefix, pk=pk) available_prefixes = prefix.get_available_prefixes() @@ -180,11 +186,15 @@ class PrefixViewSet(CustomFieldModelViewSet): return Response(serializer.data) @action(detail=True, url_path='available-ips', methods=['get', 'post']) + @advisory_lock(ADVISORY_LOCK_KEYS['available-ips']) def available_ips(self, request, pk=None): """ A convenience method for returning available IP addresses within a prefix. By default, the number of IPs returned will be equivalent to PAGINATE_COUNT. An arbitrary limit (up to MAX_PAGE_SIZE, if set) may be passed, however results will not be paginated. + + The advisory lock decorator uses a PostgreSQL advisory lock to prevent this API from being + invoked in parallel, which results in a race condition where multiple insertions can occur. """ prefix = get_object_or_404(Prefix, pk=pk) diff --git a/netbox/utilities/constants.py b/netbox/utilities/constants.py index ad6e8fd90..2cec2b532 100644 --- a/netbox/utilities/constants.py +++ b/netbox/utilities/constants.py @@ -27,3 +27,14 @@ COLOR_CHOICES = ( ('111111', 'Black'), ('ffffff', 'White'), ) + +# Keys for PostgreSQL advisory locks. These are arbitrary bigints used by +# the advisory_lock contextmanager. When a lock is acquired, +# one of these keys will be used to identify said lock. +# +# When adding a new key, pick something arbitrary and unique so +# that it is easily searchable in query logs. +ADVISORY_LOCK_KEYS = { + 'available-prefixes': 100100, + 'available-ips': 100200, +} diff --git a/requirements.txt b/requirements.txt index 9c5f624f9..b0b1b971d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ django-cors-headers==3.2.1 django-debug-toolbar==2.1 django-filter==2.2.0 django-mptt==0.9.1 +django-pglocks==1.0.4 django-prometheus==1.1.0 django-rq==2.2.0 django-tables2==2.2.1 From 79d49497ccfa76b02766ec1b3cdb271e069b6df5 Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Thu, 13 Feb 2020 08:53:46 -0500 Subject: [PATCH 10/28] allow for redis sentinel connection --- netbox/netbox/configuration.example.py | 6 +++ netbox/netbox/settings.py | 55 ++++++++++++++++++++------ 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index 885c9e0ba..7002def9b 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -28,6 +28,9 @@ REDIS = { 'webhooks': { 'HOST': 'localhost', 'PORT': 6379, + # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '', 'DATABASE': 0, 'DEFAULT_TIMEOUT': 300, @@ -36,6 +39,9 @@ REDIS = { 'caching': { 'HOST': 'localhost', 'PORT': 6379, + # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '', 'DATABASE': 1, 'DEFAULT_TIMEOUT': 300, diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 305f7efa8..4b6177318 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -170,14 +170,27 @@ if 'caching' not in REDIS: WEBHOOKS_REDIS = REDIS.get('webhooks', {}) WEBHOOKS_REDIS_HOST = WEBHOOKS_REDIS.get('HOST', 'localhost') WEBHOOKS_REDIS_PORT = WEBHOOKS_REDIS.get('PORT', 6379) +WEBHOOKS_REDIS_SENTINELS = WEBHOOKS_REDIS.get('SENTINELS', []) +WEBHOOKS_REDIS_USING_SENTINEL = all([ + isinstance(WEBHOOKS_REDIS_SENTINELS, (list, tuple)), + len(WEBHOOKS_REDIS_SENTINELS) > 0 +]) +WEBHOOKS_REDIS_SENTINEL_SERVICE = WEBHOOKS_REDIS.get('SENTINEL_SERVICE', 'default') WEBHOOKS_REDIS_PASSWORD = WEBHOOKS_REDIS.get('PASSWORD', '') WEBHOOKS_REDIS_DATABASE = WEBHOOKS_REDIS.get('DATABASE', 0) WEBHOOKS_REDIS_DEFAULT_TIMEOUT = WEBHOOKS_REDIS.get('DEFAULT_TIMEOUT', 300) WEBHOOKS_REDIS_SSL = WEBHOOKS_REDIS.get('SSL', False) + CACHING_REDIS = REDIS.get('caching', {}) CACHING_REDIS_HOST = CACHING_REDIS.get('HOST', 'localhost') CACHING_REDIS_PORT = CACHING_REDIS.get('PORT', 6379) +CACHING_REDIS_SENTINELS = CACHING_REDIS.get('SENTINELS', []) +CACHING_REDIS_USING_SENTINEL = all([ + isinstance(CACHING_REDIS_SENTINELS, (list, tuple)), + len(CACHING_REDIS_SENTINELS) > 0 +]) +CACHING_REDIS_SENTINEL_SERVICE = CACHING_REDIS.get('SENTINEL_SERVICE', 'default') CACHING_REDIS_PASSWORD = CACHING_REDIS.get('PASSWORD', '') CACHING_REDIS_DATABASE = CACHING_REDIS.get('DATABASE', 0) CACHING_REDIS_DEFAULT_TIMEOUT = CACHING_REDIS.get('DEFAULT_TIMEOUT', 300) @@ -394,28 +407,35 @@ if LDAP_CONFIG is not None: # # Caching # - -if CACHING_REDIS_SSL: - REDIS_CACHE_CON_STRING = 'rediss://' +if CACHING_REDIS_USING_SENTINEL: + CACHEOPS_SENTINEL = { + 'locations': CACHING_REDIS_SENTINELS, + 'service_name': CACHING_REDIS_SENTINEL_SERVICE, + 'db': CACHING_REDIS_DATABASE, + } else: - REDIS_CACHE_CON_STRING = 'redis://' + if CACHING_REDIS_SSL: + REDIS_CACHE_CON_STRING = 'rediss://' + else: + REDIS_CACHE_CON_STRING = 'redis://' -if CACHING_REDIS_PASSWORD: - REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, CACHING_REDIS_PASSWORD) + if CACHING_REDIS_PASSWORD: + REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, CACHING_REDIS_PASSWORD) -REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format( - REDIS_CACHE_CON_STRING, - CACHING_REDIS_HOST, - CACHING_REDIS_PORT, - CACHING_REDIS_DATABASE -) + REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format( + REDIS_CACHE_CON_STRING, + CACHING_REDIS_HOST, + CACHING_REDIS_PORT, + CACHING_REDIS_DATABASE + ) + CACHEOPS_REDIS = REDIS_CACHE_CON_STRING if not CACHE_TIMEOUT: CACHEOPS_ENABLED = False else: CACHEOPS_ENABLED = True -CACHEOPS_REDIS = REDIS_CACHE_CON_STRING + CACHEOPS_DEFAULTS = { 'timeout': CACHE_TIMEOUT } @@ -534,6 +554,15 @@ RQ_QUEUES = { 'PASSWORD': WEBHOOKS_REDIS_PASSWORD, 'DEFAULT_TIMEOUT': WEBHOOKS_REDIS_DEFAULT_TIMEOUT, 'SSL': WEBHOOKS_REDIS_SSL, + } if not WEBHOOKS_REDIS_USING_SENTINEL else { + 'SENTINELS': WEBHOOKS_REDIS_SENTINELS, + 'MASTER_NAME': WEBHOOKS_REDIS_SENTINEL_SERVICE, + 'DB': WEBHOOKS_REDIS_DATABASE, + 'PASSWORD': WEBHOOKS_REDIS_PASSWORD, + 'SOCKET_TIMEOUT': None, + 'CONNECTION_KWARGS': { + 'socket_connect_timeout': WEBHOOKS_REDIS_DEFAULT_TIMEOUT + }, } } From addc7e507eefc7a7130a7d5eb17d49d08579319a Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Thu, 13 Feb 2020 09:10:53 -0500 Subject: [PATCH 11/28] documentation on redis sentinel --- docs/configuration/required-settings.md | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/configuration/required-settings.md b/docs/configuration/required-settings.md index 70c6de276..81790eae0 100644 --- a/docs/configuration/required-settings.md +++ b/docs/configuration/required-settings.md @@ -88,6 +88,48 @@ REDIS = { It is highly recommended to keep the webhook and cache databases separate. Using the same database number on the same Redis instance for both may result in webhook processing data being lost during cache flushing events. +### Using Redis Sentinel + +If you are using [Redis Sentinel](https://redis.io/topics/sentinel) for high-availability purposes, there is minimal +configuration necessary to convert NetBox to recognize it. It requires the removal of the `HOST` and `PORT` keys from +above and the addition of two new keys. + +* `SENTINELS`: List of tuples or tuple of tuples with each inner tuple containing the name or IP address +of the Redis server and port for each sentinel instance to connect to +* `SENTINEL_SERVICE`: Name of the master / service to connect to + +Example: + +```python +REDIS = { + 'webhooks': { + 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'SENTINELS': [ + ('mysentinel.redis.example.com', 6379), + ('othersentinel.redis.example.com', 6379) + ], + 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } +} +``` + +!!! note: + It is possible to have only one or the other Redis configurations to use Sentinel functionality. It is possible + for example to have the webhook use sentinel via `HOST`/`PORT` and for caching to use Sentinel via + `SENTINELS`/`SENTINEL_SERVICE`. + + --- ## SECRET_KEY From a90cf4991dbe5586c221567d609260c56f9d0bbb Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Thu, 13 Feb 2020 10:27:56 -0500 Subject: [PATCH 12/28] fixed extraneous linting error --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 4b6177318..1a36cf424 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -409,7 +409,7 @@ if LDAP_CONFIG is not None: # if CACHING_REDIS_USING_SENTINEL: CACHEOPS_SENTINEL = { - 'locations': CACHING_REDIS_SENTINELS, + 'locations': CACHING_REDIS_SENTINELS, 'service_name': CACHING_REDIS_SENTINEL_SERVICE, 'db': CACHING_REDIS_DATABASE, } From 3cee7307ad623b6176954128803deb335d639e28 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 15:13:43 -0500 Subject: [PATCH 13/28] Add tests for naturalization functions --- netbox/utilities/ordering.py | 2 +- netbox/utilities/tests/test_ordering.py | 43 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 netbox/utilities/tests/test_ordering.py diff --git a/netbox/utilities/ordering.py b/netbox/utilities/ordering.py index d459e6f6c..a560e776e 100644 --- a/netbox/utilities/ordering.py +++ b/netbox/utilities/ordering.py @@ -68,7 +68,7 @@ def naturalize_interface(value, max_length=None): if match.group('type') is not None: output.append(match.group('type')) - # Finally, append any remaining fields, left-padding to eight digits each. + # Finally, append any remaining fields, left-padding to six digits each. for part_name in ('id', 'channel', 'vc'): part = match.group(part_name) if part is not None: diff --git a/netbox/utilities/tests/test_ordering.py b/netbox/utilities/tests/test_ordering.py new file mode 100644 index 000000000..a875c688c --- /dev/null +++ b/netbox/utilities/tests/test_ordering.py @@ -0,0 +1,43 @@ +from django.test import TestCase + +from utilities.ordering import naturalize, naturalize_interface + + +class NaturalizationTestCase(TestCase): + """ + Validate the operation of the functions which generate values suitable for natural ordering. + """ + def test_naturalize(self): + + data = ( + # Original, naturalized + ('abc', 'abc'), + ('123', '00000123'), + ('abc123', 'abc00000123'), + ('123abc', '00000123abc'), + ('123abc456', '00000123abc00000456'), + ('abc123def', 'abc00000123def'), + ('abc123def456', 'abc00000123def00000456'), + ) + + for origin, naturalized in data: + self.assertEqual(naturalize(origin), naturalized) + + def test_naturalize_interface(self): + + data = ( + # Original, naturalized + ('Gi', '9999999999999999Gi000000000000000000'), + ('Gi1', '9999999999999999Gi000001000000000000'), + ('Gi1/2', '0001999999999999Gi000002000000000000'), + ('Gi1/2/3', '0001000299999999Gi000003000000000000'), + ('Gi1/2/3/4', '0001000200039999Gi000004000000000000'), + ('Gi1/2/3/4/5', '0001000200030004Gi000005000000000000'), + ('Gi1/2/3/4/5:6', '0001000200030004Gi000005000006000000'), + ('Gi1/2/3/4/5:6.7', '0001000200030004Gi000005000006000007'), + ('Gi1:2', '9999999999999999Gi000001000002000000'), + ('Gi1:2.3', '9999999999999999Gi000001000002000003'), + ) + + for origin, naturalized in data: + self.assertEqual(naturalize_interface(origin), naturalized) From e09ab1b6de646aadc4f14eaf31fa3421aaf334f5 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 15:16:07 -0500 Subject: [PATCH 14/28] Changelog for #3984 --- docs/release-notes/version-2.7.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 63745bbab..ea200c462 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -4,6 +4,7 @@ * [#3766](https://github.com/netbox-community/netbox/issues/3766) - Allow custom script authors to specify the form widget for each variable * [#3799](https://github.com/netbox-community/netbox/issues/3799) - Greatly improve performance when ordering device components +* [#3984](https://github.com/netbox-community/netbox/issues/3984) - Add support for Redis Sentinel * [#3986](https://github.com/netbox-community/netbox/issues/3986) - Include position numbers in SVG image when rendering rack elevation * [#4093](https://github.com/netbox-community/netbox/issues/4093) - Add multiple status choices for VMs * [#4100](https://github.com/netbox-community/netbox/issues/4100) - Add device filter to component list views From bd49364f8b8ce81859966756a73b445611c84863 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 15:32:09 -0500 Subject: [PATCH 15/28] Release v2.7.5 --- docs/release-notes/version-2.7.md | 14 ++++++++------ netbox/netbox/settings.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index ea200c462..04b3972ca 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,12 +1,14 @@ -# v2.7.5 (FUTURE) +# v2.7.5 (2020-02-13) + +**Note:** This release includes several database schema migrations that calculate and store copies of names for certain objects to improve natural ordering performance (see [#3799](https://github.com/netbox-community/netbox/issues/3799)). These migrations may take a few minutes to run if you have a very large number of objects defined in NetBox. ## Enhancements * [#3766](https://github.com/netbox-community/netbox/issues/3766) - Allow custom script authors to specify the form widget for each variable * [#3799](https://github.com/netbox-community/netbox/issues/3799) - Greatly improve performance when ordering device components * [#3984](https://github.com/netbox-community/netbox/issues/3984) - Add support for Redis Sentinel -* [#3986](https://github.com/netbox-community/netbox/issues/3986) - Include position numbers in SVG image when rendering rack elevation -* [#4093](https://github.com/netbox-community/netbox/issues/4093) - Add multiple status choices for VMs +* [#3986](https://github.com/netbox-community/netbox/issues/3986) - Include position numbers in SVG image when rendering rack elevations +* [#4093](https://github.com/netbox-community/netbox/issues/4093) - Add more status choices for virtual machines * [#4100](https://github.com/netbox-community/netbox/issues/4100) - Add device filter to component list views * [#4113](https://github.com/netbox-community/netbox/issues/4113) - Add bulk edit functionality for device type components * [#4116](https://github.com/netbox-community/netbox/issues/4116) - Enable bulk edit and delete functions for device component list views @@ -14,8 +16,8 @@ ## Bug Fixes -* [#3507](https://github.com/netbox-community/netbox/issues/3507) - Fix filtering IPaddress by multiple devices -* [#3995](https://github.com/netbox-community/netbox/issues/3995) - Make dropdown menus in the navigation bar scrollable +* [#3507](https://github.com/netbox-community/netbox/issues/3507) - Fix filtering IP addresses by multiple devices +* [#3995](https://github.com/netbox-community/netbox/issues/3995) - Make dropdown menus in the navigation bar scrollable on small screens * [#4083](https://github.com/netbox-community/netbox/issues/4083) - Permit nullifying applicable choice fields via API requests * [#4089](https://github.com/netbox-community/netbox/issues/4089) - Selection of power outlet type during bulk update is optional * [#4090](https://github.com/netbox-community/netbox/issues/4090) - Render URL custom fields as links under object view @@ -24,7 +26,7 @@ * [#4108](https://github.com/netbox-community/netbox/issues/4108) - Avoid extraneous database queries when rendering search forms * [#4134](https://github.com/netbox-community/netbox/issues/4134) - Device power ports and outlets should inherit type from the parent device type * [#4137](https://github.com/netbox-community/netbox/issues/4137) - Disable occupied terminations when connecting a cable to a circuit -* [#4138](https://github.com/netbox-community/netbox/issues/4138) - Include device bay counts in rack elevation diagrams +* [#4138](https://github.com/netbox-community/netbox/issues/4138) - Restore device bay counts in rack elevation diagrams * [#4146](https://github.com/netbox-community/netbox/issues/4146) - Fix enforcement of secret role assignment for secret decryption * [#4150](https://github.com/netbox-community/netbox/issues/4150) - Correct YAML rendering of config contexts * [#4159](https://github.com/netbox-community/netbox/issues/4159) - Fix implementation of Redis caching configuration diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 1a36cf424..5f60d16f7 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -12,7 +12,7 @@ from django.core.exceptions import ImproperlyConfigured # Environment setup # -VERSION = '2.7.5-dev' +VERSION = '2.7.5' # Hostname HOSTNAME = platform.node() From 4bc94a7ecccd56f0b069ac0a50c293e248986080 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 15:39:08 -0500 Subject: [PATCH 16/28] Post-release version bump --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 5f60d16f7..f66828f69 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -12,7 +12,7 @@ from django.core.exceptions import ImproperlyConfigured # Environment setup # -VERSION = '2.7.5' +VERSION = '2.7.6-dev' # Hostname HOSTNAME = platform.node() From 5cd25b2721ec34f674767acb40807b36a6aefaed Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 21:41:00 -0500 Subject: [PATCH 17/28] Fixes #4166: Fix schema migrations to enforce maximum character length for naturalized fields --- docs/release-notes/version-2.7.md | 8 ++++++++ .../dcim/migrations/0093_device_component_ordering.py | 2 +- .../0094_device_component_template_ordering.py | 2 +- netbox/dcim/migrations/0095_primary_model_ordering.py | 2 +- netbox/dcim/migrations/0096_interface_ordering.py | 2 +- netbox/extras/management/commands/renaturalize.py | 2 +- netbox/utilities/ordering.py | 8 ++++---- netbox/utilities/tests/test_ordering.py | 10 ++++++++-- 8 files changed, 25 insertions(+), 11 deletions(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 04b3972ca..75269b6ad 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,3 +1,11 @@ +# v2.7.6 (FUTURE) + +## Bug Fixes + +* [#4166](https://github.com/netbox-community/netbox/issues/4166) - Fix schema migrations to enforce maximum character length for naturalized fields + +--- + # v2.7.5 (2020-02-13) **Note:** This release includes several database schema migrations that calculate and store copies of names for certain objects to improve natural ordering performance (see [#3799](https://github.com/netbox-community/netbox/issues/3799)). These migrations may take a few minutes to run if you have a very large number of objects defined in NetBox. diff --git a/netbox/dcim/migrations/0093_device_component_ordering.py b/netbox/dcim/migrations/0093_device_component_ordering.py index 017241c8b..4e3c941a1 100644 --- a/netbox/dcim/migrations/0093_device_component_ordering.py +++ b/netbox/dcim/migrations/0093_device_component_ordering.py @@ -6,7 +6,7 @@ import utilities.ordering def _update_model_names(model): # Update each unique field value in bulk for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): - model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name, max_length=100)) def naturalize_consoleports(apps, schema_editor): diff --git a/netbox/dcim/migrations/0094_device_component_template_ordering.py b/netbox/dcim/migrations/0094_device_component_template_ordering.py index fc39f76b2..24fe98e94 100644 --- a/netbox/dcim/migrations/0094_device_component_template_ordering.py +++ b/netbox/dcim/migrations/0094_device_component_template_ordering.py @@ -6,7 +6,7 @@ import utilities.ordering def _update_model_names(model): # Update each unique field value in bulk for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): - model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name, max_length=100)) def naturalize_consoleporttemplates(apps, schema_editor): diff --git a/netbox/dcim/migrations/0095_primary_model_ordering.py b/netbox/dcim/migrations/0095_primary_model_ordering.py index 9cef0a581..3bc780161 100644 --- a/netbox/dcim/migrations/0095_primary_model_ordering.py +++ b/netbox/dcim/migrations/0095_primary_model_ordering.py @@ -6,7 +6,7 @@ import utilities.ordering def _update_model_names(model): # Update each unique field value in bulk for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): - model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name, max_length=100)) def naturalize_sites(apps, schema_editor): diff --git a/netbox/dcim/migrations/0096_interface_ordering.py b/netbox/dcim/migrations/0096_interface_ordering.py index 284066462..f1622f504 100644 --- a/netbox/dcim/migrations/0096_interface_ordering.py +++ b/netbox/dcim/migrations/0096_interface_ordering.py @@ -6,7 +6,7 @@ import utilities.ordering def _update_model_names(model): # Update each unique field value in bulk for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): - model.objects.filter(name=name).update(_name=utilities.ordering.naturalize_interface(name)) + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize_interface(name, max_length=100)) def naturalize_interfacetemplates(apps, schema_editor): diff --git a/netbox/extras/management/commands/renaturalize.py b/netbox/extras/management/commands/renaturalize.py index 70f57c1ba..cfd037910 100644 --- a/netbox/extras/management/commands/renaturalize.py +++ b/netbox/extras/management/commands/renaturalize.py @@ -86,7 +86,7 @@ class Command(BaseCommand): # Find all unique values for the field queryset = model.objects.values_list(target_field, flat=True).order_by(target_field).distinct() for value in queryset: - naturalized_value = naturalize(value) + naturalized_value = naturalize(value, max_length=field.max_length) if options['verbosity'] >= 2: self.stdout.write(" {} -> {}".format(value, naturalized_value), ending='') diff --git a/netbox/utilities/ordering.py b/netbox/utilities/ordering.py index a560e776e..39e21733a 100644 --- a/netbox/utilities/ordering.py +++ b/netbox/utilities/ordering.py @@ -10,7 +10,7 @@ INTERFACE_NAME_REGEX = r'(^(?P[^\d\.:]+)?)' \ r'(.(?P\d+)$)?' -def naturalize(value, max_length=None, integer_places=8): +def naturalize(value, max_length, integer_places=8): """ Take an alphanumeric string and prepend all integers to `integer_places` places to ensure the strings are ordered naturally. For example: @@ -39,10 +39,10 @@ def naturalize(value, max_length=None, integer_places=8): output.append(segment) ret = ''.join(output) - return ret[:max_length] if max_length else ret + return ret[:max_length] -def naturalize_interface(value, max_length=None): +def naturalize_interface(value, max_length): """ Similar in nature to naturalize(), but takes into account a particular naming format adapted from the old InterfaceManager. @@ -77,4 +77,4 @@ def naturalize_interface(value, max_length=None): output.append('000000') ret = ''.join(output) - return ret[:max_length] if max_length else ret + return ret[:max_length] diff --git a/netbox/utilities/tests/test_ordering.py b/netbox/utilities/tests/test_ordering.py index a875c688c..958224c94 100644 --- a/netbox/utilities/tests/test_ordering.py +++ b/netbox/utilities/tests/test_ordering.py @@ -21,7 +21,10 @@ class NaturalizationTestCase(TestCase): ) for origin, naturalized in data: - self.assertEqual(naturalize(origin), naturalized) + self.assertEqual(naturalize(origin, max_length=50), naturalized) + + def test_naturalize_max_length(self): + self.assertEqual(naturalize('abc123def456', max_length=10), 'abc0000012') def test_naturalize_interface(self): @@ -40,4 +43,7 @@ class NaturalizationTestCase(TestCase): ) for origin, naturalized in data: - self.assertEqual(naturalize_interface(origin), naturalized) + self.assertEqual(naturalize_interface(origin, max_length=50), naturalized) + + def test_naturalize_interface_max_length(self): + self.assertEqual(naturalize_interface('Gi1/2/3', max_length=20), '0001000299999999Gi00') From ea21fddda6f8a2d4196b6496758d173c4b09d2ee Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 21:46:03 -0500 Subject: [PATCH 18/28] Release v2.7.6 --- docs/release-notes/version-2.7.md | 2 +- netbox/netbox/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 75269b6ad..70981658f 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,4 +1,4 @@ -# v2.7.6 (FUTURE) +# v2.7.6 (2020-02-13) ## Bug Fixes diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index f66828f69..81422b714 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -12,7 +12,7 @@ from django.core.exceptions import ImproperlyConfigured # Environment setup # -VERSION = '2.7.6-dev' +VERSION = '2.7.6' # Hostname HOSTNAME = platform.node() From 2b1d786fe21e1d1e1b4b2cb945a5144c76102e8e Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 21:51:03 -0500 Subject: [PATCH 19/28] Post-release version bump --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 81422b714..249ee9e53 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -12,7 +12,7 @@ from django.core.exceptions import ImproperlyConfigured # Environment setup # -VERSION = '2.7.6' +VERSION = '2.7.7-dev' # Hostname HOSTNAME = platform.node() From 98a59b91169bad27ee43ad8f086ace012cd5a882 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 14 Feb 2020 09:13:05 -0500 Subject: [PATCH 20/28] Fixes #4168: Role is not required when creating a virtual machine --- docs/release-notes/version-2.7.md | 8 ++++++++ netbox/virtualization/forms.py | 1 + 2 files changed, 9 insertions(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 70981658f..c235b8115 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,3 +1,11 @@ +# v2.7.7 (FUTURE) + +## Bug Fixes + +* [#4168](https://github.com/netbox-community/netbox/issues/4168) - Role is not required when creating a virtual machine + +--- + # v2.7.6 (2020-02-13) ## Bug Fixes diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index 12393d400..f35a94c99 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -351,6 +351,7 @@ class VirtualMachineForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): ) role = DynamicModelChoiceField( queryset=DeviceRole.objects.all(), + required=False, widget=APISelect( api_url="/api/dcim/device-roles/", additional_query_params={ From 2c27cc64dfb6f4df732ffb668b16774389066d18 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 14 Feb 2020 09:35:43 -0500 Subject: [PATCH 21/28] Changelog for #2519 --- docs/release-notes/version-2.7.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index c235b8115..ac2f149e1 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -2,6 +2,7 @@ ## Bug Fixes +* [#2519](https://github.com/netbox-community/netbox/issues/2519) - Avoid race condition when provisioning "next available" IPs/prefixes via the API * [#4168](https://github.com/netbox-community/netbox/issues/4168) - Role is not required when creating a virtual machine --- From c054240a69d8775ebc6941cd116cc41aa56d39ea Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Fri, 14 Feb 2020 09:39:01 -0500 Subject: [PATCH 22/28] ammend redis conn check to acccount for sentinel --- netbox/extras/apps.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/netbox/extras/apps.py b/netbox/extras/apps.py index f8c5a98e6..257e7801a 100644 --- a/netbox/extras/apps.py +++ b/netbox/extras/apps.py @@ -13,13 +13,23 @@ class ExtrasConfig(AppConfig): # Check that we can connect to the configured Redis database. try: - rs = redis.Redis( - host=settings.WEBHOOKS_REDIS_HOST, - port=settings.WEBHOOKS_REDIS_PORT, - db=settings.WEBHOOKS_REDIS_DATABASE, - password=settings.WEBHOOKS_REDIS_PASSWORD or None, - ssl=settings.WEBHOOKS_REDIS_SSL, - ) + if settings.WEBHOOKS_REDIS_USING_SENTINEL: + sentinel = redis.sentinel.Sentinel( + settings.WEBHOOKS_REDIS_SENTINELS, + socket_timeout=settings.WEBHOOKS_REDIS_DEFAULT_TIMEOUT + ) + rs = sentinel.master_for( + settings.WEBHOOKS_REDIS_SENTINEL_SERVICE, + socket_timeout=settings.WEBHOOKS_REDIS_DEFAULT_TIMEOUT + ) + else: + rs = redis.Redis( + host=settings.WEBHOOKS_REDIS_HOST, + port=settings.WEBHOOKS_REDIS_PORT, + db=settings.WEBHOOKS_REDIS_DATABASE, + password=settings.WEBHOOKS_REDIS_PASSWORD or None, + ssl=settings.WEBHOOKS_REDIS_SSL, + ) rs.ping() except redis.exceptions.ConnectionError: raise ImproperlyConfigured( From 93cc95076c556ab401bd5d5df8030271fad7ce7f Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Fri, 14 Feb 2020 10:29:09 -0500 Subject: [PATCH 23/28] fix extraneous formatting of notice boxes in required settings doc --- docs/configuration/required-settings.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration/required-settings.md b/docs/configuration/required-settings.md index 81790eae0..e86b2810a 100644 --- a/docs/configuration/required-settings.md +++ b/docs/configuration/required-settings.md @@ -80,11 +80,11 @@ REDIS = { } ``` -!!! note: +!!! note If you are upgrading from a version prior to v2.7, please note that the Redis connection configuration settings have changed. Manual modification to bring the `REDIS` section inline with the above specification is necessary -!!! warning: +!!! note It is highly recommended to keep the webhook and cache databases separate. Using the same database number on the same Redis instance for both may result in webhook processing data being lost during cache flushing events. @@ -124,7 +124,7 @@ REDIS = { } ``` -!!! note: +!!! note It is possible to have only one or the other Redis configurations to use Sentinel functionality. It is possible for example to have the webhook use sentinel via `HOST`/`PORT` and for caching to use Sentinel via `SENTINELS`/`SENTINEL_SERVICE`. From 574b00351897a917621227cd36c215d509443372 Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Fri, 14 Feb 2020 11:16:59 -0500 Subject: [PATCH 24/28] remove redis conn check from extras AppConfig --- netbox/extras/apps.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/netbox/extras/apps.py b/netbox/extras/apps.py index 257e7801a..5dad07b48 100644 --- a/netbox/extras/apps.py +++ b/netbox/extras/apps.py @@ -8,31 +8,4 @@ class ExtrasConfig(AppConfig): name = "extras" def ready(self): - import extras.signals - - # Check that we can connect to the configured Redis database. - try: - if settings.WEBHOOKS_REDIS_USING_SENTINEL: - sentinel = redis.sentinel.Sentinel( - settings.WEBHOOKS_REDIS_SENTINELS, - socket_timeout=settings.WEBHOOKS_REDIS_DEFAULT_TIMEOUT - ) - rs = sentinel.master_for( - settings.WEBHOOKS_REDIS_SENTINEL_SERVICE, - socket_timeout=settings.WEBHOOKS_REDIS_DEFAULT_TIMEOUT - ) - else: - rs = redis.Redis( - host=settings.WEBHOOKS_REDIS_HOST, - port=settings.WEBHOOKS_REDIS_PORT, - db=settings.WEBHOOKS_REDIS_DATABASE, - password=settings.WEBHOOKS_REDIS_PASSWORD or None, - ssl=settings.WEBHOOKS_REDIS_SSL, - ) - rs.ping() - except redis.exceptions.ConnectionError: - raise ImproperlyConfigured( - "Unable to connect to the Redis database. Check that the Redis configuration has been defined in " - "configuration.py." - ) From ae73c5f051b8f0f06b1b4a8aeaec3d33a8799838 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Fri, 14 Feb 2020 16:43:42 +0000 Subject: [PATCH 25/28] Added back clean --- netbox/virtualization/forms.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index 6771ee76b..2aa57608a 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -704,6 +704,22 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm): self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk) self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk) + def clean(self): + super().clean() + + # Validate VLAN assignments + tagged_vlans = self.cleaned_data['tagged_vlans'] + + # Untagged interfaces cannot be assigned tagged VLANs + if self.cleaned_data['mode'] == InterfaceModeChoices.MODE_ACCESS and tagged_vlans: + raise forms.ValidationError({ + 'mode': "An access interface cannot have tagged VLANs assigned." + }) + + # Remove all tagged VLAN assignments from "tagged all" interfaces + elif self.cleaned_data['mode'] == InterfaceModeChoices.MODE_TAGGED_ALL: + self.cleaned_data['tagged_vlans'] = [] + class InterfaceCreateForm(BootstrapMixin, forms.Form): virtual_machine = forms.ModelChoiceField( From 175ce21da9cf83c25cc9127134fe0f43390f7baa Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 14 Feb 2020 12:04:35 -0500 Subject: [PATCH 26/28] Changelog for #3840 --- docs/release-notes/version-2.7.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index ac2f149e1..5e58e0c58 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,5 +1,9 @@ # v2.7.7 (FUTURE) +## Enhancements + +* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Enhance search function when selecting VLANs for interface assignment + ## Bug Fixes * [#2519](https://github.com/netbox-community/netbox/issues/2519) - Avoid race condition when provisioning "next available" IPs/prefixes via the API From e870c4853ab6e8174af2204ff9577f7009ae348c Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 14 Feb 2020 12:04:56 -0500 Subject: [PATCH 27/28] Clean up unused imports --- netbox/extras/apps.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/netbox/extras/apps.py b/netbox/extras/apps.py index 5dad07b48..3201c3bb2 100644 --- a/netbox/extras/apps.py +++ b/netbox/extras/apps.py @@ -1,7 +1,4 @@ from django.apps import AppConfig -from django.conf import settings -from django.core.exceptions import ImproperlyConfigured -import redis class ExtrasConfig(AppConfig): From 9a6a1e99666fb2f18cd6c521edadc7e946705a19 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 14 Feb 2020 13:09:01 -0500 Subject: [PATCH 28/28] Closes #4170: Improve color contrast in rack elevation drawings --- docs/release-notes/version-2.7.md | 1 + netbox/dcim/models/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 5e58e0c58..d3b889513 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -3,6 +3,7 @@ ## Enhancements * [#3840](https://github.com/netbox-community/netbox/issues/3840) - Enhance search function when selecting VLANs for interface assignment +* [#4170](https://github.com/netbox-community/netbox/issues/4170) - Improve color contrast in rack elevation drawings ## Bug Fixes diff --git a/netbox/dcim/models/__init__.py b/netbox/dcim/models/__init__.py index f291fc825..29afef1f1 100644 --- a/netbox/dcim/models/__init__.py +++ b/netbox/dcim/models/__init__.py @@ -382,8 +382,8 @@ class RackElevationHelperMixin: # add gradients RackElevationHelperMixin._add_gradient(drawing, 'reserved', '#c7c7ff') - RackElevationHelperMixin._add_gradient(drawing, 'occupied', '#f0f0f0') - RackElevationHelperMixin._add_gradient(drawing, 'blocked', '#ffc7c7') + RackElevationHelperMixin._add_gradient(drawing, 'occupied', '#d7d7d7') + RackElevationHelperMixin._add_gradient(drawing, 'blocked', '#ffc0c0') return drawing