From 9d846d7b8729d70e86a106a15f751abc3136a856 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Wed, 15 Jan 2020 12:23:34 +0000 Subject: [PATCH 01/28] Fixes #3840: Only show valid interface VLAN choices --- docs/release-notes/version-2.6.md | 3 ++- netbox/dcim/forms.py | 42 ++++++++++++++++++++++++++----- netbox/project-static/js/forms.js | 17 +++++++------ netbox/utilities/forms.py | 9 +++++-- 4 files changed, 55 insertions(+), 16 deletions(-) diff --git a/docs/release-notes/version-2.6.md b/docs/release-notes/version-2.6.md index 792e8990a..b31e769a3 100644 --- a/docs/release-notes/version-2.6.md +++ b/docs/release-notes/version-2.6.md @@ -3,6 +3,7 @@ ## Enhancements * [#3525](https://github.com/netbox-community/netbox/issues/3525) - Enable IP address filtering with multiple address terms +* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Only show the valid list of interface VLAN choices ## Bug Fixes @@ -42,7 +43,7 @@ * [#3872](https://github.com/netbox-community/netbox/issues/3872) - Paginate related IPs on the IP address view * [#3876](https://github.com/netbox-community/netbox/issues/3876) - Fix minimum/maximum value rendering for site ASN field * [#3882](https://github.com/netbox-community/netbox/issues/3882) - Fix filtering of devices by rack group -* [#3898](https://github.com/netbox-community/netbox/issues/3898) - Fix references to deleted cables without a label +* [#3898](https://github.com/netbox-community/netbox/issues/3898) - Fix references to deleted cables without a label * [#3905](https://github.com/netbox-community/netbox/issues/3905) - Fix divide-by-zero on power feeds with low power values --- diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index cd356cc09..4b5dd33cf 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -2238,7 +2238,10 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = forms.ModelMultipleChoiceField( @@ -2247,7 +2250,10 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) @@ -2289,6 +2295,10 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm): device__in=[self.instance.device, self.instance.device.get_vc_master()], type=IFACE_TYPE_LAG ) + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', device.site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', device.site.pk) + class InterfaceCreateForm(InterfaceCommonForm, ComponentForm, forms.Form): name_pattern = ExpandableNameField( @@ -2340,7 +2350,10 @@ class InterfaceCreateForm(InterfaceCommonForm, ComponentForm, forms.Form): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = forms.ModelMultipleChoiceField( @@ -2349,7 +2362,10 @@ class InterfaceCreateForm(InterfaceCommonForm, ComponentForm, forms.Form): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) @@ -2366,6 +2382,10 @@ class InterfaceCreateForm(InterfaceCommonForm, ComponentForm, forms.Form): self.fields['lag'].queryset = Interface.objects.filter( device__in=[self.parent, self.parent.get_vc_master()], type=IFACE_TYPE_LAG ) + + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', self.parent.site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', self.parent.site.pk) else: self.fields['lag'].queryset = Interface.objects.none() @@ -2420,7 +2440,10 @@ class InterfaceBulkEditForm(InterfaceCommonForm, BootstrapMixin, AddRemoveTagsFo widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = forms.ModelMultipleChoiceField( @@ -2429,7 +2452,10 @@ class InterfaceBulkEditForm(InterfaceCommonForm, BootstrapMixin, AddRemoveTagsFo widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) @@ -2448,6 +2474,10 @@ class InterfaceBulkEditForm(InterfaceCommonForm, BootstrapMixin, AddRemoveTagsFo device__in=[device, device.get_vc_master()], type=IFACE_TYPE_LAG ) + + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', device.site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', device.site.pk) else: self.fields['lag'].choices = [] diff --git a/netbox/project-static/js/forms.js b/netbox/project-static/js/forms.js index b7dbb1cfa..1fbd211a7 100644 --- a/netbox/project-static/js/forms.js +++ b/netbox/project-static/js/forms.js @@ -187,15 +187,18 @@ $(document).ready(function() { $.each(element.attributes, function(index, attr){ if (attr.name.includes("data-additional-query-param-")){ var param_name = attr.name.split("data-additional-query-param-")[1]; - if (param_name in parameters) { - if (Array.isArray(parameters[param_name])) { - parameters[param_name].push(attr.value) + + $.each($.parseJSON(attr.value), function(index, value) { + if (param_name in parameters) { + if (Array.isArray(parameters[param_name])) { + parameters[param_name].push(value) + } else { + parameters[param_name] = [parameters[param_name], value] + } } else { - parameters[param_name] = [parameters[param_name], attr.value] + parameters[param_name] = value; } - } else { - parameters[param_name] = attr.value; - } + }) } }); diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index 39422c265..ba16774bb 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -346,12 +346,17 @@ class APISelect(SelectWithDisabled): def add_additional_query_param(self, name, value): """ - Add details for an additional query param in the form of a data-* attribute. + Add details for an additional query param in the form of a data-* JSON-encoded list attribute. :param name: The name of the query param :param value: The value of the query param """ - self.attrs['data-additional-query-param-{}'.format(name)] = value + key = 'data-additional-query-param-{}'.format(name) + + values = json.loads(self.attrs.get(key, '[]')) + values.append(value) + + self.attrs[key] = json.dumps(values) def add_conditional_query_param(self, condition, value): """ From 201416ba526dad9d0fa003bd12d63727f439107b Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Wed, 15 Jan 2020 12:38:09 +0000 Subject: [PATCH 02/28] Semicolons for completeness --- netbox/project-static/js/forms.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/netbox/project-static/js/forms.js b/netbox/project-static/js/forms.js index 1fbd211a7..60bc32849 100644 --- a/netbox/project-static/js/forms.js +++ b/netbox/project-static/js/forms.js @@ -191,14 +191,14 @@ $(document).ready(function() { $.each($.parseJSON(attr.value), function(index, value) { if (param_name in parameters) { if (Array.isArray(parameters[param_name])) { - parameters[param_name].push(value) + parameters[param_name].push(value); } else { - parameters[param_name] = [parameters[param_name], value] + parameters[param_name] = [parameters[param_name], value]; } } else { parameters[param_name] = value; } - }) + }); } }); From c8997868cee94c7ca8319e10725851578b30f375 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 16 Jan 2020 15:10:25 +0000 Subject: [PATCH 03/28] Added #3840 changelog --- docs/release-notes/version-2.7.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index ac9d81e2c..5bf9fc314 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -237,6 +237,7 @@ PATCH) to maintain backward compatibility. This behavior will be discontinued be * [#3706](https://github.com/netbox-community/netbox/issues/3706) - Increase `available_power` maximum value on PowerFeed * [#3731](https://github.com/netbox-community/netbox/issues/3731) - Change Graph.type to a ContentType foreign key field * [#3801](https://github.com/netbox-community/netbox/issues/3801) - Use YAML for export of device types +* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Only show the valid list of interface VLAN choices ## Bug Fixes From c31c8b1a2566b62b64ba0fad05d6341534aed365 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 16 Jan 2020 21:51:37 +0000 Subject: [PATCH 04/28] Moved into v2.7.1 --- docs/release-notes/version-2.7.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 5bf9fc314..45223a056 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,3 +1,11 @@ +# v2.7.1 (FUTURE) + +## Enhancements + +* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Only show the valid list of interface VLAN choices + +--- + # v2.7.0 (FUTURE) **Note:** NetBox v2.7 is the last major release that will support Python 3.5. Beginning with NetBox v2.8, Python 3.6 or @@ -237,7 +245,6 @@ PATCH) to maintain backward compatibility. This behavior will be discontinued be * [#3706](https://github.com/netbox-community/netbox/issues/3706) - Increase `available_power` maximum value on PowerFeed * [#3731](https://github.com/netbox-community/netbox/issues/3731) - Change Graph.type to a ContentType foreign key field * [#3801](https://github.com/netbox-community/netbox/issues/3801) - Use YAML for export of device types -* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Only show the valid list of interface VLAN choices ## Bug Fixes From ff822743cc1d4de33fe94a5cbb8700c97dfc63b3 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 30 Jan 2020 18:10:39 +0000 Subject: [PATCH 05/28] Corrected linter warning --- netbox/dcim/models/device_components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 68bab8037..e37569f79 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -676,7 +676,7 @@ class Interface(CableTermination, ComponentModel): self.untagged_vlan = None # Only "tagged" interfaces may have tagged VLANs assigned. ("tagged all" implies all VLANs are assigned.) - if self.pk and self.mode is not InterfaceModeChoices.MODE_TAGGED: + if self.pk and self.mode != InterfaceModeChoices.MODE_TAGGED: self.tagged_vlans.clear() return super().save(*args, **kwargs) From ae95b159bc033787886bc5b65ccf65bb1ebe840c Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 30 Jan 2020 18:26:30 +0000 Subject: [PATCH 06/28] Virtualization interfaces VLAN filtering --- netbox/virtualization/forms.py | 143 +++++++++------------------------ 1 file changed, 38 insertions(+), 105 deletions(-) diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index ae516fcb3..018e14e85 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -648,7 +648,10 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = forms.ModelMultipleChoiceField( @@ -657,7 +660,10 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tags = TagField( @@ -685,51 +691,12 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # Limit VLan choices to those in: global vlans, global groups, the current site's group, the current site - vlan_choices = [] - global_vlans = VLAN.objects.filter(site=None, group=None) - vlan_choices.append( - ('Global', [(vlan.pk, vlan) for vlan in global_vlans]) - ) - for group in VLANGroup.objects.filter(site=None): - global_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append( - (group.name, [(vlan.pk, vlan) for vlan in global_group_vlans]) - ) - + # Add current site to VLANs query params site = getattr(self.instance.parent, 'site', None) if site is not None: - - # Add non-grouped site VLANs - site_vlans = VLAN.objects.filter(site=site, group=None) - vlan_choices.append((site.name, [(vlan.pk, vlan) for vlan in site_vlans])) - - # Add grouped site VLANs - for group in VLANGroup.objects.filter(site=site): - site_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append(( - '{} / {}'.format(group.site.name, group.name), - [(vlan.pk, vlan) for vlan in site_group_vlans] - )) - - self.fields['untagged_vlan'].choices = [(None, '---------')] + vlan_choices - self.fields['tagged_vlans'].choices = vlan_choices - - def clean(self): - super().clean() - - # Validate VLAN assignments - tagged_vlans = self.cleaned_data['tagged_vlans'] - - # Untagged interfaces cannot be assigned tagged VLANs - if self.cleaned_data['mode'] == InterfaceModeChoices.MODE_ACCESS and tagged_vlans: - raise forms.ValidationError({ - 'mode': "An access interface cannot have tagged VLANs assigned." - }) - - # Remove all tagged VLAN assignments from "tagged all" interfaces - elif self.cleaned_data['mode'] == InterfaceModeChoices.MODE_TAGGED_ALL: - self.cleaned_data['tagged_vlans'] = [] + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk) class InterfaceCreateForm(ComponentForm): @@ -769,7 +736,10 @@ class InterfaceCreateForm(ComponentForm): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = forms.ModelMultipleChoiceField( @@ -778,7 +748,10 @@ class InterfaceCreateForm(ComponentForm): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tags = TagField( @@ -793,35 +766,12 @@ class InterfaceCreateForm(ComponentForm): super().__init__(*args, **kwargs) - # Limit VLan choices to those in: global vlans, global groups, the current site's group, the current site - vlan_choices = [] - global_vlans = VLAN.objects.filter(site=None, group=None) - vlan_choices.append( - ('Global', [(vlan.pk, vlan) for vlan in global_vlans]) - ) - for group in VLANGroup.objects.filter(site=None): - global_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append( - (group.name, [(vlan.pk, vlan) for vlan in global_group_vlans]) - ) - + # Add current site to VLANs query params site = getattr(self.parent.cluster, 'site', None) if site is not None: - - # Add non-grouped site VLANs - site_vlans = VLAN.objects.filter(site=site, group=None) - vlan_choices.append((site.name, [(vlan.pk, vlan) for vlan in site_vlans])) - - # Add grouped site VLANs - for group in VLANGroup.objects.filter(site=site): - site_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append(( - '{} / {}'.format(group.site.name, group.name), - [(vlan.pk, vlan) for vlan in site_group_vlans] - )) - - self.fields['untagged_vlan'].choices = [(None, '---------')] + vlan_choices - self.fields['tagged_vlans'].choices = vlan_choices + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk) class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm): @@ -854,7 +804,10 @@ class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm): widget=APISelect( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) tagged_vlans = forms.ModelMultipleChoiceField( @@ -863,7 +816,10 @@ class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm): widget=APISelectMultiple( api_url="/api/ipam/vlans/", display_field='display_name', - full=True + full=True, + additional_query_params={ + 'site_id': 'null', + }, ) ) @@ -875,35 +831,12 @@ class InterfaceBulkEditForm(BootstrapMixin, BulkEditForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # Limit VLan choices to those in: global vlans, global groups, the current site's group, the current site - vlan_choices = [] - global_vlans = VLAN.objects.filter(site=None, group=None) - vlan_choices.append( - ('Global', [(vlan.pk, vlan) for vlan in global_vlans]) - ) - for group in VLANGroup.objects.filter(site=None): - global_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append( - (group.name, [(vlan.pk, vlan) for vlan in global_group_vlans]) - ) - if self.parent_obj.cluster is not None: - site = getattr(self.parent_obj.cluster, 'site', None) - if site is not None: - - # Add non-grouped site VLANs - site_vlans = VLAN.objects.filter(site=site, group=None) - vlan_choices.append((site.name, [(vlan.pk, vlan) for vlan in site_vlans])) - - # Add grouped site VLANs - for group in VLANGroup.objects.filter(site=site): - site_group_vlans = VLAN.objects.filter(group=group) - vlan_choices.append(( - '{} / {}'.format(group.site.name, group.name), - [(vlan.pk, vlan) for vlan in site_group_vlans] - )) - - self.fields['untagged_vlan'].choices = [(None, '---------')] + vlan_choices - self.fields['tagged_vlans'].choices = vlan_choices + # Add current site to VLANs query params + site = getattr(self.parent_obj.cluster, 'site', None) + if site is not None: + # Add current site to VLANs query params + self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk) + self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk) # From ace8fac2c1232e5b93917d7c0c7afb24519c30be Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 30 Jan 2020 18:29:08 +0000 Subject: [PATCH 07/28] Removed changelog to avoid merge conflicts --- docs/release-notes/version-2.7.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 8caf5c17b..5c489a96c 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -4,7 +4,6 @@ * [#3310](https://github.com/netbox-community/netbox/issues/3310) - Pre-select site/rack for B side when creating a new cable * [#3509](https://github.com/netbox-community/netbox/issues/3509) - Add IP address variables for custom scripts -* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Only show the valid list of interface VLAN choices * [#4005](https://github.com/netbox-community/netbox/issues/4005) - Include timezone context in webhook timestamps ## Bug Fixes From 26ddd96e303b95e1c7f5224f652007a9bdebc941 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Sat, 8 Feb 2020 16:18:58 +0000 Subject: [PATCH 08/28] Cleaned duplicate code --- netbox/dcim/forms.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index ed42e9914..52047151b 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -2871,18 +2871,16 @@ class InterfaceForm(InterfaceCommonForm, BootstrapMixin, forms.ModelForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # Limit LAG choices to interfaces belonging to this device (or VC master) if self.is_bound: device = Device.objects.get(pk=self.data['device']) - self.fields['lag'].queryset = Interface.objects.filter( - device__in=[device, device.get_vc_master()], - type=InterfaceTypeChoices.TYPE_LAG - ) else: - self.fields['lag'].queryset = Interface.objects.filter( - device__in=[self.instance.device, self.instance.device.get_vc_master()], - type=InterfaceTypeChoices.TYPE_LAG - ) + device = self.instance.device + + # Limit LAG choices to interfaces belonging to this device (or VC master) + self.fields['lag'].queryset = Interface.objects.filter( + device__in=[device, device.get_vc_master()], + type=InterfaceTypeChoices.TYPE_LAG + ) # Add current site to VLANs query params self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', device.site.pk) From 2e83ce76eddc7f5d7cfb38430c70efc352b0b10e Mon Sep 17 00:00:00 2001 From: Matt Olenik Date: Fri, 13 Dec 2019 11:52:59 -0800 Subject: [PATCH 09/28] Fix race condition in available-prefix/ip APIs Implement advisory lock to prevent duplicate records being inserted when making simultaneous calls. Fixes #2519 --- base_requirements.txt | 4 ++++ netbox/ipam/api/views.py | 10 ++++++++++ netbox/utilities/constants.py | 11 +++++++++++ requirements.txt | 1 + 4 files changed, 26 insertions(+) diff --git a/base_requirements.txt b/base_requirements.txt index 8b42c835d..ed42b6c08 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -22,6 +22,10 @@ django-filter # https://github.com/django-mptt/django-mptt django-mptt +# Context managers for PostgreSQL advisory locks +# https://github.com/Xof/django-pglocks +django-pglocks + # Prometheus metrics library for Django # https://github.com/korfuri/django-prometheus django-prometheus diff --git a/netbox/ipam/api/views.py b/netbox/ipam/api/views.py index 08e21367c..262ca7908 100644 --- a/netbox/ipam/api/views.py +++ b/netbox/ipam/api/views.py @@ -1,6 +1,7 @@ from django.conf import settings from django.db.models import Count from django.shortcuts import get_object_or_404 +from django_pglocks import advisory_lock from rest_framework import status from rest_framework.decorators import action from rest_framework.exceptions import PermissionDenied @@ -10,6 +11,7 @@ from extras.api.views import CustomFieldModelViewSet from ipam import filters from ipam.models import Aggregate, IPAddress, Prefix, RIR, Role, Service, VLAN, VLANGroup, VRF from utilities.api import FieldChoicesViewSet, ModelViewSet +from utilities.constants import ADVISORY_LOCK_KEYS from utilities.utils import get_subquery from . import serializers @@ -86,9 +88,13 @@ class PrefixViewSet(CustomFieldModelViewSet): filterset_class = filters.PrefixFilterSet @action(detail=True, url_path='available-prefixes', methods=['get', 'post']) + @advisory_lock(ADVISORY_LOCK_KEYS['available-prefixes']) def available_prefixes(self, request, pk=None): """ A convenience method for returning available child prefixes within a parent. + + The advisory lock decorator uses a PostgreSQL advisory lock to prevent this API from being + invoked in parallel, which results in a race condition where multiple insertions can occur. """ prefix = get_object_or_404(Prefix, pk=pk) available_prefixes = prefix.get_available_prefixes() @@ -180,11 +186,15 @@ class PrefixViewSet(CustomFieldModelViewSet): return Response(serializer.data) @action(detail=True, url_path='available-ips', methods=['get', 'post']) + @advisory_lock(ADVISORY_LOCK_KEYS['available-ips']) def available_ips(self, request, pk=None): """ A convenience method for returning available IP addresses within a prefix. By default, the number of IPs returned will be equivalent to PAGINATE_COUNT. An arbitrary limit (up to MAX_PAGE_SIZE, if set) may be passed, however results will not be paginated. + + The advisory lock decorator uses a PostgreSQL advisory lock to prevent this API from being + invoked in parallel, which results in a race condition where multiple insertions can occur. """ prefix = get_object_or_404(Prefix, pk=pk) diff --git a/netbox/utilities/constants.py b/netbox/utilities/constants.py index ad6e8fd90..2cec2b532 100644 --- a/netbox/utilities/constants.py +++ b/netbox/utilities/constants.py @@ -27,3 +27,14 @@ COLOR_CHOICES = ( ('111111', 'Black'), ('ffffff', 'White'), ) + +# Keys for PostgreSQL advisory locks. These are arbitrary bigints used by +# the advisory_lock contextmanager. When a lock is acquired, +# one of these keys will be used to identify said lock. +# +# When adding a new key, pick something arbitrary and unique so +# that it is easily searchable in query logs. +ADVISORY_LOCK_KEYS = { + 'available-prefixes': 100100, + 'available-ips': 100200, +} diff --git a/requirements.txt b/requirements.txt index 9c5f624f9..b0b1b971d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ django-cors-headers==3.2.1 django-debug-toolbar==2.1 django-filter==2.2.0 django-mptt==0.9.1 +django-pglocks==1.0.4 django-prometheus==1.1.0 django-rq==2.2.0 django-tables2==2.2.1 From 3538eeda14a5ba1921299f2742593f0b9e434807 Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Thu, 13 Feb 2020 08:53:46 -0500 Subject: [PATCH 10/28] allow for redis sentinel connection --- netbox/netbox/configuration.example.py | 6 +++ netbox/netbox/settings.py | 55 ++++++++++++++++++++------ 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index 885c9e0ba..7002def9b 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -28,6 +28,9 @@ REDIS = { 'webhooks': { 'HOST': 'localhost', 'PORT': 6379, + # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '', 'DATABASE': 0, 'DEFAULT_TIMEOUT': 300, @@ -36,6 +39,9 @@ REDIS = { 'caching': { 'HOST': 'localhost', 'PORT': 6379, + # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '', 'DATABASE': 1, 'DEFAULT_TIMEOUT': 300, diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 305f7efa8..4b6177318 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -170,14 +170,27 @@ if 'caching' not in REDIS: WEBHOOKS_REDIS = REDIS.get('webhooks', {}) WEBHOOKS_REDIS_HOST = WEBHOOKS_REDIS.get('HOST', 'localhost') WEBHOOKS_REDIS_PORT = WEBHOOKS_REDIS.get('PORT', 6379) +WEBHOOKS_REDIS_SENTINELS = WEBHOOKS_REDIS.get('SENTINELS', []) +WEBHOOKS_REDIS_USING_SENTINEL = all([ + isinstance(WEBHOOKS_REDIS_SENTINELS, (list, tuple)), + len(WEBHOOKS_REDIS_SENTINELS) > 0 +]) +WEBHOOKS_REDIS_SENTINEL_SERVICE = WEBHOOKS_REDIS.get('SENTINEL_SERVICE', 'default') WEBHOOKS_REDIS_PASSWORD = WEBHOOKS_REDIS.get('PASSWORD', '') WEBHOOKS_REDIS_DATABASE = WEBHOOKS_REDIS.get('DATABASE', 0) WEBHOOKS_REDIS_DEFAULT_TIMEOUT = WEBHOOKS_REDIS.get('DEFAULT_TIMEOUT', 300) WEBHOOKS_REDIS_SSL = WEBHOOKS_REDIS.get('SSL', False) + CACHING_REDIS = REDIS.get('caching', {}) CACHING_REDIS_HOST = CACHING_REDIS.get('HOST', 'localhost') CACHING_REDIS_PORT = CACHING_REDIS.get('PORT', 6379) +CACHING_REDIS_SENTINELS = CACHING_REDIS.get('SENTINELS', []) +CACHING_REDIS_USING_SENTINEL = all([ + isinstance(CACHING_REDIS_SENTINELS, (list, tuple)), + len(CACHING_REDIS_SENTINELS) > 0 +]) +CACHING_REDIS_SENTINEL_SERVICE = CACHING_REDIS.get('SENTINEL_SERVICE', 'default') CACHING_REDIS_PASSWORD = CACHING_REDIS.get('PASSWORD', '') CACHING_REDIS_DATABASE = CACHING_REDIS.get('DATABASE', 0) CACHING_REDIS_DEFAULT_TIMEOUT = CACHING_REDIS.get('DEFAULT_TIMEOUT', 300) @@ -394,28 +407,35 @@ if LDAP_CONFIG is not None: # # Caching # - -if CACHING_REDIS_SSL: - REDIS_CACHE_CON_STRING = 'rediss://' +if CACHING_REDIS_USING_SENTINEL: + CACHEOPS_SENTINEL = { + 'locations': CACHING_REDIS_SENTINELS, + 'service_name': CACHING_REDIS_SENTINEL_SERVICE, + 'db': CACHING_REDIS_DATABASE, + } else: - REDIS_CACHE_CON_STRING = 'redis://' + if CACHING_REDIS_SSL: + REDIS_CACHE_CON_STRING = 'rediss://' + else: + REDIS_CACHE_CON_STRING = 'redis://' -if CACHING_REDIS_PASSWORD: - REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, CACHING_REDIS_PASSWORD) + if CACHING_REDIS_PASSWORD: + REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, CACHING_REDIS_PASSWORD) -REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format( - REDIS_CACHE_CON_STRING, - CACHING_REDIS_HOST, - CACHING_REDIS_PORT, - CACHING_REDIS_DATABASE -) + REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format( + REDIS_CACHE_CON_STRING, + CACHING_REDIS_HOST, + CACHING_REDIS_PORT, + CACHING_REDIS_DATABASE + ) + CACHEOPS_REDIS = REDIS_CACHE_CON_STRING if not CACHE_TIMEOUT: CACHEOPS_ENABLED = False else: CACHEOPS_ENABLED = True -CACHEOPS_REDIS = REDIS_CACHE_CON_STRING + CACHEOPS_DEFAULTS = { 'timeout': CACHE_TIMEOUT } @@ -534,6 +554,15 @@ RQ_QUEUES = { 'PASSWORD': WEBHOOKS_REDIS_PASSWORD, 'DEFAULT_TIMEOUT': WEBHOOKS_REDIS_DEFAULT_TIMEOUT, 'SSL': WEBHOOKS_REDIS_SSL, + } if not WEBHOOKS_REDIS_USING_SENTINEL else { + 'SENTINELS': WEBHOOKS_REDIS_SENTINELS, + 'MASTER_NAME': WEBHOOKS_REDIS_SENTINEL_SERVICE, + 'DB': WEBHOOKS_REDIS_DATABASE, + 'PASSWORD': WEBHOOKS_REDIS_PASSWORD, + 'SOCKET_TIMEOUT': None, + 'CONNECTION_KWARGS': { + 'socket_connect_timeout': WEBHOOKS_REDIS_DEFAULT_TIMEOUT + }, } } From ba9a2956a8ac6076b4f9e74e857e6b4634e2a836 Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Thu, 13 Feb 2020 09:10:53 -0500 Subject: [PATCH 11/28] documentation on redis sentinel --- docs/configuration/required-settings.md | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/configuration/required-settings.md b/docs/configuration/required-settings.md index 70c6de276..81790eae0 100644 --- a/docs/configuration/required-settings.md +++ b/docs/configuration/required-settings.md @@ -88,6 +88,48 @@ REDIS = { It is highly recommended to keep the webhook and cache databases separate. Using the same database number on the same Redis instance for both may result in webhook processing data being lost during cache flushing events. +### Using Redis Sentinel + +If you are using [Redis Sentinel](https://redis.io/topics/sentinel) for high-availability purposes, there is minimal +configuration necessary to convert NetBox to recognize it. It requires the removal of the `HOST` and `PORT` keys from +above and the addition of two new keys. + +* `SENTINELS`: List of tuples or tuple of tuples with each inner tuple containing the name or IP address +of the Redis server and port for each sentinel instance to connect to +* `SENTINEL_SERVICE`: Name of the master / service to connect to + +Example: + +```python +REDIS = { + 'webhooks': { + 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'SENTINELS': [ + ('mysentinel.redis.example.com', 6379), + ('othersentinel.redis.example.com', 6379) + ], + 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } +} +``` + +!!! note: + It is possible to have only one or the other Redis configurations to use Sentinel functionality. It is possible + for example to have the webhook use sentinel via `HOST`/`PORT` and for caching to use Sentinel via + `SENTINELS`/`SENTINEL_SERVICE`. + + --- ## SECRET_KEY From 72f0e31b84820a7eaf508e567956c0d365f2afba Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Thu, 13 Feb 2020 10:27:56 -0500 Subject: [PATCH 12/28] fixed extraneous linting error --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 4b6177318..1a36cf424 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -409,7 +409,7 @@ if LDAP_CONFIG is not None: # if CACHING_REDIS_USING_SENTINEL: CACHEOPS_SENTINEL = { - 'locations': CACHING_REDIS_SENTINELS, + 'locations': CACHING_REDIS_SENTINELS, 'service_name': CACHING_REDIS_SENTINEL_SERVICE, 'db': CACHING_REDIS_DATABASE, } From 92fab048d116c02484f249c4e9a4f51425cdbee0 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 15:13:43 -0500 Subject: [PATCH 13/28] Add tests for naturalization functions --- netbox/utilities/ordering.py | 2 +- netbox/utilities/tests/test_ordering.py | 43 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 netbox/utilities/tests/test_ordering.py diff --git a/netbox/utilities/ordering.py b/netbox/utilities/ordering.py index d459e6f6c..a560e776e 100644 --- a/netbox/utilities/ordering.py +++ b/netbox/utilities/ordering.py @@ -68,7 +68,7 @@ def naturalize_interface(value, max_length=None): if match.group('type') is not None: output.append(match.group('type')) - # Finally, append any remaining fields, left-padding to eight digits each. + # Finally, append any remaining fields, left-padding to six digits each. for part_name in ('id', 'channel', 'vc'): part = match.group(part_name) if part is not None: diff --git a/netbox/utilities/tests/test_ordering.py b/netbox/utilities/tests/test_ordering.py new file mode 100644 index 000000000..a875c688c --- /dev/null +++ b/netbox/utilities/tests/test_ordering.py @@ -0,0 +1,43 @@ +from django.test import TestCase + +from utilities.ordering import naturalize, naturalize_interface + + +class NaturalizationTestCase(TestCase): + """ + Validate the operation of the functions which generate values suitable for natural ordering. + """ + def test_naturalize(self): + + data = ( + # Original, naturalized + ('abc', 'abc'), + ('123', '00000123'), + ('abc123', 'abc00000123'), + ('123abc', '00000123abc'), + ('123abc456', '00000123abc00000456'), + ('abc123def', 'abc00000123def'), + ('abc123def456', 'abc00000123def00000456'), + ) + + for origin, naturalized in data: + self.assertEqual(naturalize(origin), naturalized) + + def test_naturalize_interface(self): + + data = ( + # Original, naturalized + ('Gi', '9999999999999999Gi000000000000000000'), + ('Gi1', '9999999999999999Gi000001000000000000'), + ('Gi1/2', '0001999999999999Gi000002000000000000'), + ('Gi1/2/3', '0001000299999999Gi000003000000000000'), + ('Gi1/2/3/4', '0001000200039999Gi000004000000000000'), + ('Gi1/2/3/4/5', '0001000200030004Gi000005000000000000'), + ('Gi1/2/3/4/5:6', '0001000200030004Gi000005000006000000'), + ('Gi1/2/3/4/5:6.7', '0001000200030004Gi000005000006000007'), + ('Gi1:2', '9999999999999999Gi000001000002000000'), + ('Gi1:2.3', '9999999999999999Gi000001000002000003'), + ) + + for origin, naturalized in data: + self.assertEqual(naturalize_interface(origin), naturalized) From 807c2f048d435a60dfe86fb5ef4ed3aa1ddfdaa5 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 15:16:07 -0500 Subject: [PATCH 14/28] Changelog for #3984 --- docs/release-notes/version-2.7.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 63745bbab..ea200c462 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -4,6 +4,7 @@ * [#3766](https://github.com/netbox-community/netbox/issues/3766) - Allow custom script authors to specify the form widget for each variable * [#3799](https://github.com/netbox-community/netbox/issues/3799) - Greatly improve performance when ordering device components +* [#3984](https://github.com/netbox-community/netbox/issues/3984) - Add support for Redis Sentinel * [#3986](https://github.com/netbox-community/netbox/issues/3986) - Include position numbers in SVG image when rendering rack elevation * [#4093](https://github.com/netbox-community/netbox/issues/4093) - Add multiple status choices for VMs * [#4100](https://github.com/netbox-community/netbox/issues/4100) - Add device filter to component list views From 08ce024473bd4b20ab75e9a3ca416a5150f2f6bc Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 15:32:09 -0500 Subject: [PATCH 15/28] Release v2.7.5 --- docs/release-notes/version-2.7.md | 14 ++++++++------ netbox/netbox/settings.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index ea200c462..04b3972ca 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,12 +1,14 @@ -# v2.7.5 (FUTURE) +# v2.7.5 (2020-02-13) + +**Note:** This release includes several database schema migrations that calculate and store copies of names for certain objects to improve natural ordering performance (see [#3799](https://github.com/netbox-community/netbox/issues/3799)). These migrations may take a few minutes to run if you have a very large number of objects defined in NetBox. ## Enhancements * [#3766](https://github.com/netbox-community/netbox/issues/3766) - Allow custom script authors to specify the form widget for each variable * [#3799](https://github.com/netbox-community/netbox/issues/3799) - Greatly improve performance when ordering device components * [#3984](https://github.com/netbox-community/netbox/issues/3984) - Add support for Redis Sentinel -* [#3986](https://github.com/netbox-community/netbox/issues/3986) - Include position numbers in SVG image when rendering rack elevation -* [#4093](https://github.com/netbox-community/netbox/issues/4093) - Add multiple status choices for VMs +* [#3986](https://github.com/netbox-community/netbox/issues/3986) - Include position numbers in SVG image when rendering rack elevations +* [#4093](https://github.com/netbox-community/netbox/issues/4093) - Add more status choices for virtual machines * [#4100](https://github.com/netbox-community/netbox/issues/4100) - Add device filter to component list views * [#4113](https://github.com/netbox-community/netbox/issues/4113) - Add bulk edit functionality for device type components * [#4116](https://github.com/netbox-community/netbox/issues/4116) - Enable bulk edit and delete functions for device component list views @@ -14,8 +16,8 @@ ## Bug Fixes -* [#3507](https://github.com/netbox-community/netbox/issues/3507) - Fix filtering IPaddress by multiple devices -* [#3995](https://github.com/netbox-community/netbox/issues/3995) - Make dropdown menus in the navigation bar scrollable +* [#3507](https://github.com/netbox-community/netbox/issues/3507) - Fix filtering IP addresses by multiple devices +* [#3995](https://github.com/netbox-community/netbox/issues/3995) - Make dropdown menus in the navigation bar scrollable on small screens * [#4083](https://github.com/netbox-community/netbox/issues/4083) - Permit nullifying applicable choice fields via API requests * [#4089](https://github.com/netbox-community/netbox/issues/4089) - Selection of power outlet type during bulk update is optional * [#4090](https://github.com/netbox-community/netbox/issues/4090) - Render URL custom fields as links under object view @@ -24,7 +26,7 @@ * [#4108](https://github.com/netbox-community/netbox/issues/4108) - Avoid extraneous database queries when rendering search forms * [#4134](https://github.com/netbox-community/netbox/issues/4134) - Device power ports and outlets should inherit type from the parent device type * [#4137](https://github.com/netbox-community/netbox/issues/4137) - Disable occupied terminations when connecting a cable to a circuit -* [#4138](https://github.com/netbox-community/netbox/issues/4138) - Include device bay counts in rack elevation diagrams +* [#4138](https://github.com/netbox-community/netbox/issues/4138) - Restore device bay counts in rack elevation diagrams * [#4146](https://github.com/netbox-community/netbox/issues/4146) - Fix enforcement of secret role assignment for secret decryption * [#4150](https://github.com/netbox-community/netbox/issues/4150) - Correct YAML rendering of config contexts * [#4159](https://github.com/netbox-community/netbox/issues/4159) - Fix implementation of Redis caching configuration diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 1a36cf424..5f60d16f7 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -12,7 +12,7 @@ from django.core.exceptions import ImproperlyConfigured # Environment setup # -VERSION = '2.7.5-dev' +VERSION = '2.7.5' # Hostname HOSTNAME = platform.node() From 4d50cad6ed6e0a7d048b30ef893c396678ef33de Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 15:39:08 -0500 Subject: [PATCH 16/28] Post-release version bump --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 5f60d16f7..f66828f69 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -12,7 +12,7 @@ from django.core.exceptions import ImproperlyConfigured # Environment setup # -VERSION = '2.7.5' +VERSION = '2.7.6-dev' # Hostname HOSTNAME = platform.node() From 909323663eca94493a2eb64e8e08bcae58290be8 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 21:41:00 -0500 Subject: [PATCH 17/28] Fixes #4166: Fix schema migrations to enforce maximum character length for naturalized fields --- docs/release-notes/version-2.7.md | 8 ++++++++ .../dcim/migrations/0093_device_component_ordering.py | 2 +- .../0094_device_component_template_ordering.py | 2 +- netbox/dcim/migrations/0095_primary_model_ordering.py | 2 +- netbox/dcim/migrations/0096_interface_ordering.py | 2 +- netbox/extras/management/commands/renaturalize.py | 2 +- netbox/utilities/ordering.py | 8 ++++---- netbox/utilities/tests/test_ordering.py | 10 ++++++++-- 8 files changed, 25 insertions(+), 11 deletions(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 04b3972ca..75269b6ad 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,3 +1,11 @@ +# v2.7.6 (FUTURE) + +## Bug Fixes + +* [#4166](https://github.com/netbox-community/netbox/issues/4166) - Fix schema migrations to enforce maximum character length for naturalized fields + +--- + # v2.7.5 (2020-02-13) **Note:** This release includes several database schema migrations that calculate and store copies of names for certain objects to improve natural ordering performance (see [#3799](https://github.com/netbox-community/netbox/issues/3799)). These migrations may take a few minutes to run if you have a very large number of objects defined in NetBox. diff --git a/netbox/dcim/migrations/0093_device_component_ordering.py b/netbox/dcim/migrations/0093_device_component_ordering.py index 017241c8b..4e3c941a1 100644 --- a/netbox/dcim/migrations/0093_device_component_ordering.py +++ b/netbox/dcim/migrations/0093_device_component_ordering.py @@ -6,7 +6,7 @@ import utilities.ordering def _update_model_names(model): # Update each unique field value in bulk for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): - model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name, max_length=100)) def naturalize_consoleports(apps, schema_editor): diff --git a/netbox/dcim/migrations/0094_device_component_template_ordering.py b/netbox/dcim/migrations/0094_device_component_template_ordering.py index fc39f76b2..24fe98e94 100644 --- a/netbox/dcim/migrations/0094_device_component_template_ordering.py +++ b/netbox/dcim/migrations/0094_device_component_template_ordering.py @@ -6,7 +6,7 @@ import utilities.ordering def _update_model_names(model): # Update each unique field value in bulk for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): - model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name, max_length=100)) def naturalize_consoleporttemplates(apps, schema_editor): diff --git a/netbox/dcim/migrations/0095_primary_model_ordering.py b/netbox/dcim/migrations/0095_primary_model_ordering.py index 9cef0a581..3bc780161 100644 --- a/netbox/dcim/migrations/0095_primary_model_ordering.py +++ b/netbox/dcim/migrations/0095_primary_model_ordering.py @@ -6,7 +6,7 @@ import utilities.ordering def _update_model_names(model): # Update each unique field value in bulk for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): - model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name, max_length=100)) def naturalize_sites(apps, schema_editor): diff --git a/netbox/dcim/migrations/0096_interface_ordering.py b/netbox/dcim/migrations/0096_interface_ordering.py index 284066462..f1622f504 100644 --- a/netbox/dcim/migrations/0096_interface_ordering.py +++ b/netbox/dcim/migrations/0096_interface_ordering.py @@ -6,7 +6,7 @@ import utilities.ordering def _update_model_names(model): # Update each unique field value in bulk for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): - model.objects.filter(name=name).update(_name=utilities.ordering.naturalize_interface(name)) + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize_interface(name, max_length=100)) def naturalize_interfacetemplates(apps, schema_editor): diff --git a/netbox/extras/management/commands/renaturalize.py b/netbox/extras/management/commands/renaturalize.py index 70f57c1ba..cfd037910 100644 --- a/netbox/extras/management/commands/renaturalize.py +++ b/netbox/extras/management/commands/renaturalize.py @@ -86,7 +86,7 @@ class Command(BaseCommand): # Find all unique values for the field queryset = model.objects.values_list(target_field, flat=True).order_by(target_field).distinct() for value in queryset: - naturalized_value = naturalize(value) + naturalized_value = naturalize(value, max_length=field.max_length) if options['verbosity'] >= 2: self.stdout.write(" {} -> {}".format(value, naturalized_value), ending='') diff --git a/netbox/utilities/ordering.py b/netbox/utilities/ordering.py index a560e776e..39e21733a 100644 --- a/netbox/utilities/ordering.py +++ b/netbox/utilities/ordering.py @@ -10,7 +10,7 @@ INTERFACE_NAME_REGEX = r'(^(?P[^\d\.:]+)?)' \ r'(.(?P\d+)$)?' -def naturalize(value, max_length=None, integer_places=8): +def naturalize(value, max_length, integer_places=8): """ Take an alphanumeric string and prepend all integers to `integer_places` places to ensure the strings are ordered naturally. For example: @@ -39,10 +39,10 @@ def naturalize(value, max_length=None, integer_places=8): output.append(segment) ret = ''.join(output) - return ret[:max_length] if max_length else ret + return ret[:max_length] -def naturalize_interface(value, max_length=None): +def naturalize_interface(value, max_length): """ Similar in nature to naturalize(), but takes into account a particular naming format adapted from the old InterfaceManager. @@ -77,4 +77,4 @@ def naturalize_interface(value, max_length=None): output.append('000000') ret = ''.join(output) - return ret[:max_length] if max_length else ret + return ret[:max_length] diff --git a/netbox/utilities/tests/test_ordering.py b/netbox/utilities/tests/test_ordering.py index a875c688c..958224c94 100644 --- a/netbox/utilities/tests/test_ordering.py +++ b/netbox/utilities/tests/test_ordering.py @@ -21,7 +21,10 @@ class NaturalizationTestCase(TestCase): ) for origin, naturalized in data: - self.assertEqual(naturalize(origin), naturalized) + self.assertEqual(naturalize(origin, max_length=50), naturalized) + + def test_naturalize_max_length(self): + self.assertEqual(naturalize('abc123def456', max_length=10), 'abc0000012') def test_naturalize_interface(self): @@ -40,4 +43,7 @@ class NaturalizationTestCase(TestCase): ) for origin, naturalized in data: - self.assertEqual(naturalize_interface(origin), naturalized) + self.assertEqual(naturalize_interface(origin, max_length=50), naturalized) + + def test_naturalize_interface_max_length(self): + self.assertEqual(naturalize_interface('Gi1/2/3', max_length=20), '0001000299999999Gi00') From 0863145c7fdc2d619a28019fa73d848be4d44681 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 21:46:03 -0500 Subject: [PATCH 18/28] Release v2.7.6 --- docs/release-notes/version-2.7.md | 2 +- netbox/netbox/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 75269b6ad..70981658f 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,4 +1,4 @@ -# v2.7.6 (FUTURE) +# v2.7.6 (2020-02-13) ## Bug Fixes diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index f66828f69..81422b714 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -12,7 +12,7 @@ from django.core.exceptions import ImproperlyConfigured # Environment setup # -VERSION = '2.7.6-dev' +VERSION = '2.7.6' # Hostname HOSTNAME = platform.node() From 598d23fc03b7e50530bc8cfae499050bba18039c Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 13 Feb 2020 21:51:03 -0500 Subject: [PATCH 19/28] Post-release version bump --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 81422b714..249ee9e53 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -12,7 +12,7 @@ from django.core.exceptions import ImproperlyConfigured # Environment setup # -VERSION = '2.7.6' +VERSION = '2.7.7-dev' # Hostname HOSTNAME = platform.node() From 1d72436bfc6d38809a52f7677cdf28de1d936cb9 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 14 Feb 2020 09:13:05 -0500 Subject: [PATCH 20/28] Fixes #4168: Role is not required when creating a virtual machine --- docs/release-notes/version-2.7.md | 8 ++++++++ netbox/virtualization/forms.py | 1 + 2 files changed, 9 insertions(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 70981658f..c235b8115 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,3 +1,11 @@ +# v2.7.7 (FUTURE) + +## Bug Fixes + +* [#4168](https://github.com/netbox-community/netbox/issues/4168) - Role is not required when creating a virtual machine + +--- + # v2.7.6 (2020-02-13) ## Bug Fixes diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index 12393d400..f35a94c99 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -351,6 +351,7 @@ class VirtualMachineForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): ) role = DynamicModelChoiceField( queryset=DeviceRole.objects.all(), + required=False, widget=APISelect( api_url="/api/dcim/device-roles/", additional_query_params={ From a4705fa73a464a88ac8931a44f55e2378f434c97 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 14 Feb 2020 09:35:43 -0500 Subject: [PATCH 21/28] Changelog for #2519 --- docs/release-notes/version-2.7.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index c235b8115..ac2f149e1 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -2,6 +2,7 @@ ## Bug Fixes +* [#2519](https://github.com/netbox-community/netbox/issues/2519) - Avoid race condition when provisioning "next available" IPs/prefixes via the API * [#4168](https://github.com/netbox-community/netbox/issues/4168) - Role is not required when creating a virtual machine --- From 03a7f6bbda63e1fa638804376686f87b39f15d4a Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Fri, 14 Feb 2020 09:39:01 -0500 Subject: [PATCH 22/28] ammend redis conn check to acccount for sentinel --- netbox/extras/apps.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/netbox/extras/apps.py b/netbox/extras/apps.py index f8c5a98e6..257e7801a 100644 --- a/netbox/extras/apps.py +++ b/netbox/extras/apps.py @@ -13,13 +13,23 @@ class ExtrasConfig(AppConfig): # Check that we can connect to the configured Redis database. try: - rs = redis.Redis( - host=settings.WEBHOOKS_REDIS_HOST, - port=settings.WEBHOOKS_REDIS_PORT, - db=settings.WEBHOOKS_REDIS_DATABASE, - password=settings.WEBHOOKS_REDIS_PASSWORD or None, - ssl=settings.WEBHOOKS_REDIS_SSL, - ) + if settings.WEBHOOKS_REDIS_USING_SENTINEL: + sentinel = redis.sentinel.Sentinel( + settings.WEBHOOKS_REDIS_SENTINELS, + socket_timeout=settings.WEBHOOKS_REDIS_DEFAULT_TIMEOUT + ) + rs = sentinel.master_for( + settings.WEBHOOKS_REDIS_SENTINEL_SERVICE, + socket_timeout=settings.WEBHOOKS_REDIS_DEFAULT_TIMEOUT + ) + else: + rs = redis.Redis( + host=settings.WEBHOOKS_REDIS_HOST, + port=settings.WEBHOOKS_REDIS_PORT, + db=settings.WEBHOOKS_REDIS_DATABASE, + password=settings.WEBHOOKS_REDIS_PASSWORD or None, + ssl=settings.WEBHOOKS_REDIS_SSL, + ) rs.ping() except redis.exceptions.ConnectionError: raise ImproperlyConfigured( From e431ef09e5b49f15fab28ec5332b88051a3ca035 Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Fri, 14 Feb 2020 10:29:09 -0500 Subject: [PATCH 23/28] fix extraneous formatting of notice boxes in required settings doc --- docs/configuration/required-settings.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration/required-settings.md b/docs/configuration/required-settings.md index 81790eae0..e86b2810a 100644 --- a/docs/configuration/required-settings.md +++ b/docs/configuration/required-settings.md @@ -80,11 +80,11 @@ REDIS = { } ``` -!!! note: +!!! note If you are upgrading from a version prior to v2.7, please note that the Redis connection configuration settings have changed. Manual modification to bring the `REDIS` section inline with the above specification is necessary -!!! warning: +!!! note It is highly recommended to keep the webhook and cache databases separate. Using the same database number on the same Redis instance for both may result in webhook processing data being lost during cache flushing events. @@ -124,7 +124,7 @@ REDIS = { } ``` -!!! note: +!!! note It is possible to have only one or the other Redis configurations to use Sentinel functionality. It is possible for example to have the webhook use sentinel via `HOST`/`PORT` and for caching to use Sentinel via `SENTINELS`/`SENTINEL_SERVICE`. From ec0f45e20dffde593f157b789a922aca1c9fe64a Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Fri, 14 Feb 2020 11:16:59 -0500 Subject: [PATCH 24/28] remove redis conn check from extras AppConfig --- netbox/extras/apps.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/netbox/extras/apps.py b/netbox/extras/apps.py index 257e7801a..5dad07b48 100644 --- a/netbox/extras/apps.py +++ b/netbox/extras/apps.py @@ -8,31 +8,4 @@ class ExtrasConfig(AppConfig): name = "extras" def ready(self): - import extras.signals - - # Check that we can connect to the configured Redis database. - try: - if settings.WEBHOOKS_REDIS_USING_SENTINEL: - sentinel = redis.sentinel.Sentinel( - settings.WEBHOOKS_REDIS_SENTINELS, - socket_timeout=settings.WEBHOOKS_REDIS_DEFAULT_TIMEOUT - ) - rs = sentinel.master_for( - settings.WEBHOOKS_REDIS_SENTINEL_SERVICE, - socket_timeout=settings.WEBHOOKS_REDIS_DEFAULT_TIMEOUT - ) - else: - rs = redis.Redis( - host=settings.WEBHOOKS_REDIS_HOST, - port=settings.WEBHOOKS_REDIS_PORT, - db=settings.WEBHOOKS_REDIS_DATABASE, - password=settings.WEBHOOKS_REDIS_PASSWORD or None, - ssl=settings.WEBHOOKS_REDIS_SSL, - ) - rs.ping() - except redis.exceptions.ConnectionError: - raise ImproperlyConfigured( - "Unable to connect to the Redis database. Check that the Redis configuration has been defined in " - "configuration.py." - ) From 7aba8e3ec48a03b867ae6a500a77588c6edc6a16 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Fri, 14 Feb 2020 16:43:42 +0000 Subject: [PATCH 25/28] Added back clean --- netbox/virtualization/forms.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index 6771ee76b..2aa57608a 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -704,6 +704,22 @@ class InterfaceForm(BootstrapMixin, forms.ModelForm): self.fields['untagged_vlan'].widget.add_additional_query_param('site_id', site.pk) self.fields['tagged_vlans'].widget.add_additional_query_param('site_id', site.pk) + def clean(self): + super().clean() + + # Validate VLAN assignments + tagged_vlans = self.cleaned_data['tagged_vlans'] + + # Untagged interfaces cannot be assigned tagged VLANs + if self.cleaned_data['mode'] == InterfaceModeChoices.MODE_ACCESS and tagged_vlans: + raise forms.ValidationError({ + 'mode': "An access interface cannot have tagged VLANs assigned." + }) + + # Remove all tagged VLAN assignments from "tagged all" interfaces + elif self.cleaned_data['mode'] == InterfaceModeChoices.MODE_TAGGED_ALL: + self.cleaned_data['tagged_vlans'] = [] + class InterfaceCreateForm(BootstrapMixin, forms.Form): virtual_machine = forms.ModelChoiceField( From e4df02887b2f65b4a36b00396799e3afa342b224 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 14 Feb 2020 12:04:35 -0500 Subject: [PATCH 26/28] Changelog for #3840 --- docs/release-notes/version-2.7.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index ac2f149e1..5e58e0c58 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,5 +1,9 @@ # v2.7.7 (FUTURE) +## Enhancements + +* [#3840](https://github.com/netbox-community/netbox/issues/3840) - Enhance search function when selecting VLANs for interface assignment + ## Bug Fixes * [#2519](https://github.com/netbox-community/netbox/issues/2519) - Avoid race condition when provisioning "next available" IPs/prefixes via the API From 7ce1289bb2ad5e1f2d723a6e7eeb5ac315d85df7 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 14 Feb 2020 12:04:56 -0500 Subject: [PATCH 27/28] Clean up unused imports --- netbox/extras/apps.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/netbox/extras/apps.py b/netbox/extras/apps.py index 5dad07b48..3201c3bb2 100644 --- a/netbox/extras/apps.py +++ b/netbox/extras/apps.py @@ -1,7 +1,4 @@ from django.apps import AppConfig -from django.conf import settings -from django.core.exceptions import ImproperlyConfigured -import redis class ExtrasConfig(AppConfig): From ce89fa74b9797be6d7d5ed235065ed85f8bf90d6 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 14 Feb 2020 13:09:01 -0500 Subject: [PATCH 28/28] Closes #4170: Improve color contrast in rack elevation drawings --- docs/release-notes/version-2.7.md | 1 + netbox/dcim/models/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 5e58e0c58..d3b889513 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -3,6 +3,7 @@ ## Enhancements * [#3840](https://github.com/netbox-community/netbox/issues/3840) - Enhance search function when selecting VLANs for interface assignment +* [#4170](https://github.com/netbox-community/netbox/issues/4170) - Improve color contrast in rack elevation drawings ## Bug Fixes diff --git a/netbox/dcim/models/__init__.py b/netbox/dcim/models/__init__.py index f291fc825..29afef1f1 100644 --- a/netbox/dcim/models/__init__.py +++ b/netbox/dcim/models/__init__.py @@ -382,8 +382,8 @@ class RackElevationHelperMixin: # add gradients RackElevationHelperMixin._add_gradient(drawing, 'reserved', '#c7c7ff') - RackElevationHelperMixin._add_gradient(drawing, 'occupied', '#f0f0f0') - RackElevationHelperMixin._add_gradient(drawing, 'blocked', '#ffc7c7') + RackElevationHelperMixin._add_gradient(drawing, 'occupied', '#d7d7d7') + RackElevationHelperMixin._add_gradient(drawing, 'blocked', '#ffc0c0') return drawing