From 0d27abc6fc22a8d40183a59eceef5dda57e99eae Mon Sep 17 00:00:00 2001 From: John Anderson Date: Fri, 6 Nov 2020 16:47:07 -0500 Subject: [PATCH 01/10] fixes #5314 - Fix config context rendering when multiple tags are assgined to an object --- docs/release-notes/version-2.9.md | 1 + netbox/extras/querysets.py | 4 ++-- netbox/extras/tests/test_models.py | 35 ++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index 042f75691..ce7289da4 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -10,6 +10,7 @@ ### Bug Fixes * [#5271](https://github.com/netbox-community/netbox/issues/5271) - Fix auto-population of region field when editing a device +* [#5314](https://github.com/netbox-community/netbox/issues/5314) - Fix config context rendering when multiple tags are assgined to an object --- diff --git a/netbox/extras/querysets.py b/netbox/extras/querysets.py index 9bfa5da83..92402c45f 100644 --- a/netbox/extras/querysets.py +++ b/netbox/extras/querysets.py @@ -60,7 +60,7 @@ class ConfigContextQuerySet(RestrictedQuerySet): Q(tenants=obj.tenant) | Q(tenants=None), Q(tags__slug__in=obj.tags.slugs()) | Q(tags=None), is_active=True, - ).order_by('weight', 'name') + ).order_by('weight', 'name').distinct() if aggregate_data: return queryset.aggregate( @@ -95,7 +95,7 @@ class ConfigContextModelQuerySet(RestrictedQuerySet): _data=EmptyGroupByJSONBAgg('data', ordering=['weight', 'name']) ).values("_data") ) - ) + ).distinct() def _get_config_context_filters(self): # Construct the set of Q objects for the specific object types diff --git a/netbox/extras/tests/test_models.py b/netbox/extras/tests/test_models.py index 280da75b6..80beb0296 100644 --- a/netbox/extras/tests/test_models.py +++ b/netbox/extras/tests/test_models.py @@ -75,6 +75,7 @@ class ConfigContextTest(TestCase): self.tenantgroup = TenantGroup.objects.create(name="Tenant Group") self.tenant = Tenant.objects.create(name="Tenant", group=self.tenantgroup) self.tag = Tag.objects.create(name="Tag", slug="tag") + self.tag2 = Tag.objects.create(name="Tag2", slug="tag2") self.device = Device.objects.create( name='Device 1', @@ -328,3 +329,37 @@ class ConfigContextTest(TestCase): annotated_queryset = VirtualMachine.objects.filter(name=virtual_machine.name).annotate_config_context_data() self.assertEqual(virtual_machine.get_config_context(), annotated_queryset[0].get_config_context()) + + def test_multiple_tags_return_distinct_objects(self): + """ + Tagged items use a generic relationship, which results in duplicate rows being returned when queried. + This is combatted by by appending distinct() to the config context querysets. This test creates a config + context assigned to two tags and ensures objects related by those same two tags result in only a single + config context record being returned. + + See https://github.com/netbox-community/netbox/issues/5314 + """ + tag_context = ConfigContext.objects.create( + name="tag", + weight=100, + data={ + "tag": 1 + } + ) + tag_context.tags.add(self.tag) + tag_context.tags.add(self.tag2) + + device = Device.objects.create( + name="Device 3", + site=self.site, + tenant=self.tenant, + platform=self.platform, + device_role=self.devicerole, + device_type=self.devicetype + ) + device.tags.add(self.tag) + device.tags.add(self.tag2) + + annotated_queryset = Device.objects.filter(name=device.name).annotate_config_context_data() + self.assertEqual(ConfigContext.objects.get_for_object(device).count(), 1) + self.assertEqual(device.get_config_context(), annotated_queryset[0].get_config_context()) From 7cf4095983a5c17a495067005c87968a53560666 Mon Sep 17 00:00:00 2001 From: 991jo Date: Mon, 9 Nov 2020 07:56:26 +0100 Subject: [PATCH 02/10] Fixed a UI Bug where there is no whitespace after grouped custom links. See #5325 for details --- netbox/extras/templatetags/custom_links.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/extras/templatetags/custom_links.py b/netbox/extras/templatetags/custom_links.py index 9fbec489d..78630336e 100644 --- a/netbox/extras/templatetags/custom_links.py +++ b/netbox/extras/templatetags/custom_links.py @@ -16,7 +16,7 @@ GROUP_BUTTON = '
\n' \ '{} \n' \ '\n' \ '
    \n' \ - '{}
' + '{}\n' GROUP_LINK = '
  • {}
    • \n' From d2c4c7fb10b95104f362b35c1095914cf6f25599 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 9 Nov 2020 14:19:22 -0500 Subject: [PATCH 03/10] Closes #5324: Add missing template extension tags for plugins for VM interface view --- docs/release-notes/version-2.9.md | 3 ++- netbox/templates/virtualization/vminterface.html | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index ce7289da4..fe36b010e 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -10,7 +10,8 @@ ### Bug Fixes * [#5271](https://github.com/netbox-community/netbox/issues/5271) - Fix auto-population of region field when editing a device -* [#5314](https://github.com/netbox-community/netbox/issues/5314) - Fix config context rendering when multiple tags are assgined to an object +* [#5314](https://github.com/netbox-community/netbox/issues/5314) - Fix config context rendering when multiple tags are assigned to an object +* [#5324](https://github.com/netbox-community/netbox/issues/5324) - Add missing template extension tags for plugins for VM interface view --- diff --git a/netbox/templates/virtualization/vminterface.html b/netbox/templates/virtualization/vminterface.html index 8d46b52fd..976812a44 100644 --- a/netbox/templates/virtualization/vminterface.html +++ b/netbox/templates/virtualization/vminterface.html @@ -1,5 +1,6 @@ {% extends 'base.html' %} {% load helpers %} +{% load plugins %} {% block header %}
    @@ -12,6 +13,7 @@
    + {% plugin_buttons vminterface %} {% if perms.virtualization.change_vminterface %} Edit @@ -82,9 +84,11 @@
    + {% plugin_left_page vminterface %}
    {% include 'extras/inc/tags_panel.html' with tags=vminterface.tags.all %} + {% plugin_right_page vminterface %}
    @@ -97,4 +101,9 @@ {% include 'panel_table.html' with table=vlan_table heading="VLANs" %}
    +
    +
    + {% plugin_full_width_page vminterface %} +
    +
    {% endblock %} From c3f2cc7648d5f858935b7f6e575dc4ac0e87eda5 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 9 Nov 2020 14:20:07 -0500 Subject: [PATCH 04/10] Correct buttons() method in dummy plugin template content --- netbox/extras/tests/dummy_plugin/template_content.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/extras/tests/dummy_plugin/template_content.py b/netbox/extras/tests/dummy_plugin/template_content.py index fed17ca0b..6151454ea 100644 --- a/netbox/extras/tests/dummy_plugin/template_content.py +++ b/netbox/extras/tests/dummy_plugin/template_content.py @@ -13,7 +13,7 @@ class SiteContent(PluginTemplateExtension): def full_width_page(self): return "SITE CONTENT - FULL WIDTH PAGE" - def full_buttons(self): + def buttons(self): return "SITE CONTENT - BUTTONS" From bf836d4d463188ac1ccf384b01a9e7127c8f52ea Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 9 Nov 2020 15:58:59 -0500 Subject: [PATCH 05/10] Fixes #5331: Fix filtering of sites by null region --- docs/release-notes/version-2.9.md | 1 + netbox/utilities/filters.py | 5 ++--- netbox/utilities/tests/test_filters.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index fe36b010e..eee056dbb 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -12,6 +12,7 @@ * [#5271](https://github.com/netbox-community/netbox/issues/5271) - Fix auto-population of region field when editing a device * [#5314](https://github.com/netbox-community/netbox/issues/5314) - Fix config context rendering when multiple tags are assigned to an object * [#5324](https://github.com/netbox-community/netbox/issues/5324) - Add missing template extension tags for plugins for VM interface view +* [#5331](https://github.com/netbox-community/netbox/issues/5331) - Fix filtering of sites by null region --- diff --git a/netbox/utilities/filters.py b/netbox/utilities/filters.py index f628ca917..18eeab7d2 100644 --- a/netbox/utilities/filters.py +++ b/netbox/utilities/filters.py @@ -68,11 +68,10 @@ class TreeNodeMultipleChoiceFilter(django_filters.ModelMultipleChoiceFilter): """ Filters for a set of Models, including all descendant models within a Tree. Example: [,] """ - def get_filter_predicate(self, v): - # null value filtering + # Null value filtering if v is None: - return {self.field_name.replace('in', 'isnull'): True} + return {f"{self.field_name}__isnull": True} return super().get_filter_predicate(v) def filter(self, qs, value): diff --git a/netbox/utilities/tests/test_filters.py b/netbox/utilities/tests/test_filters.py index f70d7e1db..56eaabd4c 100644 --- a/netbox/utilities/tests/test_filters.py +++ b/netbox/utilities/tests/test_filters.py @@ -23,7 +23,8 @@ class TreeNodeMultipleChoiceFilterTest(TestCase): class SiteFilterSet(django_filters.FilterSet): region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='region__in', + field_name='region', + lookup_expr='in', to_field_name='slug', ) From ab33b3aa84a6342320a239415dab998a29c1c803 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 9 Nov 2020 16:04:09 -0500 Subject: [PATCH 06/10] Fixes #5328: Fix CreatedUpdatedFilterTest when running in non-UTC timezone --- docs/release-notes/version-2.9.md | 1 + netbox/extras/tests/test_api.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index eee056dbb..f4dc03cc5 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -12,6 +12,7 @@ * [#5271](https://github.com/netbox-community/netbox/issues/5271) - Fix auto-population of region field when editing a device * [#5314](https://github.com/netbox-community/netbox/issues/5314) - Fix config context rendering when multiple tags are assigned to an object * [#5324](https://github.com/netbox-community/netbox/issues/5324) - Add missing template extension tags for plugins for VM interface view +* [#5328](https://github.com/netbox-community/netbox/issues/5328) - Fix CreatedUpdatedFilterTest when running in non-UTC timezone * [#5331](https://github.com/netbox-community/netbox/issues/5331) - Fix filtering of sites by null region diff --git a/netbox/extras/tests/test_api.py b/netbox/extras/tests/test_api.py index c768534a2..9e9c418f9 100644 --- a/netbox/extras/tests/test_api.py +++ b/netbox/extras/tests/test_api.py @@ -3,7 +3,7 @@ from unittest import skipIf from django.contrib.contenttypes.models import ContentType from django.urls import reverse -from django.utils import timezone +from django.utils.timezone import make_aware from django_rq.queues import get_connection from rest_framework import status from rq import Worker @@ -369,8 +369,8 @@ class CreatedUpdatedFilterTest(APITestCase): # change the created and last_updated of one Rack.objects.filter(pk=self.rack2.pk).update( - last_updated=datetime.datetime(2001, 2, 3, 1, 2, 3, 4, tzinfo=timezone.utc), - created=datetime.datetime(2001, 2, 3) + last_updated=make_aware(datetime.datetime(2001, 2, 3, 1, 2, 3, 4)), + created=make_aware(datetime.datetime(2001, 2, 3)) ) def test_get_rack_created(self): From dc7da4f0f6821b5081b44238d7be39f4e2006ecc Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 9 Nov 2020 16:22:45 -0500 Subject: [PATCH 07/10] Fixes #5316: Dry running scripts should not trigger webhooks --- docs/release-notes/version-2.9.md | 1 + netbox/extras/scripts.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index f4dc03cc5..63e5d7117 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -11,6 +11,7 @@ * [#5271](https://github.com/netbox-community/netbox/issues/5271) - Fix auto-population of region field when editing a device * [#5314](https://github.com/netbox-community/netbox/issues/5314) - Fix config context rendering when multiple tags are assigned to an object +* [#5316](https://github.com/netbox-community/netbox/issues/5316) - Dry running scripts should not trigger webhooks * [#5324](https://github.com/netbox-community/netbox/issues/5324) - Add missing template extension tags for plugins for VM interface view * [#5328](https://github.com/netbox-community/netbox/issues/5328) - Fix CreatedUpdatedFilterTest when running in non-UTC timezone * [#5331](https://github.com/netbox-community/netbox/issues/5331) - Fix filtering of sites by null region diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 9d5380655..f63f02782 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -441,8 +441,11 @@ def run_script(data, request, commit=True, *args, **kwargs): f"with NetBox v2.10." ) - with change_logging(request): - + def _run_script(): + """ + Core script execution task. We capture this within a subfunction to allow for conditionally wrapping it with + the change_logging context manager (which is bypassed if commit == False). + """ try: with transaction.atomic(): script.output = script.run(**kwargs) @@ -469,6 +472,14 @@ def run_script(data, request, commit=True, *args, **kwargs): logger.info(f"Script completed in {job_result.duration}") + # Execute the script. If commit is True, wrap it with the change_logging context manager to ensure we process + # change logging, webhooks, etc. + if commit: + with change_logging(request): + _run_script() + else: + _run_script() + # Delete any previous terminal state results JobResult.objects.filter( obj_type=job_result.obj_type, From 9b2dc89c6c6367fc3596eede1504e7d49d4f87ae Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 9 Nov 2020 16:48:11 -0500 Subject: [PATCH 08/10] Closes #5327: Be more strict when capturing anticipated ImportError exceptions --- docs/release-notes/version-2.9.md | 1 + netbox/dcim/api/views.py | 16 ++++++++------- netbox/netbox/authentication.py | 23 +++++++++++++-------- netbox/netbox/settings.py | 34 ++++++++++++++++++------------- 4 files changed, 44 insertions(+), 30 deletions(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index 63e5d7117..f96eac458 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -6,6 +6,7 @@ * [#5304](https://github.com/netbox-community/netbox/issues/5304) - Return server error messages as JSON when handling REST API requests * [#5310](https://github.com/netbox-community/netbox/issues/5310) - Link to rack groups within rack list table +* [#5327](https://github.com/netbox-community/netbox/issues/5327) - Be more strict when capturing anticipated ImportError exceptions ### Bug Fixes diff --git a/netbox/dcim/api/views.py b/netbox/dcim/api/views.py index 427aecd5f..2d7107ef4 100644 --- a/netbox/dcim/api/views.py +++ b/netbox/dcim/api/views.py @@ -396,9 +396,7 @@ class DeviceViewSet(CustomFieldModelViewSet, ConfigContextQuerySetMixin): if device.platform is None: raise ServiceUnavailable("No platform is configured for this device.") if not device.platform.napalm_driver: - raise ServiceUnavailable("No NAPALM driver is configured for this device's platform {}.".format( - device.platform - )) + raise ServiceUnavailable(f"No NAPALM driver is configured for this device's platform: {device.platform}.") # Check for primary IP address from NetBox object if device.primary_ip: @@ -407,21 +405,25 @@ class DeviceViewSet(CustomFieldModelViewSet, ConfigContextQuerySetMixin): # Raise exception for no IP address and no Name if device.name does not exist if not device.name: raise ServiceUnavailable( - "This device does not have a primary IP address or device name to lookup configured.") + "This device does not have a primary IP address or device name to lookup configured." + ) try: # Attempt to complete a DNS name resolution if no primary_ip is set host = socket.gethostbyname(device.name) except socket.gaierror: # Name lookup failure raise ServiceUnavailable( - f"Name lookup failure, unable to resolve IP address for {device.name}. Please set Primary IP or setup name resolution.") + f"Name lookup failure, unable to resolve IP address for {device.name}. Please set Primary IP or " + f"setup name resolution.") # Check that NAPALM is installed try: import napalm from napalm.base.exceptions import ModuleImportError - except ImportError: - raise ServiceUnavailable("NAPALM is not installed. Please see the documentation for instructions.") + except ModuleNotFoundError as e: + if getattr(e, 'name') == 'napalm': + raise ServiceUnavailable("NAPALM is not installed. Please see the documentation for instructions.") + raise e # Validate the configured driver try: diff --git a/netbox/netbox/authentication.py b/netbox/netbox/authentication.py index 21fb3e229..0eee2c13e 100644 --- a/netbox/netbox/authentication.py +++ b/netbox/netbox/authentication.py @@ -137,19 +137,24 @@ class LDAPBackend: def __new__(cls, *args, **kwargs): try: - import ldap from django_auth_ldap.backend import LDAPBackend as LDAPBackend_, LDAPSettings - except ImportError: - raise ImproperlyConfigured( - "LDAP authentication has been configured, but django-auth-ldap is not installed." - ) + import ldap + except ModuleNotFoundError as e: + if getattr(e, 'name') == 'django_auth_ldap': + raise ImproperlyConfigured( + "LDAP authentication has been configured, but django-auth-ldap is not installed." + ) + raise e try: from netbox import ldap_config - except ImportError: - raise ImproperlyConfigured( - "ldap_config.py does not exist" - ) + except ModuleNotFoundError as e: + if getattr(e, 'name') == 'ldap_config': + raise ImproperlyConfigured( + "LDAP configuration file not found: Check that ldap_config.py has been created alongside " + "configuration.py." + ) + raise e try: getattr(ldap_config, 'AUTH_LDAP_SERVER_URI') diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 36d3fb980..824739345 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -38,10 +38,12 @@ if platform.python_version_tuple() < ('3', '6'): # Import configuration parameters try: from netbox import configuration -except ImportError: - raise ImproperlyConfigured( - "Configuration file is not present. Please define netbox/netbox/configuration.py per the documentation." - ) +except ModuleNotFoundError as e: + if getattr(e, 'name') == 'configuration': + raise ImproperlyConfigured( + "Configuration file is not present. Please define netbox/netbox/configuration.py per the documentation." + ) + raise # Enforce required configuration parameters for parameter in ['ALLOWED_HOSTS', 'DATABASE', 'SECRET_KEY', 'REDIS']: @@ -183,11 +185,13 @@ if STORAGE_BACKEND is not None: try: import storages.utils - except ImportError: - raise ImproperlyConfigured( - "STORAGE_BACKEND is set to {} but django-storages is not present. It can be installed by running 'pip " - "install django-storages'.".format(STORAGE_BACKEND) - ) + except ModuleNotFoundError as e: + if getattr(e, 'name') == 'storages': + raise ImproperlyConfigured( + f"STORAGE_BACKEND is set to {STORAGE_BACKEND} but django-storages is not present. It can be " + f"installed by running 'pip install django-storages'." + ) + raise e # Monkey-patch django-storages to fetch settings from STORAGE_CONFIG def _setting(name, default=None): @@ -596,11 +600,13 @@ for plugin_name in PLUGINS: # Import plugin module try: plugin = importlib.import_module(plugin_name) - except ImportError: - raise ImproperlyConfigured( - "Unable to import plugin {}: Module not found. Check that the plugin module has been installed within the " - "correct Python environment.".format(plugin_name) - ) + except ModuleNotFoundError as e: + if getattr(e, 'name') == plugin_name: + raise ImproperlyConfigured( + "Unable to import plugin {}: Module not found. Check that the plugin module has been installed within the " + "correct Python environment.".format(plugin_name) + ) + raise e # Determine plugin config and add to INSTALLED_APPS. try: From a6f01548032b9ff09076a453cbc7100ab7fcb9ab Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 9 Nov 2020 20:51:35 -0500 Subject: [PATCH 09/10] Release v2.9.9 --- docs/release-notes/version-2.9.md | 2 +- netbox/netbox/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md index f96eac458..6c7839818 100644 --- a/docs/release-notes/version-2.9.md +++ b/docs/release-notes/version-2.9.md @@ -1,6 +1,6 @@ # NetBox v2.9 -## v2.9.9 (FUTURE) +## v2.9.9 (2020-11-09) ### Enhancements diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 824739345..39d8f5871 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -16,7 +16,7 @@ from django.core.validators import URLValidator # Environment setup # -VERSION = '2.9.9-dev' +VERSION = '2.9.9' # Hostname HOSTNAME = platform.node() From 340e665817b2d7d40f1fb96bbea43e3c69df36f0 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 9 Nov 2020 20:57:35 -0500 Subject: [PATCH 10/10] Post release version bump --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 39d8f5871..df48449c9 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -16,7 +16,7 @@ from django.core.validators import URLValidator # Environment setup # -VERSION = '2.9.9' +VERSION = '2.9.10-dev' # Hostname HOSTNAME = platform.node()