From 701f40e2a8e46ebcf8bf11707f1c418bbc06c9c8 Mon Sep 17 00:00:00 2001 From: Alexander Haase Date: Sun, 16 Feb 2025 20:04:12 +0100 Subject: [PATCH 01/25] Show parent contacts for nested models When contacts of a nested model are displayed, the contacts of the parents are also displayed. --- netbox/tenancy/views.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/netbox/tenancy/views.py b/netbox/tenancy/views.py index 0988d2e65..9bb542f82 100644 --- a/netbox/tenancy/views.py +++ b/netbox/tenancy/views.py @@ -2,6 +2,7 @@ from django.contrib.contenttypes.models import ContentType from django.shortcuts import get_object_or_404 from django.utils.translation import gettext_lazy as _ +from netbox.models import NestedGroupModel from netbox.views import generic from utilities.query import count_related from utilities.views import GetRelatedModelsMixin, ViewTab, register_model_view @@ -23,19 +24,18 @@ class ObjectContactsView(generic.ObjectChildrenView): ) def get_children(self, request, parent): - return ContactAssignment.objects.restrict(request.user, 'view').filter( - object_type=ContentType.objects.get_for_model(parent), - object_id=parent.pk - ).order_by('priority', 'contact', 'role') + qs = ContactAssignment.objects.restrict(request.user, 'view') + for obj in [parent]: + qs = qs.filter( + object_type=ContentType.objects.get_for_model(obj), + object_id__in=( + obj.get_ancestors(include_self=True).values_list('pk', flat=True) + if isinstance(obj, NestedGroupModel) + else [obj.pk] + ), + ) - def get_table(self, *args, **kwargs): - table = super().get_table(*args, **kwargs) - - # Hide object columns - table.columns.hide('object_type') - table.columns.hide('object') - - return table + return qs.order_by('priority', 'contact', 'role') # From d5316de9c84daf258b43af002e1f156ac41c18cb Mon Sep 17 00:00:00 2001 From: Alexander Haase Date: Tue, 18 Feb 2025 23:02:57 +0100 Subject: [PATCH 02/25] Move contact queryset into model --- netbox/netbox/models/features.py | 21 +++++++++++++++++++++ netbox/tenancy/views.py | 16 ++-------------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index a97227770..70027a9fc 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -5,6 +5,7 @@ from functools import cached_property from django.contrib.contenttypes.fields import GenericRelation from django.core.validators import ValidationError from django.db import models +from django.db.models import Q from django.utils import timezone from django.utils.translation import gettext_lazy as _ from taggit.managers import TaggableManager @@ -363,6 +364,26 @@ class ContactsMixin(models.Model): class Meta: abstract = True + def get_contacts(self): + """ + Return a `QuerySet` matching all contacts assigned to this object. + """ + from tenancy.models import ContactAssignment + from . import NestedGroupModel + + filter = Q() + for obj in [self]: + filter |= Q( + object_type=ObjectType.objects.get_for_model(obj), + object_id__in=( + obj.get_ancestors(include_self=True).values_list('pk', flat=True) + if isinstance(obj, NestedGroupModel) + else [obj.pk] + ), + ) + + return ContactAssignment.objects.filter(filter) + class BookmarksMixin(models.Model): """ diff --git a/netbox/tenancy/views.py b/netbox/tenancy/views.py index 9bb542f82..3b5029bd7 100644 --- a/netbox/tenancy/views.py +++ b/netbox/tenancy/views.py @@ -2,7 +2,6 @@ from django.contrib.contenttypes.models import ContentType from django.shortcuts import get_object_or_404 from django.utils.translation import gettext_lazy as _ -from netbox.models import NestedGroupModel from netbox.views import generic from utilities.query import count_related from utilities.views import GetRelatedModelsMixin, ViewTab, register_model_view @@ -18,24 +17,13 @@ class ObjectContactsView(generic.ObjectChildrenView): template_name = 'tenancy/object_contacts.html' tab = ViewTab( label=_('Contacts'), - badge=lambda obj: obj.contacts.count(), + badge=lambda obj: obj.get_contacts().count(), permission='tenancy.view_contactassignment', weight=5000 ) def get_children(self, request, parent): - qs = ContactAssignment.objects.restrict(request.user, 'view') - for obj in [parent]: - qs = qs.filter( - object_type=ContentType.objects.get_for_model(obj), - object_id__in=( - obj.get_ancestors(include_self=True).values_list('pk', flat=True) - if isinstance(obj, NestedGroupModel) - else [obj.pk] - ), - ) - - return qs.order_by('priority', 'contact', 'role') + return parent.get_contacts().restrict(request.user, 'view').order_by('priority', 'contact', 'role') # From 72adda11974f81fbc0cc9edfc74654009c74b1b9 Mon Sep 17 00:00:00 2001 From: Alexander Haase Date: Tue, 18 Feb 2025 23:08:47 +0100 Subject: [PATCH 03/25] Allow exclusion of inherited contacts --- netbox/netbox/models/features.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index 70027a9fc..ba895d5ed 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -364,9 +364,11 @@ class ContactsMixin(models.Model): class Meta: abstract = True - def get_contacts(self): + def get_contacts(self, inherited=True): """ Return a `QuerySet` matching all contacts assigned to this object. + + :param inherited: If `True`, inherited contacts from parent objects are included. """ from tenancy.models import ContactAssignment from . import NestedGroupModel @@ -377,7 +379,7 @@ class ContactsMixin(models.Model): object_type=ObjectType.objects.get_for_model(obj), object_id__in=( obj.get_ancestors(include_self=True).values_list('pk', flat=True) - if isinstance(obj, NestedGroupModel) + if (isinstance(obj, NestedGroupModel) and inherited) else [obj.pk] ), ) From ef89fc1264534539895dd2066b0230226b18dd87 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 20 Feb 2025 14:53:53 -0500 Subject: [PATCH 04/25] Closes #18071: Remvoe legacy staged changes functionality --- docs/models/extras/branch.md | 16 -- docs/models/extras/stagedchange.md | 29 --- docs/plugins/development/staged-changes.md | 39 ---- mkdocs.yml | 3 - netbox/extras/choices.py | 17 -- .../extras/migrations/0123_remove_staging.py | 27 +++ netbox/extras/models/__init__.py | 1 - netbox/extras/models/staging.py | 150 ------------ netbox/netbox/staging.py | 148 ------------ netbox/netbox/tests/test_staging.py | 216 ------------------ 10 files changed, 27 insertions(+), 619 deletions(-) delete mode 100644 docs/models/extras/branch.md delete mode 100644 docs/models/extras/stagedchange.md delete mode 100644 docs/plugins/development/staged-changes.md create mode 100644 netbox/extras/migrations/0123_remove_staging.py delete mode 100644 netbox/extras/models/staging.py delete mode 100644 netbox/netbox/staging.py delete mode 100644 netbox/netbox/tests/test_staging.py diff --git a/docs/models/extras/branch.md b/docs/models/extras/branch.md deleted file mode 100644 index 4599fed85..000000000 --- a/docs/models/extras/branch.md +++ /dev/null @@ -1,16 +0,0 @@ -# Branches - -!!! danger "Deprecated Feature" - This feature has been deprecated in NetBox v4.2 and will be removed in a future release. Please consider using the [netbox-branching plugin](https://github.com/netboxlabs/netbox-branching), which provides much more robust functionality. - -A branch is a collection of related [staged changes](./stagedchange.md) that have been prepared for merging into the active database. A branch can be merged by executing its `commit()` method. Deleting a branch will delete all its related changes. - -## Fields - -### Name - -The branch's name. - -### User - -The user to which the branch belongs (optional). diff --git a/docs/models/extras/stagedchange.md b/docs/models/extras/stagedchange.md deleted file mode 100644 index 0693a32d3..000000000 --- a/docs/models/extras/stagedchange.md +++ /dev/null @@ -1,29 +0,0 @@ -# Staged Changes - -!!! danger "Deprecated Feature" - This feature has been deprecated in NetBox v4.2 and will be removed in a future release. Please consider using the [netbox-branching plugin](https://github.com/netboxlabs/netbox-branching), which provides much more robust functionality. - -A staged change represents the creation of a new object or the modification or deletion of an existing object to be performed at some future point. Each change must be assigned to a [branch](./branch.md). - -Changes can be applied individually via the `apply()` method, however it is recommended to apply changes in bulk using the parent branch's `commit()` method. - -## Fields - -!!! warning - Staged changes are not typically created or manipulated directly, but rather effected through the use of the [`checkout()`](../../plugins/development/staged-changes.md) context manager. - -### Branch - -The [branch](./branch.md) to which this change belongs. - -### Action - -The type of action this change represents: `create`, `update`, or `delete`. - -### Object - -A generic foreign key referencing the existing object to which this change applies. - -### Data - -JSON representation of the changes being made to the object (not applicable for deletions). diff --git a/docs/plugins/development/staged-changes.md b/docs/plugins/development/staged-changes.md deleted file mode 100644 index a8fd1d232..000000000 --- a/docs/plugins/development/staged-changes.md +++ /dev/null @@ -1,39 +0,0 @@ -# Staged Changes - -!!! danger "Deprecated Feature" - This feature has been deprecated in NetBox v4.2 and will be removed in a future release. Please consider using the [netbox-branching plugin](https://github.com/netboxlabs/netbox-branching), which provides much more robust functionality. - -NetBox provides a programmatic API to stage the creation, modification, and deletion of objects without actually committing those changes to the active database. This can be useful for performing a "dry run" of bulk operations, or preparing a set of changes for administrative approval, for example. - -To begin staging changes, first create a [branch](../../models/extras/branch.md): - -```python -from extras.models import Branch - -branch1 = Branch.objects.create(name='branch1') -``` - -Then, activate the branch using the `checkout()` context manager and begin making your changes. This initiates a new database transaction. - -```python -from extras.models import Branch -from netbox.staging import checkout - -branch1 = Branch.objects.get(name='branch1') -with checkout(branch1): - Site.objects.create(name='New Site', slug='new-site') - # ... -``` - -Upon exiting the context, the database transaction is automatically rolled back and your changes recorded as [staged changes](../../models/extras/stagedchange.md). Re-entering a branch will trigger a new database transaction and automatically apply any staged changes associated with the branch. - -To apply the changes within a branch, call the branch's `commit()` method: - -```python -from extras.models import Branch - -branch1 = Branch.objects.get(name='branch1') -branch1.commit() -``` - -Committing a branch is an all-or-none operation: Any exceptions will revert the entire set of changes. After successfully committing a branch, all its associated StagedChange objects are automatically deleted (however the branch itself will remain and can be reused). diff --git a/mkdocs.yml b/mkdocs.yml index db6798eae..a5b2d5355 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -150,7 +150,6 @@ nav: - GraphQL API: 'plugins/development/graphql-api.md' - Background Jobs: 'plugins/development/background-jobs.md' - Dashboard Widgets: 'plugins/development/dashboard-widgets.md' - - Staged Changes: 'plugins/development/staged-changes.md' - Exceptions: 'plugins/development/exceptions.md' - Migrating to v4.0: 'plugins/development/migration-v4.md' - Administration: @@ -226,7 +225,6 @@ nav: - VirtualDeviceContext: 'models/dcim/virtualdevicecontext.md' - Extras: - Bookmark: 'models/extras/bookmark.md' - - Branch: 'models/extras/branch.md' - ConfigContext: 'models/extras/configcontext.md' - ConfigTemplate: 'models/extras/configtemplate.md' - CustomField: 'models/extras/customfield.md' @@ -239,7 +237,6 @@ nav: - Notification: 'models/extras/notification.md' - NotificationGroup: 'models/extras/notificationgroup.md' - SavedFilter: 'models/extras/savedfilter.md' - - StagedChange: 'models/extras/stagedchange.md' - Subscription: 'models/extras/subscription.md' - Tag: 'models/extras/tag.md' - Webhook: 'models/extras/webhook.md' diff --git a/netbox/extras/choices.py b/netbox/extras/choices.py index 4525d8689..3ecc7e57f 100644 --- a/netbox/extras/choices.py +++ b/netbox/extras/choices.py @@ -212,23 +212,6 @@ class WebhookHttpMethodChoices(ChoiceSet): ) -# -# Staging -# - -class ChangeActionChoices(ChoiceSet): - - ACTION_CREATE = 'create' - ACTION_UPDATE = 'update' - ACTION_DELETE = 'delete' - - CHOICES = ( - (ACTION_CREATE, _('Create'), 'green'), - (ACTION_UPDATE, _('Update'), 'blue'), - (ACTION_DELETE, _('Delete'), 'red'), - ) - - # # Dashboard widgets # diff --git a/netbox/extras/migrations/0123_remove_staging.py b/netbox/extras/migrations/0123_remove_staging.py new file mode 100644 index 000000000..643cd912d --- /dev/null +++ b/netbox/extras/migrations/0123_remove_staging.py @@ -0,0 +1,27 @@ +# Generated by Django 5.1.5 on 2025-02-20 19:46 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('extras', '0122_charfield_null_choices'), + ] + + operations = [ + migrations.RemoveField( + model_name='stagedchange', + name='branch', + ), + migrations.RemoveField( + model_name='stagedchange', + name='object_type', + ), + migrations.DeleteModel( + name='Branch', + ), + migrations.DeleteModel( + name='StagedChange', + ), + ] diff --git a/netbox/extras/models/__init__.py b/netbox/extras/models/__init__.py index e85721034..f214b1268 100644 --- a/netbox/extras/models/__init__.py +++ b/netbox/extras/models/__init__.py @@ -5,5 +5,4 @@ from .models import * from .notifications import * from .scripts import * from .search import * -from .staging import * from .tags import * diff --git a/netbox/extras/models/staging.py b/netbox/extras/models/staging.py deleted file mode 100644 index 68d37de7f..000000000 --- a/netbox/extras/models/staging.py +++ /dev/null @@ -1,150 +0,0 @@ -import logging -import warnings - -from django.contrib.contenttypes.fields import GenericForeignKey -from django.db import models, transaction -from django.utils.translation import gettext_lazy as _ -from mptt.models import MPTTModel - -from extras.choices import ChangeActionChoices -from netbox.models import ChangeLoggedModel -from netbox.models.features import * -from utilities.serialization import deserialize_object - -__all__ = ( - 'Branch', - 'StagedChange', -) - -logger = logging.getLogger('netbox.staging') - - -class Branch(ChangeLoggedModel): - """ - A collection of related StagedChanges. - """ - name = models.CharField( - verbose_name=_('name'), - max_length=100, - unique=True - ) - description = models.CharField( - verbose_name=_('description'), - max_length=200, - blank=True - ) - user = models.ForeignKey( - to='users.User', - on_delete=models.SET_NULL, - blank=True, - null=True - ) - - class Meta: - ordering = ('name',) - verbose_name = _('branch') - verbose_name_plural = _('branches') - - def __init__(self, *args, **kwargs): - warnings.warn( - 'The staged changes functionality has been deprecated and will be removed in a future release.', - DeprecationWarning - ) - super().__init__(*args, **kwargs) - - def __str__(self): - return f'{self.name} ({self.pk})' - - def merge(self): - logger.info(f'Merging changes in branch {self}') - with transaction.atomic(): - for change in self.staged_changes.all(): - change.apply() - self.staged_changes.all().delete() - - -class StagedChange(CustomValidationMixin, EventRulesMixin, models.Model): - """ - The prepared creation, modification, or deletion of an object to be applied to the active database at a - future point. - """ - branch = models.ForeignKey( - to=Branch, - on_delete=models.CASCADE, - related_name='staged_changes' - ) - action = models.CharField( - verbose_name=_('action'), - max_length=20, - choices=ChangeActionChoices - ) - object_type = models.ForeignKey( - to='contenttypes.ContentType', - on_delete=models.CASCADE, - related_name='+' - ) - object_id = models.PositiveBigIntegerField( - blank=True, - null=True - ) - object = GenericForeignKey( - ct_field='object_type', - fk_field='object_id' - ) - data = models.JSONField( - verbose_name=_('data'), - blank=True, - null=True - ) - - class Meta: - ordering = ('pk',) - indexes = ( - models.Index(fields=('object_type', 'object_id')), - ) - verbose_name = _('staged change') - verbose_name_plural = _('staged changes') - - def __init__(self, *args, **kwargs): - warnings.warn( - 'The staged changes functionality has been deprecated and will be removed in a future release.', - DeprecationWarning - ) - super().__init__(*args, **kwargs) - - def __str__(self): - action = self.get_action_display() - app_label, model_name = self.object_type.natural_key() - return f"{action} {app_label}.{model_name} ({self.object_id})" - - @property - def model(self): - return self.object_type.model_class() - - def apply(self): - """ - Apply the staged create/update/delete action to the database. - """ - if self.action == ChangeActionChoices.ACTION_CREATE: - instance = deserialize_object(self.model, self.data, pk=self.object_id) - logger.info(f'Creating {self.model._meta.verbose_name} {instance}') - instance.save() - - if self.action == ChangeActionChoices.ACTION_UPDATE: - instance = deserialize_object(self.model, self.data, pk=self.object_id) - logger.info(f'Updating {self.model._meta.verbose_name} {instance}') - instance.save() - - if self.action == ChangeActionChoices.ACTION_DELETE: - instance = self.model.objects.get(pk=self.object_id) - logger.info(f'Deleting {self.model._meta.verbose_name} {instance}') - instance.delete() - - # Rebuild the MPTT tree where applicable - if issubclass(self.model, MPTTModel): - self.model.objects.rebuild() - - apply.alters_data = True - - def get_action_color(self): - return ChangeActionChoices.colors.get(self.action) diff --git a/netbox/netbox/staging.py b/netbox/netbox/staging.py deleted file mode 100644 index e6b946403..000000000 --- a/netbox/netbox/staging.py +++ /dev/null @@ -1,148 +0,0 @@ -import logging - -from django.contrib.contenttypes.models import ContentType -from django.db import transaction -from django.db.models.signals import m2m_changed, pre_delete, post_save - -from extras.choices import ChangeActionChoices -from extras.models import StagedChange -from utilities.serialization import serialize_object - -logger = logging.getLogger('netbox.staging') - - -class checkout: - """ - Context manager for staging changes to NetBox objects. Staged changes are saved out-of-band - (as Change instances) for application at a later time, without modifying the production - database. - - branch = Branch.objects.create(name='my-branch') - with checkout(branch): - # All changes made herein will be rolled back and stored for later - - Note that invoking the context disabled transaction autocommit to facilitate manual rollbacks, - and restores its original value upon exit. - """ - def __init__(self, branch): - self.branch = branch - self.queue = {} - - def __enter__(self): - - # Disable autocommit to effect a new transaction - logger.debug(f"Entering transaction for {self.branch}") - self._autocommit = transaction.get_autocommit() - transaction.set_autocommit(False) - - # Apply any existing Changes assigned to this Branch - staged_changes = self.branch.staged_changes.all() - if change_count := staged_changes.count(): - logger.debug(f"Applying {change_count} pre-staged changes...") - for change in staged_changes: - change.apply() - else: - logger.debug("No pre-staged changes found") - - # Connect signal handlers - logger.debug("Connecting signal handlers") - post_save.connect(self.post_save_handler) - m2m_changed.connect(self.post_save_handler) - pre_delete.connect(self.pre_delete_handler) - - def __exit__(self, exc_type, exc_val, exc_tb): - - # Disconnect signal handlers - logger.debug("Disconnecting signal handlers") - post_save.disconnect(self.post_save_handler) - m2m_changed.disconnect(self.post_save_handler) - pre_delete.disconnect(self.pre_delete_handler) - - # Roll back the transaction to return the database to its original state - logger.debug("Rolling back database transaction") - transaction.rollback() - logger.debug(f"Restoring autocommit state ({self._autocommit})") - transaction.set_autocommit(self._autocommit) - - # Process queued changes - self.process_queue() - - # - # Queuing - # - - @staticmethod - def get_key_for_instance(instance): - return ContentType.objects.get_for_model(instance), instance.pk - - def process_queue(self): - """ - Create Change instances for all actions stored in the queue. - """ - if not self.queue: - logger.debug("No queued changes; aborting") - return - logger.debug(f"Processing {len(self.queue)} queued changes") - - # Iterate through the in-memory queue, creating Change instances - changes = [] - for key, change in self.queue.items(): - logger.debug(f' {key}: {change}') - object_type, pk = key - action, data = change - - changes.append(StagedChange( - branch=self.branch, - action=action, - object_type=object_type, - object_id=pk, - data=data - )) - - # Save all Change instances to the database - StagedChange.objects.bulk_create(changes) - - # - # Signal handlers - # - - def post_save_handler(self, sender, instance, **kwargs): - """ - Hooks to the post_save signal when a branch is active to queue create and update actions. - """ - key = self.get_key_for_instance(instance) - object_type = instance._meta.verbose_name - - # Creating a new object - if kwargs.get('created'): - logger.debug(f"[{self.branch}] Staging creation of {object_type} {instance} (PK: {instance.pk})") - data = serialize_object(instance, resolve_tags=False) - self.queue[key] = (ChangeActionChoices.ACTION_CREATE, data) - return - - # Ignore pre_* many-to-many actions - if 'action' in kwargs and kwargs['action'] not in ('post_add', 'post_remove', 'post_clear'): - return - - # Object has already been created/updated in the queue; update its queued representation - if key in self.queue: - logger.debug(f"[{self.branch}] Updating staged value for {object_type} {instance} (PK: {instance.pk})") - data = serialize_object(instance, resolve_tags=False) - self.queue[key] = (self.queue[key][0], data) - return - - # Modifying an existing object for the first time - logger.debug(f"[{self.branch}] Staging changes to {object_type} {instance} (PK: {instance.pk})") - data = serialize_object(instance, resolve_tags=False) - self.queue[key] = (ChangeActionChoices.ACTION_UPDATE, data) - - def pre_delete_handler(self, sender, instance, **kwargs): - """ - Hooks to the pre_delete signal when a branch is active to queue delete actions. - """ - key = self.get_key_for_instance(instance) - object_type = instance._meta.verbose_name - - # Delete an existing object - logger.debug(f"[{self.branch}] Staging deletion of {object_type} {instance} (PK: {instance.pk})") - self.queue[key] = (ChangeActionChoices.ACTION_DELETE, None) diff --git a/netbox/netbox/tests/test_staging.py b/netbox/netbox/tests/test_staging.py deleted file mode 100644 index 0a73b2987..000000000 --- a/netbox/netbox/tests/test_staging.py +++ /dev/null @@ -1,216 +0,0 @@ -from django.db.models.signals import post_save -from django.test import TransactionTestCase - -from circuits.models import Provider, Circuit, CircuitType -from extras.choices import ChangeActionChoices -from extras.models import Branch, StagedChange, Tag -from ipam.models import ASN, RIR -from netbox.search.backends import search_backend -from netbox.staging import checkout -from utilities.testing import create_tags - - -class StagingTestCase(TransactionTestCase): - - def setUp(self): - # Disconnect search backend to avoid issues with cached ObjectTypes being deleted - # from the database upon transaction rollback - post_save.disconnect(search_backend.caching_handler) - - create_tags('Alpha', 'Bravo', 'Charlie') - - rir = RIR.objects.create(name='RIR 1', slug='rir-1') - asns = ( - ASN(asn=65001, rir=rir), - ASN(asn=65002, rir=rir), - ASN(asn=65003, rir=rir), - ) - ASN.objects.bulk_create(asns) - - providers = ( - Provider(name='Provider A', slug='provider-a'), - Provider(name='Provider B', slug='provider-b'), - Provider(name='Provider C', slug='provider-c'), - ) - Provider.objects.bulk_create(providers) - - circuit_type = CircuitType.objects.create(name='Circuit Type 1', slug='circuit-type-1') - - Circuit.objects.bulk_create(( - Circuit(provider=providers[0], cid='Circuit A1', type=circuit_type), - Circuit(provider=providers[0], cid='Circuit A2', type=circuit_type), - Circuit(provider=providers[0], cid='Circuit A3', type=circuit_type), - Circuit(provider=providers[1], cid='Circuit B1', type=circuit_type), - Circuit(provider=providers[1], cid='Circuit B2', type=circuit_type), - Circuit(provider=providers[1], cid='Circuit B3', type=circuit_type), - Circuit(provider=providers[2], cid='Circuit C1', type=circuit_type), - Circuit(provider=providers[2], cid='Circuit C2', type=circuit_type), - Circuit(provider=providers[2], cid='Circuit C3', type=circuit_type), - )) - - def test_object_creation(self): - branch = Branch.objects.create(name='Branch 1') - tags = Tag.objects.all() - asns = ASN.objects.all() - - with checkout(branch): - provider = Provider.objects.create(name='Provider D', slug='provider-d') - provider.asns.set(asns) - circuit = Circuit.objects.create(provider=provider, cid='Circuit D1', type=CircuitType.objects.first()) - circuit.tags.set(tags) - - # Sanity-checking - self.assertEqual(Provider.objects.count(), 4) - self.assertListEqual(list(provider.asns.all()), list(asns)) - self.assertEqual(Circuit.objects.count(), 10) - self.assertListEqual(list(circuit.tags.all()), list(tags)) - - # Verify that changes have been rolled back after exiting the context - self.assertEqual(Provider.objects.count(), 3) - self.assertEqual(Circuit.objects.count(), 9) - self.assertEqual(StagedChange.objects.count(), 5) - - # Verify that changes are replayed upon entering the context - with checkout(branch): - self.assertEqual(Provider.objects.count(), 4) - self.assertEqual(Circuit.objects.count(), 10) - provider = Provider.objects.get(name='Provider D') - self.assertListEqual(list(provider.asns.all()), list(asns)) - circuit = Circuit.objects.get(cid='Circuit D1') - self.assertListEqual(list(circuit.tags.all()), list(tags)) - - # Verify that changes are applied and deleted upon branch merge - branch.merge() - self.assertEqual(Provider.objects.count(), 4) - self.assertEqual(Circuit.objects.count(), 10) - provider = Provider.objects.get(name='Provider D') - self.assertListEqual(list(provider.asns.all()), list(asns)) - circuit = Circuit.objects.get(cid='Circuit D1') - self.assertListEqual(list(circuit.tags.all()), list(tags)) - self.assertEqual(StagedChange.objects.count(), 0) - - def test_object_modification(self): - branch = Branch.objects.create(name='Branch 1') - tags = Tag.objects.all() - asns = ASN.objects.all() - - with checkout(branch): - provider = Provider.objects.get(name='Provider A') - provider.name = 'Provider X' - provider.save() - provider.asns.set(asns) - circuit = Circuit.objects.get(cid='Circuit A1') - circuit.cid = 'Circuit X' - circuit.save() - circuit.tags.set(tags) - - # Sanity-checking - self.assertEqual(Provider.objects.count(), 3) - self.assertEqual(Provider.objects.get(pk=provider.pk).name, 'Provider X') - self.assertListEqual(list(provider.asns.all()), list(asns)) - self.assertEqual(Circuit.objects.count(), 9) - self.assertEqual(Circuit.objects.get(pk=circuit.pk).cid, 'Circuit X') - self.assertListEqual(list(circuit.tags.all()), list(tags)) - - # Verify that changes have been rolled back after exiting the context - self.assertEqual(Provider.objects.count(), 3) - self.assertEqual(Provider.objects.get(pk=provider.pk).name, 'Provider A') - provider = Provider.objects.get(pk=provider.pk) - self.assertListEqual(list(provider.asns.all()), []) - self.assertEqual(Circuit.objects.count(), 9) - circuit = Circuit.objects.get(pk=circuit.pk) - self.assertEqual(circuit.cid, 'Circuit A1') - self.assertListEqual(list(circuit.tags.all()), []) - self.assertEqual(StagedChange.objects.count(), 5) - - # Verify that changes are replayed upon entering the context - with checkout(branch): - self.assertEqual(Provider.objects.count(), 3) - self.assertEqual(Provider.objects.get(pk=provider.pk).name, 'Provider X') - provider = Provider.objects.get(pk=provider.pk) - self.assertListEqual(list(provider.asns.all()), list(asns)) - self.assertEqual(Circuit.objects.count(), 9) - circuit = Circuit.objects.get(pk=circuit.pk) - self.assertEqual(circuit.cid, 'Circuit X') - self.assertListEqual(list(circuit.tags.all()), list(tags)) - - # Verify that changes are applied and deleted upon branch merge - branch.merge() - self.assertEqual(Provider.objects.count(), 3) - self.assertEqual(Provider.objects.get(pk=provider.pk).name, 'Provider X') - provider = Provider.objects.get(pk=provider.pk) - self.assertListEqual(list(provider.asns.all()), list(asns)) - self.assertEqual(Circuit.objects.count(), 9) - circuit = Circuit.objects.get(pk=circuit.pk) - self.assertEqual(circuit.cid, 'Circuit X') - self.assertListEqual(list(circuit.tags.all()), list(tags)) - self.assertEqual(StagedChange.objects.count(), 0) - - def test_object_deletion(self): - branch = Branch.objects.create(name='Branch 1') - - with checkout(branch): - provider = Provider.objects.get(name='Provider A') - provider.circuits.all().delete() - provider.delete() - - # Sanity-checking - self.assertEqual(Provider.objects.count(), 2) - self.assertEqual(Circuit.objects.count(), 6) - - # Verify that changes have been rolled back after exiting the context - self.assertEqual(Provider.objects.count(), 3) - self.assertEqual(Circuit.objects.count(), 9) - self.assertEqual(StagedChange.objects.count(), 4) - - # Verify that changes are replayed upon entering the context - with checkout(branch): - self.assertEqual(Provider.objects.count(), 2) - self.assertEqual(Circuit.objects.count(), 6) - - # Verify that changes are applied and deleted upon branch merge - branch.merge() - self.assertEqual(Provider.objects.count(), 2) - self.assertEqual(Circuit.objects.count(), 6) - self.assertEqual(StagedChange.objects.count(), 0) - - def test_exit_enter_context(self): - branch = Branch.objects.create(name='Branch 1') - - with checkout(branch): - - # Create a new object - provider = Provider.objects.create(name='Provider D', slug='provider-d') - provider.save() - - # Check that a create Change was recorded - self.assertEqual(StagedChange.objects.count(), 1) - change = StagedChange.objects.first() - self.assertEqual(change.action, ChangeActionChoices.ACTION_CREATE) - self.assertEqual(change.data['name'], provider.name) - - with checkout(branch): - - # Update the staged object - provider = Provider.objects.get(name='Provider D') - provider.comments = 'New comments' - provider.save() - - # Check that a second Change object has been created for the object - self.assertEqual(StagedChange.objects.count(), 2) - change = StagedChange.objects.last() - self.assertEqual(change.action, ChangeActionChoices.ACTION_UPDATE) - self.assertEqual(change.data['name'], provider.name) - self.assertEqual(change.data['comments'], provider.comments) - - with checkout(branch): - - # Delete the staged object - provider = Provider.objects.get(name='Provider D') - provider.delete() - - # Check that a third Change has recorded the object's deletion - self.assertEqual(StagedChange.objects.count(), 3) - change = StagedChange.objects.last() - self.assertEqual(change.action, ChangeActionChoices.ACTION_DELETE) - self.assertIsNone(change.data) From ca6b686b88ac68550712adf74b4f68cbcac3f8ee Mon Sep 17 00:00:00 2001 From: Alexander Haase Date: Sat, 22 Feb 2025 00:06:44 +0100 Subject: [PATCH 05/25] Limit inherited contacts to model --- netbox/netbox/models/features.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index ba895d5ed..60084c361 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -373,16 +373,14 @@ class ContactsMixin(models.Model): from tenancy.models import ContactAssignment from . import NestedGroupModel - filter = Q() - for obj in [self]: - filter |= Q( - object_type=ObjectType.objects.get_for_model(obj), - object_id__in=( - obj.get_ancestors(include_self=True).values_list('pk', flat=True) - if (isinstance(obj, NestedGroupModel) and inherited) - else [obj.pk] - ), - ) + filter = Q( + object_type=ObjectType.objects.get_for_model(self), + object_id__in=( + self.get_ancestors(include_self=True).values_list('pk', flat=True) + if (isinstance(self, NestedGroupModel) and inherited) + else [self.pk] + ), + ) return ContactAssignment.objects.filter(filter) From 2eaee8bf45d4620ad5b624331a5d37993acbd0f6 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Sun, 16 Feb 2025 09:47:10 +0100 Subject: [PATCH 06/25] Close #18635: Show only the semantic version This modifies the 'netbox-version' to only show the semantic version of Netbox and adds 'netbox-full-version' to show the full version. Related issues: - https://github.com/netbox-community/netbox/issues/15908 - https://github.com/netbox-community/ansible_modules/issues/1381 --- netbox/netbox/api/views.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/netbox/netbox/api/views.py b/netbox/netbox/api/views.py index d58d1affe..1befda371 100644 --- a/netbox/netbox/api/views.py +++ b/netbox/netbox/api/views.py @@ -4,15 +4,15 @@ from django import __version__ as DJANGO_VERSION from django.apps import apps from django.conf import settings from django_rq.queues import get_connection -from drf_spectacular.utils import extend_schema from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema from rest_framework.response import Response from rest_framework.reverse import reverse from rest_framework.views import APIView from rq.worker import Worker -from netbox.plugins.utils import get_installed_plugins from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired +from netbox.plugins.utils import get_installed_plugins class APIRootView(APIView): @@ -66,7 +66,8 @@ class StatusView(APIView): return Response({ 'django-version': DJANGO_VERSION, 'installed-apps': installed_apps, - 'netbox-version': settings.RELEASE.full_version, + 'netbox-version': settings.RELEASE.version, + 'netbox-full-version': settings.RELEASE.full_version, 'plugins': get_installed_plugins(), 'python-version': platform.python_version(), 'rq-workers-running': Worker.count(get_connection('default')), From 08b2fc424a2d657aa75113805627a26cc09f390c Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Tue, 25 Feb 2025 06:13:30 -0800 Subject: [PATCH 07/25] 18296 Add Tenancy to VLAN Groups (#18690) * 18296 add tenant to vlan groups * 18296 add tenant to vlan groups * 18296 add tenant to vlan groups * 18296 add tenant to vlan groups * 18296 review changes --- netbox/ipam/api/serializers_/vlans.py | 3 +- netbox/ipam/filtersets.py | 2 +- netbox/ipam/forms/bulk_edit.py | 6 +++ netbox/ipam/forms/bulk_import.py | 9 ++++- netbox/ipam/forms/filtersets.py | 3 +- netbox/ipam/forms/model_forms.py | 5 ++- netbox/ipam/graphql/types.py | 1 + .../ipam/migrations/0077_vlangroup_tenant.py | 26 +++++++++++++ netbox/ipam/models/vlans.py | 7 ++++ netbox/ipam/tables/vlans.py | 8 ++-- netbox/ipam/tests/test_filtersets.py | 38 +++++++++++++++++-- netbox/templates/ipam/vlangroup.html | 9 +++++ 12 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 netbox/ipam/migrations/0077_vlangroup_tenant.py diff --git a/netbox/ipam/api/serializers_/vlans.py b/netbox/ipam/api/serializers_/vlans.py index 9b5501dc5..a6f428343 100644 --- a/netbox/ipam/api/serializers_/vlans.py +++ b/netbox/ipam/api/serializers_/vlans.py @@ -37,6 +37,7 @@ class VLANGroupSerializer(NetBoxModelSerializer): scope = serializers.SerializerMethodField(read_only=True) vid_ranges = IntegerRangeSerializer(many=True, required=False) utilization = serializers.CharField(read_only=True) + tenant = TenantSerializer(nested=True, required=False, allow_null=True) # Related object counts vlan_count = RelatedObjectCountField('vlans') @@ -45,7 +46,7 @@ class VLANGroupSerializer(NetBoxModelSerializer): model = VLANGroup fields = [ 'id', 'url', 'display_url', 'display', 'name', 'slug', 'scope_type', 'scope_id', 'scope', 'vid_ranges', - 'description', 'tags', 'custom_fields', 'created', 'last_updated', 'vlan_count', 'utilization' + 'tenant', 'description', 'tags', 'custom_fields', 'created', 'last_updated', 'vlan_count', 'utilization' ] brief_fields = ('id', 'url', 'display', 'name', 'slug', 'description', 'vlan_count') validators = [] diff --git a/netbox/ipam/filtersets.py b/netbox/ipam/filtersets.py index 81cbd2ef8..d9507ec2e 100644 --- a/netbox/ipam/filtersets.py +++ b/netbox/ipam/filtersets.py @@ -857,7 +857,7 @@ class FHRPGroupAssignmentFilterSet(ChangeLoggedModelFilterSet): ) -class VLANGroupFilterSet(OrganizationalModelFilterSet): +class VLANGroupFilterSet(OrganizationalModelFilterSet, TenancyFilterSet): scope_type = ContentTypeFilter() region = django_filters.NumberFilter( method='filter_scope' diff --git a/netbox/ipam/forms/bulk_edit.py b/netbox/ipam/forms/bulk_edit.py index 7f3216cfd..f1aa6d845 100644 --- a/netbox/ipam/forms/bulk_edit.py +++ b/netbox/ipam/forms/bulk_edit.py @@ -430,11 +430,17 @@ class VLANGroupBulkEditForm(NetBoxModelBulkEditForm): label=_('VLAN ID ranges'), required=False ) + tenant = DynamicModelChoiceField( + label=_('Tenant'), + queryset=Tenant.objects.all(), + required=False + ) model = VLANGroup fieldsets = ( FieldSet('site', 'vid_ranges', 'description'), FieldSet('scope_type', 'scope', name=_('Scope')), + FieldSet('tenant', name=_('Tenancy')), ) nullable_fields = ('description', 'scope') diff --git a/netbox/ipam/forms/bulk_import.py b/netbox/ipam/forms/bulk_import.py index c1f2dedd7..85583ca18 100644 --- a/netbox/ipam/forms/bulk_import.py +++ b/netbox/ipam/forms/bulk_import.py @@ -438,10 +438,17 @@ class VLANGroupImportForm(NetBoxModelImportForm): vid_ranges = NumericRangeArrayField( required=False ) + tenant = CSVModelChoiceField( + label=_('Tenant'), + queryset=Tenant.objects.all(), + required=False, + to_field_name='name', + help_text=_('Assigned tenant') + ) class Meta: model = VLANGroup - fields = ('name', 'slug', 'scope_type', 'scope_id', 'vid_ranges', 'description', 'tags') + fields = ('name', 'slug', 'scope_type', 'scope_id', 'vid_ranges', 'tenant', 'description', 'tags') labels = { 'scope_id': 'Scope ID', } diff --git a/netbox/ipam/forms/filtersets.py b/netbox/ipam/forms/filtersets.py index 3f951512b..f60003c56 100644 --- a/netbox/ipam/forms/filtersets.py +++ b/netbox/ipam/forms/filtersets.py @@ -411,12 +411,13 @@ class FHRPGroupFilterForm(NetBoxModelFilterSetForm): tag = TagFilterField(model) -class VLANGroupFilterForm(NetBoxModelFilterSetForm): +class VLANGroupFilterForm(TenancyFilterForm, NetBoxModelFilterSetForm): fieldsets = ( FieldSet('q', 'filter_id', 'tag'), FieldSet('region', 'sitegroup', 'site', 'location', 'rack', name=_('Location')), FieldSet('cluster_group', 'cluster', name=_('Cluster')), FieldSet('contains_vid', name=_('VLANs')), + FieldSet('tenant_group_id', 'tenant_id', name=_('Tenant')), ) model = VLANGroup region = DynamicModelMultipleChoiceField( diff --git a/netbox/ipam/forms/model_forms.py b/netbox/ipam/forms/model_forms.py index c381f99c9..22f98f6f0 100644 --- a/netbox/ipam/forms/model_forms.py +++ b/netbox/ipam/forms/model_forms.py @@ -598,7 +598,7 @@ class FHRPGroupAssignmentForm(forms.ModelForm): return group -class VLANGroupForm(NetBoxModelForm): +class VLANGroupForm(TenancyForm, NetBoxModelForm): slug = SlugField() vid_ranges = NumericRangeArrayField( label=_('VLAN IDs') @@ -621,12 +621,13 @@ class VLANGroupForm(NetBoxModelForm): FieldSet('name', 'slug', 'description', 'tags', name=_('VLAN Group')), FieldSet('vid_ranges', name=_('Child VLANs')), FieldSet('scope_type', 'scope', name=_('Scope')), + FieldSet('tenant_group', 'tenant', name=_('Tenancy')), ) class Meta: model = VLANGroup fields = [ - 'name', 'slug', 'description', 'vid_ranges', 'scope_type', 'tags', + 'name', 'slug', 'description', 'vid_ranges', 'scope_type', 'tenant_group', 'tenant', 'tags', ] def __init__(self, *args, **kwargs): diff --git a/netbox/ipam/graphql/types.py b/netbox/ipam/graphql/types.py index e6ecca984..b16cf29fe 100644 --- a/netbox/ipam/graphql/types.py +++ b/netbox/ipam/graphql/types.py @@ -266,6 +266,7 @@ class VLANGroupType(OrganizationalObjectType): vlans: List[VLANType] vid_ranges: List[str] + tenant: Annotated["TenantType", strawberry.lazy('tenancy.graphql.types')] | None @strawberry_django.field def scope(self) -> Annotated[Union[ diff --git a/netbox/ipam/migrations/0077_vlangroup_tenant.py b/netbox/ipam/migrations/0077_vlangroup_tenant.py new file mode 100644 index 000000000..9fb67cf53 --- /dev/null +++ b/netbox/ipam/migrations/0077_vlangroup_tenant.py @@ -0,0 +1,26 @@ +# Generated by Django 5.1.3 on 2025-02-20 17:49 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ipam', '0076_natural_ordering'), + ('tenancy', '0017_natural_ordering'), + ] + + operations = [ + migrations.AddField( + model_name='vlangroup', + name='tenant', + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name='vlan_groups', + to='tenancy.tenant', + ), + ), + ] diff --git a/netbox/ipam/models/vlans.py b/netbox/ipam/models/vlans.py index 91e39c6d3..b639fd185 100644 --- a/netbox/ipam/models/vlans.py +++ b/netbox/ipam/models/vlans.py @@ -62,6 +62,13 @@ class VLANGroup(OrganizationalModel): verbose_name=_('VLAN ID ranges'), default=default_vid_ranges ) + tenant = models.ForeignKey( + to='tenancy.Tenant', + on_delete=models.PROTECT, + related_name='vlan_groups', + blank=True, + null=True + ) _total_vlan_ids = models.PositiveBigIntegerField( default=VLAN_VID_MAX - VLAN_VID_MIN + 1 ) diff --git a/netbox/ipam/tables/vlans.py b/netbox/ipam/tables/vlans.py index aa1900e41..c22975be0 100644 --- a/netbox/ipam/tables/vlans.py +++ b/netbox/ipam/tables/vlans.py @@ -28,7 +28,7 @@ AVAILABLE_LABEL = mark_safe('AvailableUtilization {% utilization_graph object.utilization %} + + {% trans "Tenant" %} + + {% if object.tenant.group %} + {{ object.tenant.group|linkify }} / + {% endif %} + {{ object.tenant|linkify|placeholder }} + + {% include 'inc/panels/tags.html' %} From 7e669d1a14e3133d11ca826d3d3158a6c08065c3 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 20 Feb 2025 17:02:05 -0500 Subject: [PATCH 08/25] Closes #18072: Remove support for single model registration from PluginTemplateExtension --- netbox/netbox/plugins/registration.py | 12 ++---------- netbox/netbox/plugins/templates.py | 11 ++++++++--- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/netbox/netbox/plugins/registration.py b/netbox/netbox/plugins/registration.py index 515405f1b..0001d50c9 100644 --- a/netbox/netbox/plugins/registration.py +++ b/netbox/netbox/plugins/registration.py @@ -1,7 +1,7 @@ import inspect -import warnings from django.utils.translation import gettext_lazy as _ + from netbox.registry import registry from .navigation import PluginMenu, PluginMenuButton, PluginMenuItem from .templates import PluginTemplateExtension @@ -35,16 +35,8 @@ def register_template_extensions(class_list): ) if template_extension.models: - # Registration for multiple models + # Registration for specific models models = template_extension.models - elif template_extension.model: - # Registration for a single model (deprecated) - warnings.warn( - "PluginTemplateExtension.model is deprecated and will be removed in a future release. Use " - "'models' instead.", - DeprecationWarning - ) - models = [template_extension.model] else: # Global registration (no specific models) models = [None] diff --git a/netbox/netbox/plugins/templates.py b/netbox/netbox/plugins/templates.py index 4ea90b4db..58f9ad80e 100644 --- a/netbox/netbox/plugins/templates.py +++ b/netbox/netbox/plugins/templates.py @@ -11,8 +11,14 @@ class PluginTemplateExtension: This class is used to register plugin content to be injected into core NetBox templates. It contains methods that are overridden by plugin authors to return template content. - The `model` attribute on the class defines the which model detail page this class renders content for. It - should be set as a string in the form '.'. render() provides the following context data: + The `models` attribute on the class defines the which specific model detail pages this class renders content + for. It should be defined as a list of strings in the following form: + + models = ['.', '.'] + + If `models` is left as None, the extension will render for _all_ models. + + The `render()` method provides the following context data: * object - The object being viewed (object views only) * model - The type of object being viewed (list views only) @@ -21,7 +27,6 @@ class PluginTemplateExtension: * config - Plugin-specific configuration parameters """ models = None - model = None # Deprecated; use `models` instead def __init__(self, context): self.context = context From d1712c45bb83f1506d8eef5ec46bb54ebbefc320 Mon Sep 17 00:00:00 2001 From: Mathias Guillemot <84408567+Mathias-gt@users.noreply.github.com> Date: Tue, 25 Feb 2025 23:06:07 +0800 Subject: [PATCH 09/25] Closes: #18434 - Add SPB in L2VPN (#18523) * Add SPB in L2VPN * Change category as Other Co-authored-by: Daniel Sheppard --------- Co-authored-by: Daniel Sheppard --- netbox/vpn/choices.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/netbox/vpn/choices.py b/netbox/vpn/choices.py index 9847e1b97..7aea90232 100644 --- a/netbox/vpn/choices.py +++ b/netbox/vpn/choices.py @@ -228,6 +228,7 @@ class L2VPNTypeChoices(ChoiceSet): TYPE_MPLS_EVPN = 'mpls-evpn' TYPE_PBB_EVPN = 'pbb-evpn' TYPE_EVPN_VPWS = 'evpn-vpws' + TYPE_SPB = 'spb' CHOICES = ( ('VPLS', ( @@ -255,6 +256,9 @@ class L2VPNTypeChoices(ChoiceSet): (TYPE_EPTREE, _('Ethernet Private Tree')), (TYPE_EVPTREE, _('Ethernet Virtual Private Tree')), )), + ('Other', ( + (TYPE_SPB, _('SPB')), + )), ) P2P = ( From f7fdf079493d7b6a89caec0b7eb5b95f1c13bc28 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 25 Feb 2025 12:06:44 -0500 Subject: [PATCH 10/25] Closes #17793: Introduce a REST API endpoint for tagged objects (#18679) * Closes #17793: Introduce a REST API endpoint for tagged objects * Add missing object_id filter to TaggedItemFilterSet --- netbox/extras/api/serializers_/tags.py | 44 +++++++++++++++++++- netbox/extras/api/urls.py | 1 + netbox/extras/api/views.py | 9 ++++- netbox/extras/filtersets.py | 36 +++++++++++++++++ netbox/extras/models/tags.py | 2 + netbox/extras/tests/test_api.py | 28 +++++++++++++ netbox/extras/tests/test_filtersets.py | 56 ++++++++++++++++++++++++++ 7 files changed, 173 insertions(+), 3 deletions(-) diff --git a/netbox/extras/api/serializers_/tags.py b/netbox/extras/api/serializers_/tags.py index e4e62845a..ea964ff52 100644 --- a/netbox/extras/api/serializers_/tags.py +++ b/netbox/extras/api/serializers_/tags.py @@ -1,10 +1,16 @@ +from drf_spectacular.utils import extend_schema_field +from rest_framework import serializers + from core.models import ObjectType -from extras.models import Tag +from extras.models import Tag, TaggedItem +from netbox.api.exceptions import SerializerNotFound from netbox.api.fields import ContentTypeField, RelatedObjectCountField -from netbox.api.serializers import ValidatedModelSerializer +from netbox.api.serializers import BaseModelSerializer, ValidatedModelSerializer +from utilities.api import get_serializer_for_model __all__ = ( 'TagSerializer', + 'TaggedItemSerializer', ) @@ -25,3 +31,37 @@ class TagSerializer(ValidatedModelSerializer): 'tagged_items', 'created', 'last_updated', ] brief_fields = ('id', 'url', 'display', 'name', 'slug', 'color', 'description') + + +class TaggedItemSerializer(BaseModelSerializer): + object_type = ContentTypeField( + source='content_type', + read_only=True + ) + object = serializers.SerializerMethodField( + read_only=True + ) + tag = TagSerializer( + nested=True, + read_only=True + ) + + class Meta: + model = TaggedItem + fields = [ + 'id', 'url', 'display', 'object_type', 'object_id', 'object', 'tag', + ] + brief_fields = ('id', 'url', 'display', 'object_type', 'object_id', 'object', 'tag') + + @extend_schema_field(serializers.JSONField()) + def get_object(self, obj): + """ + Serialize a nested representation of the tagged object. + """ + try: + serializer = get_serializer_for_model(obj.content_object) + except SerializerNotFound: + return obj.object_repr + data = serializer(obj.content_object, nested=True, context={'request': self.context['request']}).data + + return data diff --git a/netbox/extras/api/urls.py b/netbox/extras/api/urls.py index bbcb8f0ef..88121b640 100644 --- a/netbox/extras/api/urls.py +++ b/netbox/extras/api/urls.py @@ -19,6 +19,7 @@ router.register('notifications', views.NotificationViewSet) router.register('notification-groups', views.NotificationGroupViewSet) router.register('subscriptions', views.SubscriptionViewSet) router.register('tags', views.TagViewSet) +router.register('tagged-objects', views.TaggedItemViewSet) router.register('image-attachments', views.ImageAttachmentViewSet) router.register('journal-entries', views.JournalEntryViewSet) router.register('config-contexts', views.ConfigContextViewSet) diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index e4c3c7f3e..49a44f5f1 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -6,6 +6,7 @@ from rest_framework import status from rest_framework.decorators import action from rest_framework.exceptions import PermissionDenied from rest_framework.generics import RetrieveUpdateDestroyAPIView +from rest_framework.mixins import ListModelMixin, RetrieveModelMixin from rest_framework.renderers import JSONRenderer from rest_framework.response import Response from rest_framework.routers import APIRootView @@ -20,7 +21,7 @@ from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired from netbox.api.features import SyncedDataMixin from netbox.api.metadata import ContentTypeMetadata from netbox.api.renderers import TextRenderer -from netbox.api.viewsets import NetBoxModelViewSet +from netbox.api.viewsets import BaseViewSet, NetBoxModelViewSet from utilities.exceptions import RQWorkerNotRunningException from utilities.request import copy_safe_request from . import serializers @@ -172,6 +173,12 @@ class TagViewSet(NetBoxModelViewSet): filterset_class = filtersets.TagFilterSet +class TaggedItemViewSet(RetrieveModelMixin, ListModelMixin, BaseViewSet): + queryset = TaggedItem.objects.prefetch_related('content_type', 'content_object', 'tag') + serializer_class = serializers.TaggedItemSerializer + filterset_class = filtersets.TaggedItemFilterSet + + # # Image attachments # diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index 4f40ce500..98302d0f4 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -31,6 +31,7 @@ __all__ = ( 'SavedFilterFilterSet', 'ScriptFilterSet', 'TagFilterSet', + 'TaggedItemFilterSet', 'WebhookFilterSet', ) @@ -492,6 +493,41 @@ class TagFilterSet(ChangeLoggedModelFilterSet): ) +class TaggedItemFilterSet(BaseFilterSet): + q = django_filters.CharFilter( + method='search', + label=_('Search'), + ) + object_type = ContentTypeFilter( + field_name='content_type' + ) + object_type_id = django_filters.ModelMultipleChoiceFilter( + queryset=ContentType.objects.all(), + field_name='content_type_id' + ) + tag_id = django_filters.ModelMultipleChoiceFilter( + queryset=Tag.objects.all() + ) + tag = django_filters.ModelMultipleChoiceFilter( + field_name='tag__slug', + queryset=Tag.objects.all(), + to_field_name='slug', + ) + + class Meta: + model = TaggedItem + fields = ('id', 'object_id') + + def search(self, queryset, name, value): + if not value.strip(): + return queryset + return queryset.filter( + Q(tag__name__icontains=value) | + Q(tag__slug__icontains=value) | + Q(tag__description__icontains=value) + ) + + class ConfigContextFilterSet(ChangeLoggedModelFilterSet): q = django_filters.CharFilter( method='search', diff --git a/netbox/extras/models/tags.py b/netbox/extras/models/tags.py index d1e329f03..baf72baa1 100644 --- a/netbox/extras/models/tags.py +++ b/netbox/extras/models/tags.py @@ -9,6 +9,7 @@ from netbox.choices import ColorChoices from netbox.models import ChangeLoggedModel from netbox.models.features import CloningMixin, ExportTemplatesMixin from utilities.fields import ColorField +from utilities.querysets import RestrictedQuerySet __all__ = ( 'Tag', @@ -72,6 +73,7 @@ class TaggedItem(GenericTaggedItemBase): ) _netbox_private = True + objects = RestrictedQuerySet.as_manager() class Meta: indexes = [models.Index(fields=["content_type", "object_id"])] diff --git a/netbox/extras/tests/test_api.py b/netbox/extras/tests/test_api.py index 63baf44d3..17f03350d 100644 --- a/netbox/extras/tests/test_api.py +++ b/netbox/extras/tests/test_api.py @@ -538,6 +538,34 @@ class TagTest(APIViewTestCases.APIViewTestCase): Tag.objects.bulk_create(tags) +class TaggedItemTest( + APIViewTestCases.GetObjectViewTestCase, + APIViewTestCases.ListObjectsViewTestCase +): + model = TaggedItem + brief_fields = ['display', 'id', 'object', 'object_id', 'object_type', 'tag', 'url'] + + @classmethod + def setUpTestData(cls): + + tags = ( + Tag(name='Tag 1', slug='tag-1'), + Tag(name='Tag 2', slug='tag-2'), + Tag(name='Tag 3', slug='tag-3'), + ) + Tag.objects.bulk_create(tags) + + sites = ( + Site(name='Site 1', slug='site-1'), + Site(name='Site 2', slug='site-2'), + Site(name='Site 3', slug='site-3'), + ) + Site.objects.bulk_create(sites) + sites[0].tags.set([tags[0], tags[1]]) + sites[1].tags.set([tags[1], tags[2]]) + sites[2].tags.set([tags[2], tags[0]]) + + # TODO: Standardize to APIViewTestCase (needs create & update tests) class ImageAttachmentTest( APIViewTestCases.GetObjectViewTestCase, diff --git a/netbox/extras/tests/test_filtersets.py b/netbox/extras/tests/test_filtersets.py index cf914e665..9684b3dbe 100644 --- a/netbox/extras/tests/test_filtersets.py +++ b/netbox/extras/tests/test_filtersets.py @@ -1250,6 +1250,62 @@ class TagTestCase(TestCase, ChangeLoggedFilterSetTests): ) +class TaggedItemFilterSetTestCase(TestCase): + queryset = TaggedItem.objects.all() + filterset = TaggedItemFilterSet + + @classmethod + def setUpTestData(cls): + tags = ( + Tag(name='Tag 1', slug='tag-1'), + Tag(name='Tag 2', slug='tag-2'), + Tag(name='Tag 3', slug='tag-3'), + ) + Tag.objects.bulk_create(tags) + + sites = ( + Site(name='Site 1', slug='site-1'), + Site(name='Site 2', slug='site-2'), + Site(name='Site 3', slug='site-3'), + ) + Site.objects.bulk_create(sites) + sites[0].tags.add(tags[0]) + sites[1].tags.add(tags[1]) + sites[2].tags.add(tags[2]) + + tenants = ( + Tenant(name='Tenant 1', slug='tenant-1'), + Tenant(name='Tenant 2', slug='tenant-2'), + Tenant(name='Tenant 3', slug='tenant-3'), + ) + Tenant.objects.bulk_create(tenants) + tenants[0].tags.add(tags[0]) + tenants[1].tags.add(tags[1]) + tenants[2].tags.add(tags[2]) + + def test_tag(self): + tags = Tag.objects.all()[:2] + params = {'tag': [tags[0].slug, tags[1].slug]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 4) + params = {'tag_id': [tags[0].pk, tags[1].pk]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 4) + + def test_object_type(self): + object_type = ObjectType.objects.get_for_model(Site) + params = {'object_type': 'dcim.site'} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3) + params = {'object_type_id': [object_type.pk]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3) + + def test_object_id(self): + site_ids = Site.objects.values_list('pk', flat=True) + params = { + 'object_type': 'dcim.site', + 'object_id': site_ids[:2], + } + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + class ChangeLoggedFilterSetTestCase(TestCase): """ Evaluate base ChangeLoggedFilterSet filters using the Site model. From 26c7c8f08deb398202b56163f6d7d2bcf5c5293e Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 25 Feb 2025 12:13:53 -0500 Subject: [PATCH 11/25] Closes #18623: Upgrade Tabler to v1.0 (#18624) * Upgrade Tabler to v1.0 * Fix navigation menu colors * Reduce table column heading font size --- netbox/project-static/dist/netbox.css | Bin 554815 -> 554086 bytes netbox/project-static/package.json | 5 +-- netbox/project-static/styles/_variables.scss | 4 --- .../styles/transitional/_navigation.scss | 34 +++++------------- .../styles/transitional/_tables.scss | 5 +++ netbox/project-static/yarn.lock | 18 +++++----- 6 files changed, 26 insertions(+), 40 deletions(-) diff --git a/netbox/project-static/dist/netbox.css b/netbox/project-static/dist/netbox.css index 2cb549a0d2013315c376bcc38be6f4856c7d6fec..5dbcf97cf07fb1a97d39e1f74221a09148b10a03 100644 GIT binary patch delta 15150 zcmai5d3;pW_5Zwi_bnmIB$>%fSV9)U5^%yyHg<@Br7XtCuhfbH2{1s+N)j-*5p308 ze^ncrlGC3k1Vyo;f{o=_WeK=jO@dYh5r{=7qCl|ztW*oXbMJfa&U-Th{*&BywtMb5 z=YH>*JM;FP%RF^Db7x$9eAe9B#kGyK8Q0g=)Mjr=H(W)4xE=k|95oGOHdf0 z`_^IL+$3jrbSyY0Gs@#j8D${sobOOO?hBsR$*Zb#az|dg?}t!HWAu}@4OFM& zSle2{l*=|UBOOf}n;nev{bQ>H_?@8mS|8mUds=J4meE5(B+8jkU0%PuqGr4=*QEPUwt1u1f&Ut|OcZm` z^_81PwjSLwk)!n-1@f<_2fO(nuL@D#~8stthB|Fq7M+t6R z9`h(KH;?_#rSUq}_Fn5_ySyEvw*6cb{Y81uJWRouzzf+z3iOl`sr66K%y$%xg|)*J zk7{^20rsB|U9Doz>hb)3mCrY^mxcSP%+KBZ&)`IAYs zq##`o24bFIq98oT!iI!FFuzqvfSH1j0B2W830k5<*qkK>aszPVQ*w$B(B8@xZcBpR zXCw(8eol&k>xu+7oKJSdLX9lMLf04}<$;zWL2D^;d>aoZXFCQ%>UEBIZO?Sa#t(%6 z^!69M@r6V0&aE$Rtf=(`nhGe%BNS+}2Z-l0pr>2*L&7vsfdv!99LwV$Cx{Czk0U3F z0n6h(6U7yl$CycCzU6VzB(cWw*f&WWV}DdYeL-|W*<|r%3**ztVu9r`e~LI-tGH2o zFhR(J&)ySV+N3hEUBHJe-QpleUM}n}L*jjl#eD|R3q9#XL4o}TMYnctspzDP3m=H? zZKcaZfvT+g0admw7h@21Ialbe_a8dMtf3YYH^+kA!=1f<){K|`B zDw7v{%y~Me={*+`GKp?jcG$7mxq@4@^nF5T)MF?IuBo$&c#)wWk)Uu1^I%QAX|>^ zkptI63(Og|CU{{cNpbSB(HCk+nx$h9%fA<0&TvO**(5bacz9tYaV3R&qgQpPX0TG6 z$)Tx@I?<&HL!mS15k(QSM9r+R3qOb{Nuh9=Rl3Tp@ov6TDABfrCnQtwTu7i5VXFo6+aU4+HOM3N?R#@PLWOcyXb=X_2QQ( z)42{A?G55l9QWNzQsBczaVw5j#*_Z=`YQ1dRq)>~4ul_8iGOx-6A+B z2IoTD^P&pfd&RL9NFKRQ^xL^I{ByFHW)V5CPrQqZBtZ8wjyNb;r_6?ogL0gH((LXO zM_WWI{Xop74BU`j?bE-B zDIv%&`AE#9ELv$=*TVY4A{L_2X^-wTRaW)3&VDARBWe^04I}*lmcJt^r+@mm_Oe1ZUfkCGw7U3#IawZ(+B`pd#{(&et zLnxuUNAa8`l0$=j!02FYs}PFnDK<}xeCo2%%c1#0(G#TtD4a6|9xbyrD5A4jxI+d< z-#U!NLuBGB!*#oFoJBVrWAReuEQ;=o#lyu}2=J*_0(cJ%0(Z|;(RZ^MF%I)CTf_{& zru*I=FP(v1_aRy)+#f@p0=zw327ohpPskaAtmFE;!w6FbuHXHM_yj{73$B0gsknvG z3|wD#9GjeLj*F)aT(9jFFB-V+fpryG7vD4bR=jp(ACjA-s7#7gRslM0%pK@s)BH&GBZk*S)+iU`WYqG75& zZ!}ExG}-3oC}+b|t2bPNW9o>=7E?FM3&z|=;Sg&~Zmxiv<`A0`*>>utd7<-tXd*66 z(bW+b(l1iXF+&~&ajA_67{m3ZQN(p#ygK3<=J}%Rh)cD99C6*r)=II0E~STgP8)3| zZx)OpkGSiJI2tJEw*7D=%%#?OXf2rQvA5AFHZP16Fu?VUTz=KJFJ!IRjB!piJ1)q1WamXK^haF3w`mOSQOjoJD(U<_6CzXVLcBL$$bvCd)uy z-9-qYugx(7O)2`iD1*CYFgh=bo-2Sa73cn#;Jg2_m~sX1-LQpl?0s7xVc6@{vIyB^ z!Cux=uaXJ&&a9_?MIk2ky0*Du2o+`6+t`nA?0qek7}z@q-j5?^DE5{)$zUxzfg~~P z1;k#r*6t)84!|Uh*Z_Ea8u4h=>Ev$)Dq9;`w{{_u45y5PS;VU~4k4C?_Ea_rHMGly z5woH7!lB_L6A`#-c@zYuL~~Oa2%Jl4W-1VYBd#YX-z*cO!woR&Wvi|h>FWrx4FeAxJ>2Dm2kPM<1$q-7iX#%XDS{|^oGJs zMR=dY@Wp=10p~Nw)uEZ1Nv3AZ+!$bIqVw?#dcj~O#$e+5%=-;v2EH^qI#cZ*L1$WY z(feWQm03l`#JX^ptE?ZU!eL&rezq1KjCd~XCMT{iTOGd7mz_PxIl+u zdcC1l7a}myHdu$^uZoD5L-CE%?NBV#;AP88KOrAQh2qIGbttAih_-hY;k~d+%b#O` z;*RS%6uTko2G$HWRfTE<5tQC?Lg{ouVtIL(-7u9jlR>-T;^`#veu3%6@k*fWFqMnv zCCUJoE=OV$N9%^+{B&})8wRTFvN&VH93cGYH@~z)%)Bn@ZVE?PV=`X4=HB8cOFyAg zTR(Ea;C5RhsUwdaR^4lSRD|z39rVKP7~IL$Fo(fvQXSr~q6uF6x(N%g8guZ~gn$)q zmJ{m;qP7Jsbo>fkVVf7m%m)q}x(&Q+NR4RMS<@TEVFs?09_A?nSO2W&;L38!kxci^ zxND%MY}iT(Jdj4pblDV}I|eHN*UpYw4GdeU29KDTf5cW#w3IW5)cw)Fhw*W+!y9D0 z7~pPkI#)4wPe=t(70+W!oI*;p)wOxXSrfTv8T}t3?8FS8itQW{H&9wpSmb+2M zIggxS&8`=2`Z@U)HOiWi{M=3CG6mapI8L~kT);8jGk9R(e8fuQo?*o;guSw5_?K`_fooc4U09Zd0HPaET+hqdgmNMGieBG8= zztQ?#re+q4iNE^k8|8aEXsIUhRql6bs4S#&=2=f&CTTHB3C%Pg0*hs&XR*d-Sr#0pbp}!3)6phNx9RF<4I^cgQR%1YtJ=LGBe#KizQpMEsyKVUTvX9;U|q zvRwLZyn{SVX~v~5VsR>5x{I7+Seyd0?qT?g zSe&Xok3TD-q6o&Rkh+fiAC3`>`@_n6$!;98OW$?BB;Vi|!8lDj`YVFJhoN8QAsAN< zWuw#ENi$IwSCucR9G@{po4b>2l~bVK*)#lqa_8-cM%18Vx%m{Sgd*=x0@vDk7eDYL?88ZlO}!CevUir}Cg~2JL&G`ViHSP#&W}j|+Gj|KQ1UM#gR@M{hq`Vu z8IXaE*Ev$Hk6Jz_chS*ca2`Ykj`WFJ21!os=n--zd`^8`%{fvm)cK`hmO0kuNO6?5 z5c-`U1$2%YXbDJLAmcny=_pN~!_@6ZSJ+xw9Rue|ghVJkPLko+n`8u_fC81LNF2of zn`A=YaiVJY*bv>@(rUZOSm^RgUO*9+hh=~kj7Lny zDy?jQR1ucb!89){ox`Abk!qN^r%atrqik4UiKP|*EqHlOhr#hQtiiHRh%MfVRx(@) zTH>K$(e976FC-mWCP_Xh&6Y-i8P^%q{Zz`B2&Mg{8lBPGUjjNxrzvJrdg-GFQ@mhX z>J-T`5#1eW&^}Wd1UtXNZ{B&BG>q}nDm zn*pljfwDnz3>0VLFDH673OGXfHbE}ZphY_2pzg)O=FbI{gt^$cK`O9YLn#G1Gvs*i zx8aZ3&}4roYn2k2wI_Zry{=n}si;|2Ygus1QrX=GxahhfIW@*$1#32}mxn`pmh6Oc zFG*=A;DU^OQlWP5AEaG|Tb*A-x|O&IBR6;>#+U_-ipbP+BW8dE1vU*uC-*&uWkw~L zf8~!#?-^z@&IyWTwZ-4z)cZDJ>d@jK`;t*q){bqK3XC+I$7hQw%Vp1#`18K^a%96J zDq{1f_DLy^@tGfit&(gQ>w8Kf)|@g&%Kj+%>^`7a5^VZ6mM8LD%*c&TOV67z%9SOn z5#|#9;*hPz1D(h=&1qOvrTW}4i>p=zEjg+S$i=1@t3Bx6iL;iyfO+=kVb!kPX~gl% z9enE2KS_~O@5`4Z=y_2ZWizK0cS!l>vbSP}o?nlLK>ZBa4YWL3{>uc1uYS%Wi*J%7pgEu%P{y1jUww#FsJ2 zJ>N+K0fnfcB5my}(ry;SIA|6eBs{HsPoYbR`^?Q%HbEw~B>_^Bw7$JktiCR_KDg8p zQI>~J;vkUrnl!?$+4m}zZ{KLTEhoz!#%b1u(Zk1RwBnmBx_NL9G3Rj5;;LDJ}L21SS??0npQfW3i zmG}+Dr0!d3pe?Xk&o|NpBPh-n1#3_cjtg}(p$wY~NKj$tMLhTsQl_v_Ac%~@S*C0< zagBwe{d)xBmJhJ~332Js4(XRLB5bE-ES7aV%KwklW2|DZ&(Y749)!qQZJUYm=k;<6 z^ED27ciDVpHGfIO1DVxNk0^(A0!YEbK$f$$1W8WO7p$zV4Bs~U6ogs~l2_GPX8HFj zvQz6(<$oCt_7p|}W?8K4YO9j_+u{oe3HWQox=Lfad>SVs!eRBtLD^VG#8FzBEX&a2 zlm|keDPoPgHabzBWhCp;I9rJD0K?*Y87sKVg)Ie(0u`~U0}^83o;vIeXyn^mc($;X z<5{))pH8Z<(5|j8M^w*dSA68RUUaO!S7V@ zt%)_VTOOZ5zm?tal`tIASIO}b{km0WY13E9(gb^hIbZu=r`(T(6PN#4u96dr0{CVH z{n;tL5m0-(&kr{)7Sgp#ugJ$G_BF||{W5z4N852gmXq+4<@&$Msc`OnbhP=9oT(EA zz>dFRv$FGT++i&r$#wM6Z((_0=V5t(p{#v&SZM=Rpn@8V7sHk4a!n{wyF>6%CEPikhm5nxOfEe82YPAZ14!TpFRIYF#->S}9DbQez=)E`I*)y-~&9 z<_;we`jQkGwx=jVc$Yt4d*cqpC59ydE;cKL+8^#wJ{H0_8gg8YFd`K%(%${0ax`3A z=TzLU9E-|KQFeR6CetoFu8bEv*W|9K_caEa8qNQ{hQ{)G7_wCv zcO_ou6G{LUwc$Go_ia@Mr9@f?)+~ibwko3sMrDmJU)C6`hf!OVbU3|Lab3Z{{wI_) zn4B*-=~S^=_LIs@5*!_>^domKsa*=0bClH9$A-pe?awGx0wg}GoJlSpzoK?^uzviC z@`e$)6-#Psav<}F?CsaEYVnF-`O;who$yGCf}iX0qnU;u%_b?ZWQtg+ZF*5D6X4hs zu^L)-DKXlJUCL|$ZY~xp;j)H-eMD0x3ea7Q@4#GoT#VWE7s|fwTGXt1St$`<&$Z%h z+L2e3p9+vaRjhy=dr)cC9;&o$s<=YiwwJO7P7^D%#7B#!@0JXJdI@+538Xfl0;mfa6h2V8)ChbjX@#E{n<=P!@QGwY%L4hycro#ht_=~?% z*)cOv_LX<(l-p;ZH^1#dZ$@@e-SnR#>w#`6^*cH|)1&wWaLg2gu>F0dpEmV<1>X%@ zJyWdGIzCW-hAzw$D`De@$jbYWvTmD&tS3IAGoF}*GbSIRQn$=TslLOwoc}pYm-GH? zoOaO>)cW=lsx@m4YAyeiYP~u~ybWd_Q)0Exjw#|3o$50p?auCSKt_UAcAe^1<0ssK=({x7G(O<>`=RA}^+mWdSsew_OVma>Z|QBOLBYhZnrRDicNq#q)BrLVD~u__2- z7pv2>?-#0b1YMw%-uB@|O1Zi=Mn{W`a;B*>92w;dGT>yHT4vSIHZ4{M5hz}!5<< N+#>D03bj7|{{bFzB~$}*IK?qAabZ6~^Ez#$Kh=4l!MiJ;FO^7DxP6(0=B4M60 z>MRmy&6#mw8<$ZL2oXv~hXhAO1q8-j0YOl4VTt=F%R5zf>Dzrf?|XkFw{F!rr_NTt zd#dj3&ras{p3dDeB))Qnzo{XRpED!S>}7qR^K&kl6G6`O*1g|E6612M_38WSmitxhO2c$H}*(ch8kJBD+ZZKPaIWMr46ezUE4*Cxr~m9i>9MGqgA2VQ!{(& z<}ihST@zNgu~t)h6>m^Q4^bcGvhb%kTlg%wWe8qO;{8)}%+ z58M`3xTByM)<1q0UEc z4y{--mXgYvWSFo>N(-&rn;&X=bV5qZ=|cNp6t^to`79;$?V}@bj%9293yw@yM&Yz| zC~sUyUY{JwTRWBwxu|qoNolD25ht9zC`mnkTU$mo%tx|IDS6M5_2W3FZd_r3rY$vQ z+lC4OrSw0MDyc?5?i}6;C(lapp;`Mpq4`f{z{vt}n0;c18 z_)n+*f9AaQ%Y?0;JNRgw(|fM{vOt}(XZcsoB#5uzhVa#`O^{t_8y>p-LSpF6^Ua*6 zr|QCIBZkp*WBC{9@{&?gnakbgfWS^J8J;iXobW{+k1liYPClo=3%xgTH^I?s!lkx!xa$hr5RzGC>w1TrOXWYw0sjWs z4mK}8No`eo`DUXp8RE)$kLk9hoL^|VbyV<0rrV_ze3N>+`V-j+D<|?2xqm1|XR=u_QIXR&AP1&yZog#YQ-*%|H65R5F-Wv2zoo>@k`F53eexE)% z^d+^WDYW<%T~zLFoo&x+*qZ-_ew(#dUaU&_c%N#A0*~+4Et+>wmz?pI-goeAUFfQJ zb>128>vYZ!b-x_@NN@e|uw1Xsk^ix-^O2)E$K#*qbTucmKC!yNS93cYd{!=loqyuf zpz?V+gL<|V+}*NFc2D9%31zhC%B^n<23lHuO|8ocXj!A-B+c2#A+J8E_`KC&7wba)wY^{KUzv9!ua}A$t*M(>!UM7O`l7bohK`N1ZjnBxjNTl_i zCb2aTtN?B%p9!;f^T`Q@JlgMN>?kdFCZEe$#k59=S$~s70=aw^pTt>h@`qV0eauU?-@>n>ky!B)pKdWENMoxbndXS*ANdUM z%;Bjukwac41mm&fR(=hO?tl19i)}#~U7eAI7bT{_x)Fj5-O~kXrFj&Cj8Zr+c<1pa zt-@%G&E&BDAaj(t&kG~x^FLf^@evKOPNy1Yag{&pD^FmKV!+cAdR^$K@IwfYL;^monEM_LBP$e;UB}O(Y92!w9bdCWK|9Cl)bEt zk+@DsgO6+Y_i1Rc9apxrC9uTB2)oA$LkD|}dzAPdKOEk!o)2SE=KbFMt-DeH0fFX zPW5){v;1_^K=CGgda-9J4h(~Jn-GHZM{F|rZWI5;5cJmi7kDSSRtHh_USFmaUb5>2 z{&gKWX(V3dQxbLZ2((=PBA+@CS~51$`%nID9Wqgrr zss9)6pv>#`@F}6izfOdfQ+!gAnO*}xN&u^0=5wKCFTaO!>}B!LR!jw~-OHz7s_XY& zQn`=cMnzM&!wcS0_4E~HhPYk(_|$Iqpo&cs?tdc~B?4pFpKp}U`PdKo9V>hlN~$mzE%W20Jf zm}Qv@CZJs8`Y%)@Sn-WTgy%~O5&$J5E5GDZXed{G8^egIZ)3p$B_%I@rNTkg*8{?V zSy?@GfYOt1zE;D;eb1*P=+0L6va~LuOF(m&mO7LW(oZ8S{PI1&XR2i_s0UwK9>%DT zm@4|P7Y_c2N*wtiTqmIeXx{+9L2DzYS%gtJFwpTr?tf6;b3cV^B$P+{&^CQ-#6pY+ zky<#_(&*UFXzw3>=1-y^+Kv`PYoeqW?0vsbzF+tskdL;l2KD|D69f;OL&m?HV|YN@ z1K~l$oJLTOok#9x&+})fy4a452@$+zOxSuMj0s4jVS?u(pB7pD{O)2H5Ez}l`hgZe z2~57;g+bwj%x$X$_S~WE7+9CpckT{N(5aYg7*@)ugoK7aP@67HfpJ5(J}0 z7-bRFW|kFEGBhU&1C|WrMZ53?dp_X?3*Fr75C*Cik{#>_R9it2FhRM@>jw65UyX z6(m-s38ip%wm{2<{%nDkar%;n?rb5VY=CvSI7MBKU~H9;iS}7n4BH;f&JhN#9l$*j zg-sqI7<(iXM*E_dZREZviP>%j2%}=SFj9!IBWYDL$IQWYN3jG2M+qsAJW8NQK-*Tk z0;6J^G#VLmM+>wvSb^<NJ#r*BPhgb+UI##d2tEx3d}D-Em@-D#j&stsc>@&3 z7))hB!dSs53veuQ8n00njB#O18*-<_Qhx;!lYLW!9Ljli z>HtffE+5S!Mz$dFXv)-mm34{J^N}R8np&<)obAlN>_<*` zbcR5S0NRe0vZ76JVxk$=xYQiTnMv8MWh6$4dae2`XtYEeJ zzDtK4EqdVjjo@^er`N*@%}a%3*f~q+N5yHItp=T)g%RjMNU}K;YW1g`g}gFbFg6J+ zoa!mZY{4i92<{3P(BO4=^ifCx6CADtd^ZT0K`nq@LL`dC!hwvB`$4*6YH;G-*D?OraaH^=NOVog0 z-8b0VP}9RpgwrUGwxb0hWTaWn7v5~cWseMajbX?>WU-!_kmK#bFm8;+w}=%O&9FE0 zI3rl3#X2}ecu0kmrNZ|(KW)d{IXKlQ>lSNaX2%IRcM5&Ts<#Eba_F9o4{DZIM&&RR zez{Zl7KsruhSC)qSvd@aqss)rTt1NgWkLzdzGXsI49G}P)iPrSJx*A@9G?MI%Z2Ml zxu|<4Y9GR|4`y0iW$Je>QN3P#Yfj)Xp+g`gPYL%+;2WYOldm=lgJi*7I(8z=zY$*_ zt}RsGCMHAPcZjeTM@jK;_FgW5<1F1U#OZQ(B2Dn5gJ z{E@IXYF>i2?}+N7{umB3&j@a+2EK|tBTS$gOrfpuQ4XM!OUA%r<2Hlpl25X~5w;~+ zb#anRQG9H0i~+!bw6MS2nPO@9j(U%3;)VU0W~%vB;ysY#5DN%K=SU25oJ?%kzoar# zoIBw3;65yrM!TVD~EuoZT%bFmNa6$Hd$2qZ- zto@z%zKsbfxKVMaLh|6^Jvgolv4b|mB+b2t8`PD8i;4+4+sDO$J4H%-aF$d`9=T6E zt_tfN%_rFK1wX2E6;;FzdJhw_{}WDzg!@HJO7EXV8Ft<;+L@Gx{w%(tO4)o;j$_{m zFv~iJNluee#vn?}gO#graUNYQ4uW8}=!6R=@!k9Rl{i80C$Wq)t`eV9MRq=muh_R7 zvx#B3^B)jLs5(XtQcX$U1L8F1hhcKSpzlc{d@^H^t^m{LeNpRqZsz6fL7ii!* zK)Z4N5|aH_u}?E;i##|yb>G8cirFN6kU44dZ-oSyv_`xHj(#Z%WYZ(!Rcg4idNjA7 zDr4aOwK#D&R^dUa)t0rW!n#W&=9VO|ZxEf#$F}w2CRL%nuQY`mqopfUHyB+^jf;$Pm&xoeCdD`sz>@>)zjeA%jPv!{Ky18QNwJUWvFcs%)cfg3OWH3dSD}`K<6i- z8|EL?7Nqx>_^vAIPZ!MzI&Y?wt6F08vp6o#u!&^L2@${2pUrQ@&Etrq zc`)IO>CtHieURjy72DM*H+%Sa!$qA7LLB<8$x)}@_n}$X&K`XB2G5yt(gAONs2Rk}u5@7#JLc*S zYsx^7!Kq4n^k-QKtF~o6PRd+o^fMWUI0;Juw)ih%^3ccGw%tIZ%m?of!5XUq?4pzJMv|{)dF0S3}Ng*c;e`-p$#K zKOva+niQnBsnGeVMBno2<8Dm?R|byV(t~)}wL>a^^?RgY*zWI>+_as>qyCg&G?S!K zje}V)OJaB!iO0g(UP)59@z{q7^fG~Xp?czlio^?@A1`5%j7}999yUZ`7v(4a*dyH` zl7|mU)f~xtOS)SME#D}SeIH2EhpXQ!!{y&e$>uMHN!xeQQaM%s>{UI_(7%F(**{5M zQgu$6oHW)|-|BDF&pj$U7`q(zvVw;CMfhcdKj_2XkN9gD4NUl}lro3h1JS3c@fLS|;G&JBt(bVZ)ESHetp!^;e*;z{7Xps+_ItfxH+oFh|@(!$Pr8fCP z6D1%eZcBb6pNQ^+bxUQDT)9jpsbN9no(JR#4nBTR_CNv1_vty{i4-1VhR$$;J-bm2`sW>TguzfG=;mhLVsT~O;GYqrb3 ziXPI`o%y^~r{coWB7ZfJ|6v{S`HTED<)rrov>)!tm*eG;XhBhF4XOX9JS}<-)i^Sw z*D6SJ$;*8)hDyp8xZUK=m#jW6E3Wo>O3CP#t@dbW3uu;^hZR*Jw>Uo(H}gK zu1ckbqdzDD?2BK$S0LU?m3#%7uU4LeS<{tPQhbeaJQ3FAap_Q5!HL{h*fB?$N* Date: Tue, 25 Feb 2025 18:36:16 +0100 Subject: [PATCH 12/25] Optimize contact lookup query --- netbox/netbox/models/features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index 60084c361..e58037b85 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -376,7 +376,7 @@ class ContactsMixin(models.Model): filter = Q( object_type=ObjectType.objects.get_for_model(self), object_id__in=( - self.get_ancestors(include_self=True).values_list('pk', flat=True) + self.get_ancestors(include_self=True) if (isinstance(self, NestedGroupModel) and inherited) else [self.pk] ), From b9b42cd3b44a6857b2c86f8f566100288abe2573 Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Wed, 26 Feb 2025 11:28:02 -0600 Subject: [PATCH 13/25] Fixes: #15924 - Prevent API payload from allowing tagged_vlans while interface mode is set to tagged-all (#17211) --- .../api/serializers_/device_components.py | 51 +++++- netbox/dcim/forms/common.py | 18 +- netbox/dcim/models/device_components.py | 2 + netbox/dcim/tests/test_api.py | 70 ++++++++ netbox/dcim/tests/test_forms.py | 166 +++++++++++++++++- 5 files changed, 292 insertions(+), 15 deletions(-) diff --git a/netbox/dcim/api/serializers_/device_components.py b/netbox/dcim/api/serializers_/device_components.py index a6767bb6f..b591030aa 100644 --- a/netbox/dcim/api/serializers_/device_components.py +++ b/netbox/dcim/api/serializers_/device_components.py @@ -1,3 +1,4 @@ +from django.utils.translation import gettext as _ from django.contrib.contenttypes.models import ContentType from drf_spectacular.utils import extend_schema_field from rest_framework import serializers @@ -232,8 +233,56 @@ class InterfaceSerializer(NetBoxModelSerializer, CabledObjectSerializer, Connect def validate(self, data): - # Validate many-to-many VLAN assignments if not self.nested: + + # Validate 802.1q mode and vlan(s) + mode = None + tagged_vlans = [] + + # Gather Information + if self.instance: + mode = data.get('mode') if 'mode' in data.keys() else self.instance.mode + untagged_vlan = data.get('untagged_vlan') if 'untagged_vlan' in data.keys() else \ + self.instance.untagged_vlan + qinq_svlan = data.get('qinq_svlan') if 'qinq_svlan' in data.keys() else \ + self.instance.qinq_svlan + tagged_vlans = data.get('tagged_vlans') if 'tagged_vlans' in data.keys() else \ + self.instance.tagged_vlans.all() + else: + mode = data.get('mode', None) + untagged_vlan = data.get('untagged_vlan') if 'untagged_vlan' in data.keys() else None + qinq_svlan = data.get('qinq_svlan') if 'qinq_svlan' in data.keys() else None + tagged_vlans = data.get('tagged_vlans') if 'tagged_vlans' in data.keys() else None + + errors = {} + + # Non Q-in-Q mode with service vlan set + if mode != InterfaceModeChoices.MODE_Q_IN_Q and qinq_svlan: + errors.update({ + 'qinq_svlan': _("Interface mode does not support q-in-q service vlan") + }) + # Routed mode + if not mode: + # Untagged vlan + if untagged_vlan: + errors.update({ + 'untagged_vlan': _("Interface mode does not support untagged vlan") + }) + # Tagged vlan + if tagged_vlans: + errors.update({ + 'tagged_vlans': _("Interface mode does not support tagged vlans") + }) + # Non-tagged mode + elif mode in (InterfaceModeChoices.MODE_TAGGED_ALL, InterfaceModeChoices.MODE_ACCESS) and tagged_vlans: + errors.update({ + 'tagged_vlans': _("Interface mode does not support tagged vlans") + }) + + if errors: + raise serializers.ValidationError(errors) + + # Validate many-to-many VLAN assignments device = self.instance.device if self.instance else data.get('device') for vlan in data.get('tagged_vlans', []): if vlan.site not in [device.site, None]: diff --git a/netbox/dcim/forms/common.py b/netbox/dcim/forms/common.py index 8ca258f34..23109f66b 100644 --- a/netbox/dcim/forms/common.py +++ b/netbox/dcim/forms/common.py @@ -43,20 +43,14 @@ class InterfaceCommonForm(forms.Form): super().clean() parent_field = 'device' if 'device' in self.cleaned_data else 'virtual_machine' - tagged_vlans = self.cleaned_data.get('tagged_vlans') - - # Untagged interfaces cannot be assigned tagged VLANs - if self.cleaned_data['mode'] == InterfaceModeChoices.MODE_ACCESS and tagged_vlans: - raise forms.ValidationError({ - 'mode': _("An access interface cannot have tagged VLANs assigned.") - }) - - # Remove all tagged VLAN assignments from "tagged all" interfaces - elif self.cleaned_data['mode'] == InterfaceModeChoices.MODE_TAGGED_ALL: - self.cleaned_data['tagged_vlans'] = [] + if 'tagged_vlans' in self.fields.keys(): + tagged_vlans = self.cleaned_data.get('tagged_vlans') if self.is_bound else \ + self.get_initial_for_field(self.fields['tagged_vlans'], 'tagged_vlans') + else: + tagged_vlans = [] # Validate tagged VLANs; must be a global VLAN or in the same site - elif self.cleaned_data['mode'] == InterfaceModeChoices.MODE_TAGGED and tagged_vlans: + if self.cleaned_data['mode'] == InterfaceModeChoices.MODE_TAGGED and tagged_vlans: valid_sites = [None, self.cleaned_data[parent_field].site] invalid_vlans = [str(v) for v in tagged_vlans if v.site not in valid_sites] diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index ce9e5607f..8a8e8f4cc 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -934,6 +934,8 @@ class Interface(ModularComponentModel, BaseInterface, CabledObjectModel, PathEnd raise ValidationError({'rf_channel_width': _("Cannot specify custom width with channel selected.")}) # VLAN validation + if not self.mode and self.untagged_vlan: + raise ValidationError({'untagged_vlan': _("Interface mode does not support an untagged vlan.")}) # Validate untagged VLAN if self.untagged_vlan and self.untagged_vlan.site not in [self.device.site, None]: diff --git a/netbox/dcim/tests/test_api.py b/netbox/dcim/tests/test_api.py index 99a446aef..08f93f6ea 100644 --- a/netbox/dcim/tests/test_api.py +++ b/netbox/dcim/tests/test_api.py @@ -1,3 +1,5 @@ +import json + from django.test import override_settings from django.urls import reverse from django.utils.translation import gettext as _ @@ -1748,6 +1750,23 @@ class InterfaceTest(Mixins.ComponentTraceMixin, APIViewTestCases.APIViewTestCase }, ] + def _perform_interface_test_with_invalid_data(self, mode: str = None, invalid_data: dict = {}): + device = Device.objects.first() + data = { + 'device': device.pk, + 'name': 'Interface 1', + 'type': InterfaceTypeChoices.TYPE_1GE_FIXED, + } + data.update({'mode': mode}) + data.update(invalid_data) + + response = self.client.post(self._get_list_url(), data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_400_BAD_REQUEST) + content = json.loads(response.content) + for key in invalid_data.keys(): + self.assertIn(key, content) + self.assertIsNone(content.get('data')) + def test_bulk_delete_child_interfaces(self): interface1 = Interface.objects.get(name='Interface 1') device = interface1.device @@ -1775,6 +1794,57 @@ class InterfaceTest(Mixins.ComponentTraceMixin, APIViewTestCases.APIViewTestCase self.client.delete(self._get_list_url(), data, format='json', **self.header) self.assertEqual(device.interfaces.count(), 2) # Child & parent were both deleted + def test_create_child_interfaces_mode_invalid_data(self): + """ + POST data to test interface mode check and invalid tagged/untagged VLANS. + """ + self.add_permissions('dcim.add_interface') + + vlans = VLAN.objects.all()[0:3] + + # Routed mode, untagged, tagged and qinq service vlan + invalid_data = { + 'untagged_vlan': vlans[0].pk, + 'tagged_vlans': [vlans[1].pk, vlans[2].pk], + 'qinq_svlan': vlans[2].pk + } + self._perform_interface_test_with_invalid_data(None, invalid_data) + + # Routed mode, untagged and tagged vlan + invalid_data = { + 'untagged_vlan': vlans[0].pk, + 'tagged_vlans': [vlans[1].pk, vlans[2].pk], + } + self._perform_interface_test_with_invalid_data(None, invalid_data) + + # Routed mode, untagged vlan + invalid_data = { + 'untagged_vlan': vlans[0].pk, + } + self._perform_interface_test_with_invalid_data(None, invalid_data) + + invalid_data = { + 'tagged_vlans': [vlans[1].pk, vlans[2].pk], + } + # Routed mode, qinq service vlan + self._perform_interface_test_with_invalid_data(None, invalid_data) + # Access mode, tagged vlans + self._perform_interface_test_with_invalid_data(InterfaceModeChoices.MODE_ACCESS, invalid_data) + # All tagged mode, tagged vlans + self._perform_interface_test_with_invalid_data(InterfaceModeChoices.MODE_TAGGED_ALL, invalid_data) + + invalid_data = { + 'qinq_svlan': vlans[0].pk, + } + # Routed mode, qinq service vlan + self._perform_interface_test_with_invalid_data(None, invalid_data) + # Access mode, qinq service vlan + self._perform_interface_test_with_invalid_data(InterfaceModeChoices.MODE_ACCESS, invalid_data) + # Tagged mode, qinq service vlan + self._perform_interface_test_with_invalid_data(InterfaceModeChoices.MODE_TAGGED, invalid_data) + # Tagged-all mode, qinq service vlan + self._perform_interface_test_with_invalid_data(InterfaceModeChoices.MODE_TAGGED_ALL, invalid_data) + class FrontPortTest(APIViewTestCases.APIViewTestCase): model = FrontPort diff --git a/netbox/dcim/tests/test_forms.py b/netbox/dcim/tests/test_forms.py index 7a57bf3f0..89b7508f3 100644 --- a/netbox/dcim/tests/test_forms.py +++ b/netbox/dcim/tests/test_forms.py @@ -1,8 +1,9 @@ from django.test import TestCase -from dcim.choices import DeviceFaceChoices, DeviceStatusChoices, InterfaceTypeChoices +from dcim.choices import DeviceFaceChoices, DeviceStatusChoices, InterfaceTypeChoices, InterfaceModeChoices from dcim.forms import * from dcim.models import * +from ipam.models import VLAN from utilities.testing import create_test_device from virtualization.models import Cluster, ClusterGroup, ClusterType @@ -117,11 +118,23 @@ class DeviceTestCase(TestCase): self.assertIn('position', form.errors) -class LabelTestCase(TestCase): +class InterfaceTestCase(TestCase): @classmethod def setUpTestData(cls): cls.device = create_test_device('Device 1') + cls.vlans = ( + VLAN(name='VLAN 1', vid=1), + VLAN(name='VLAN 2', vid=2), + VLAN(name='VLAN 3', vid=3), + ) + VLAN.objects.bulk_create(cls.vlans) + cls.interface = Interface.objects.create( + device=cls.device, + name='Interface 1', + type=InterfaceTypeChoices.TYPE_1GE_GBIC, + mode=InterfaceModeChoices.MODE_TAGGED, + ) def test_interface_label_count_valid(self): """ @@ -151,3 +164,152 @@ class LabelTestCase(TestCase): self.assertFalse(form.is_valid()) self.assertIn('label', form.errors) + + def test_create_interface_mode_valid_data(self): + """ + Test that saving valid interface mode and tagged/untagged vlans works properly + """ + + # Validate access mode + data = { + 'device': self.device.pk, + 'name': 'ethernet1/1', + 'type': InterfaceTypeChoices.TYPE_1GE_GBIC, + 'mode': InterfaceModeChoices.MODE_ACCESS, + 'untagged_vlan': self.vlans[0].pk + } + form = InterfaceCreateForm(data) + + self.assertTrue(form.is_valid()) + + # Validate tagged vlans + data = { + 'device': self.device.pk, + 'name': 'ethernet1/2', + 'type': InterfaceTypeChoices.TYPE_1GE_GBIC, + 'mode': InterfaceModeChoices.MODE_TAGGED, + 'untagged_vlan': self.vlans[0].pk, + 'tagged_vlans': [self.vlans[1].pk, self.vlans[2].pk] + } + form = InterfaceCreateForm(data) + self.assertTrue(form.is_valid()) + + # Validate tagged vlans + data = { + 'device': self.device.pk, + 'name': 'ethernet1/3', + 'type': InterfaceTypeChoices.TYPE_1GE_GBIC, + 'mode': InterfaceModeChoices.MODE_TAGGED_ALL, + 'untagged_vlan': self.vlans[0].pk, + } + form = InterfaceCreateForm(data) + self.assertTrue(form.is_valid()) + + def test_create_interface_mode_access_invalid_data(self): + """ + Test that saving invalid interface mode and tagged/untagged vlans works properly + """ + data = { + 'device': self.device.pk, + 'name': 'ethernet1/4', + 'type': InterfaceTypeChoices.TYPE_1GE_GBIC, + 'mode': InterfaceModeChoices.MODE_ACCESS, + 'untagged_vlan': self.vlans[0].pk, + 'tagged_vlans': [self.vlans[1].pk, self.vlans[2].pk] + } + form = InterfaceCreateForm(data) + + self.assertTrue(form.is_valid()) + self.assertIn('untagged_vlan', form.cleaned_data.keys()) + self.assertNotIn('tagged_vlans', form.cleaned_data.keys()) + self.assertNotIn('qinq_svlan', form.cleaned_data.keys()) + + def test_edit_interface_mode_access_invalid_data(self): + """ + Test that saving invalid interface mode and tagged/untagged vlans works properly + """ + data = { + 'device': self.device.pk, + 'name': 'Ethernet 1/5', + 'type': InterfaceTypeChoices.TYPE_1GE_GBIC, + 'mode': InterfaceModeChoices.MODE_ACCESS, + 'tagged_vlans': [self.vlans[0].pk, self.vlans[1].pk, self.vlans[2].pk] + } + form = InterfaceForm(data, instance=self.interface) + + self.assertTrue(form.is_valid()) + self.assertIn('untagged_vlan', form.cleaned_data.keys()) + self.assertNotIn('tagged_vlans', form.cleaned_data.keys()) + self.assertNotIn('qinq_svlan', form.cleaned_data.keys()) + + def test_create_interface_mode_tagged_all_invalid_data(self): + """ + Test that saving invalid interface mode and tagged/untagged vlans works properly + """ + data = { + 'device': self.device.pk, + 'name': 'ethernet1/6', + 'type': InterfaceTypeChoices.TYPE_1GE_GBIC, + 'mode': InterfaceModeChoices.MODE_TAGGED_ALL, + 'tagged_vlans': [self.vlans[0].pk, self.vlans[1].pk, self.vlans[2].pk] + } + form = InterfaceCreateForm(data) + + self.assertTrue(form.is_valid()) + self.assertIn('untagged_vlan', form.cleaned_data.keys()) + self.assertNotIn('tagged_vlans', form.cleaned_data.keys()) + self.assertNotIn('qinq_svlan', form.cleaned_data.keys()) + + def test_edit_interface_mode_tagged_all_invalid_data(self): + """ + Test that saving invalid interface mode and tagged/untagged vlans works properly + """ + data = { + 'device': self.device.pk, + 'name': 'Ethernet 1/7', + 'type': InterfaceTypeChoices.TYPE_1GE_GBIC, + 'mode': InterfaceModeChoices.MODE_TAGGED_ALL, + 'tagged_vlans': [self.vlans[0].pk, self.vlans[1].pk, self.vlans[2].pk] + } + form = InterfaceForm(data) + self.assertTrue(form.is_valid()) + self.assertIn('untagged_vlan', form.cleaned_data.keys()) + self.assertNotIn('tagged_vlans', form.cleaned_data.keys()) + self.assertNotIn('qinq_svlan', form.cleaned_data.keys()) + + def test_create_interface_mode_routed_invalid_data(self): + """ + Test that saving invalid interface mode (routed) and tagged/untagged vlans works properly + """ + data = { + 'device': self.device.pk, + 'name': 'ethernet1/6', + 'type': InterfaceTypeChoices.TYPE_1GE_GBIC, + 'mode': None, + 'untagged_vlan': self.vlans[0].pk, + 'tagged_vlans': [self.vlans[0].pk, self.vlans[1].pk, self.vlans[2].pk] + } + form = InterfaceCreateForm(data) + + self.assertTrue(form.is_valid()) + self.assertNotIn('untagged_vlan', form.cleaned_data.keys()) + self.assertNotIn('tagged_vlans', form.cleaned_data.keys()) + self.assertNotIn('qinq_svlan', form.cleaned_data.keys()) + + def test_edit_interface_mode_routed_invalid_data(self): + """ + Test that saving invalid interface mode (routed) and tagged/untagged vlans works properly + """ + data = { + 'device': self.device.pk, + 'name': 'Ethernet 1/7', + 'type': InterfaceTypeChoices.TYPE_1GE_GBIC, + 'mode': None, + 'untagged_vlan': self.vlans[0].pk, + 'tagged_vlans': [self.vlans[0].pk, self.vlans[1].pk, self.vlans[2].pk] + } + form = InterfaceForm(data) + self.assertTrue(form.is_valid()) + self.assertNotIn('untagged_vlan', form.cleaned_data.keys()) + self.assertNotIn('tagged_vlans', form.cleaned_data.keys()) + self.assertNotIn('qinq_svlan', form.cleaned_data.keys()) From dbac09349bc752a214ab92e3500a0a32fff68a8d Mon Sep 17 00:00:00 2001 From: Jason Novinger Date: Wed, 26 Feb 2025 15:58:46 -0600 Subject: [PATCH 14/25] For #18352, adds choices, model field, migration Adds: - dcim.choices.PowerOutletStatusChoices - dcim.models.device_components.PowerOutlet.status field with `choices` set to PowerOutletStatusChoices - adds migration for PowerOutlet.status field - updates breaking view tests --- netbox/dcim/choices.py | 17 +++++++++++++++++ .../migrations/0201_add_power_outlet_status.py | 16 ++++++++++++++++ netbox/dcim/models/device_components.py | 6 ++++++ netbox/dcim/tests/test_views.py | 3 +++ 4 files changed, 42 insertions(+) create mode 100644 netbox/dcim/migrations/0201_add_power_outlet_status.py diff --git a/netbox/dcim/choices.py b/netbox/dcim/choices.py index c5b6cbcad..8bd41b3d2 100644 --- a/netbox/dcim/choices.py +++ b/netbox/dcim/choices.py @@ -1627,6 +1627,23 @@ class PowerFeedPhaseChoices(ChoiceSet): ) +# +# PowerOutlets +# +class PowerOutletStatusChoices(ChoiceSet): + key = 'PowerOutlet.status' + + STATUS_ENABLED = 'enabled' + STATUS_DISABLED = 'disabled' + STATUS_FAULTY = 'faulty' + + CHOICES = [ + (STATUS_ENABLED, _('Enabled'), 'green'), + (STATUS_DISABLED, _('Disabled'), 'red'), + (STATUS_FAULTY, _('Faulty'), 'gray'), + ] + + # # VDC # diff --git a/netbox/dcim/migrations/0201_add_power_outlet_status.py b/netbox/dcim/migrations/0201_add_power_outlet_status.py new file mode 100644 index 000000000..21fd32186 --- /dev/null +++ b/netbox/dcim/migrations/0201_add_power_outlet_status.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dcim', '0200_populate_mac_addresses'), + ] + + operations = [ + migrations.AddField( + model_name='poweroutlet', + name='status', + field=models.CharField(default='enabled', max_length=50), + ), + ] diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 8a8e8f4cc..49b709944 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -449,6 +449,12 @@ class PowerOutlet(ModularComponentModel, CabledObjectModel, PathEndpoint, Tracki """ A physical power outlet (output) within a Device which provides power to a PowerPort. """ + status = models.CharField( + verbose_name=_('status'), + max_length=50, + choices=PowerOutletStatusChoices, + default=PowerOutletStatusChoices.STATUS_ENABLED + ) type = models.CharField( verbose_name=_('type'), max_length=50, diff --git a/netbox/dcim/tests/test_views.py b/netbox/dcim/tests/test_views.py index b84217882..4dea94c7d 100644 --- a/netbox/dcim/tests/test_views.py +++ b/netbox/dcim/tests/test_views.py @@ -2513,6 +2513,7 @@ class PowerOutletTestCase(ViewTestCases.DeviceComponentViewTestCase): 'device': device.pk, 'name': 'Power Outlet X', 'type': PowerOutletTypeChoices.TYPE_IEC_C13, + 'status': PowerOutletStatusChoices.STATUS_ENABLED, 'power_port': powerports[1].pk, 'feed_leg': PowerOutletFeedLegChoices.FEED_LEG_B, 'description': 'A power outlet', @@ -2523,6 +2524,7 @@ class PowerOutletTestCase(ViewTestCases.DeviceComponentViewTestCase): 'device': device.pk, 'name': 'Power Outlet [4-6]', 'type': PowerOutletTypeChoices.TYPE_IEC_C13, + 'status': PowerOutletStatusChoices.STATUS_ENABLED, 'power_port': powerports[1].pk, 'feed_leg': PowerOutletFeedLegChoices.FEED_LEG_B, 'description': 'A power outlet', @@ -2531,6 +2533,7 @@ class PowerOutletTestCase(ViewTestCases.DeviceComponentViewTestCase): cls.bulk_edit_data = { 'type': PowerOutletTypeChoices.TYPE_IEC_C15, + 'status': PowerOutletStatusChoices.STATUS_ENABLED, 'power_port': powerports[1].pk, 'feed_leg': PowerOutletFeedLegChoices.FEED_LEG_B, 'description': 'New description', From d9d7955c19ea8b40c2f7121b5ccb23ad3cd8420c Mon Sep 17 00:00:00 2001 From: Jason Novinger Date: Wed, 26 Feb 2025 16:33:04 -0600 Subject: [PATCH 15/25] For #18352, adds PowerOutlet.status field to forms and filtersets --- netbox/dcim/filtersets.py | 6 +++++- netbox/dcim/forms/bulk_edit.py | 7 +++++-- netbox/dcim/forms/filtersets.py | 7 ++++++- netbox/dcim/forms/model_forms.py | 4 ++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/netbox/dcim/filtersets.py b/netbox/dcim/filtersets.py index 60c3c4d38..e46730da8 100644 --- a/netbox/dcim/filtersets.py +++ b/netbox/dcim/filtersets.py @@ -1591,11 +1591,15 @@ class PowerOutletFilterSet( queryset=PowerPort.objects.all(), label=_('Power port (ID)'), ) + status = django_filters.MultipleChoiceFilter( + choices=PowerOutletStatusChoices, + null_value=None + ) class Meta: model = PowerOutlet fields = ( - 'id', 'name', 'label', 'feed_leg', 'description', 'color', 'mark_connected', 'cable_end', + 'id', 'name', 'status', 'label', 'feed_leg', 'description', 'color', 'mark_connected', 'cable_end', ) diff --git a/netbox/dcim/forms/bulk_edit.py b/netbox/dcim/forms/bulk_edit.py index da5a45f15..3b9a183cd 100644 --- a/netbox/dcim/forms/bulk_edit.py +++ b/netbox/dcim/forms/bulk_edit.py @@ -1379,7 +1379,10 @@ class PowerPortBulkEditForm( class PowerOutletBulkEditForm( ComponentBulkEditForm, - form_from_model(PowerOutlet, ['label', 'type', 'color', 'feed_leg', 'power_port', 'mark_connected', 'description']) + form_from_model( + PowerOutlet, + ['label', 'type', 'status', 'color', 'feed_leg', 'power_port', 'mark_connected', 'description'] + ) ): mark_connected = forms.NullBooleanField( label=_('Mark connected'), @@ -1389,7 +1392,7 @@ class PowerOutletBulkEditForm( model = PowerOutlet fieldsets = ( - FieldSet('module', 'type', 'label', 'description', 'mark_connected', 'color'), + FieldSet('module', 'type', 'label', 'status', 'description', 'mark_connected', 'color'), FieldSet('feed_leg', 'power_port', name=_('Power')), ) nullable_fields = ('module', 'label', 'type', 'feed_leg', 'power_port', 'description') diff --git a/netbox/dcim/forms/filtersets.py b/netbox/dcim/forms/filtersets.py index 37b8afd17..d794c6893 100644 --- a/netbox/dcim/forms/filtersets.py +++ b/netbox/dcim/forms/filtersets.py @@ -1305,7 +1305,7 @@ class PowerOutletFilterForm(PathEndpointFilterForm, DeviceComponentFilterForm): model = PowerOutlet fieldsets = ( FieldSet('q', 'filter_id', 'tag'), - FieldSet('name', 'label', 'type', 'color', name=_('Attributes')), + FieldSet('name', 'label', 'type', 'color', 'status', name=_('Attributes')), FieldSet('region_id', 'site_group_id', 'site_id', 'location_id', 'rack_id', name=_('Location')), FieldSet( 'device_type_id', 'device_role_id', 'device_id', 'device_status', 'virtual_chassis_id', @@ -1323,6 +1323,11 @@ class PowerOutletFilterForm(PathEndpointFilterForm, DeviceComponentFilterForm): label=_('Color'), required=False ) + status = forms.MultipleChoiceField( + label=_('Status'), + choices=PowerOutletStatusChoices, + required=False + ) class InterfaceFilterForm(PathEndpointFilterForm, DeviceComponentFilterForm): diff --git a/netbox/dcim/forms/model_forms.py b/netbox/dcim/forms/model_forms.py index 5a3a27d25..91e23e8b1 100644 --- a/netbox/dcim/forms/model_forms.py +++ b/netbox/dcim/forms/model_forms.py @@ -1308,7 +1308,7 @@ class PowerOutletForm(ModularDeviceComponentForm): fieldsets = ( FieldSet( - 'device', 'module', 'name', 'label', 'type', 'color', 'power_port', 'feed_leg', 'mark_connected', + 'device', 'module', 'name', 'label', 'type', 'status', 'color', 'power_port', 'feed_leg', 'mark_connected', 'description', 'tags', ), ) @@ -1316,7 +1316,7 @@ class PowerOutletForm(ModularDeviceComponentForm): class Meta: model = PowerOutlet fields = [ - 'device', 'module', 'name', 'label', 'type', 'color', 'power_port', 'feed_leg', 'mark_connected', + 'device', 'module', 'name', 'label', 'type', 'status', 'color', 'power_port', 'feed_leg', 'mark_connected', 'description', 'tags', ] From 9556b0c480a55631000ac87c83bbd0cb2bb289f8 Mon Sep 17 00:00:00 2001 From: Jason Novinger Date: Wed, 26 Feb 2025 16:42:44 -0600 Subject: [PATCH 16/25] Adds status field to PowerOutletSerializer --- netbox/dcim/api/serializers_/device_components.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/netbox/dcim/api/serializers_/device_components.py b/netbox/dcim/api/serializers_/device_components.py index b591030aa..8b9cd42df 100644 --- a/netbox/dcim/api/serializers_/device_components.py +++ b/netbox/dcim/api/serializers_/device_components.py @@ -156,10 +156,10 @@ class PowerOutletSerializer(NetBoxModelSerializer, CabledObjectSerializer, Conne class Meta: model = PowerOutlet fields = [ - 'id', 'url', 'display_url', 'display', 'device', 'module', 'name', 'label', 'type', 'color', 'power_port', - 'feed_leg', 'description', 'mark_connected', 'cable', 'cable_end', 'link_peers', 'link_peers_type', - 'connected_endpoints', 'connected_endpoints_type', 'connected_endpoints_reachable', 'tags', 'custom_fields', - 'created', 'last_updated', '_occupied', + 'id', 'url', 'display_url', 'display', 'device', 'module', 'name', 'label', 'type', 'status', 'color', + 'power_port', 'feed_leg', 'description', 'mark_connected', 'cable', 'cable_end', 'link_peers', + 'link_peers_type', 'connected_endpoints', 'connected_endpoints_type', 'connected_endpoints_reachable', + 'tags', 'custom_fields', 'created', 'last_updated', '_occupied', ] brief_fields = ('id', 'url', 'display', 'device', 'name', 'description', 'cable', '_occupied') From 1d5c67a0a8f947bcb49b290ae75a3906b7949050 Mon Sep 17 00:00:00 2001 From: Jason Novinger Date: Wed, 26 Feb 2025 16:44:03 -0600 Subject: [PATCH 17/25] Adds PowerOutlet.status field to PowerOutlet model tables --- netbox/dcim/tables/devices.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/netbox/dcim/tables/devices.py b/netbox/dcim/tables/devices.py index d4f2f74b3..f69c58994 100644 --- a/netbox/dcim/tables/devices.py +++ b/netbox/dcim/tables/devices.py @@ -530,9 +530,11 @@ class PowerOutletTable(ModularDeviceComponentTable, PathEndpointTable): fields = ( 'pk', 'id', 'name', 'device', 'module_bay', 'module', 'label', 'type', 'description', 'power_port', 'color', 'feed_leg', 'mark_connected', 'cable', 'cable_color', 'link_peer', 'connection', 'inventory_items', - 'tags', 'created', 'last_updated', + 'tags', 'created', 'last_updated', 'status', + ) + default_columns = ( + 'pk', 'name', 'device', 'label', 'type', 'status', 'color', 'power_port', 'feed_leg', 'description', ) - default_columns = ('pk', 'name', 'device', 'label', 'type', 'color', 'power_port', 'feed_leg', 'description') class DevicePowerOutletTable(PowerOutletTable): @@ -550,9 +552,11 @@ class DevicePowerOutletTable(PowerOutletTable): fields = ( 'pk', 'id', 'name', 'module_bay', 'module', 'label', 'type', 'color', 'power_port', 'feed_leg', 'description', 'mark_connected', 'cable', 'cable_color', 'link_peer', 'connection', 'tags', 'actions', + 'status', ) default_columns = ( - 'pk', 'name', 'label', 'type', 'color', 'power_port', 'feed_leg', 'description', 'cable', 'connection', + 'pk', 'name', 'label', 'type', 'status', 'color', 'power_port', 'feed_leg', 'description', 'cable', + 'connection', ) From f2a09333d7cc72406cffa314ef2ff02b0626f97a Mon Sep 17 00:00:00 2001 From: Jason Novinger Date: Wed, 26 Feb 2025 16:44:54 -0600 Subject: [PATCH 18/25] Updates PowerOutletIndex to display status field in results This seemed inline with status fields on other model search indexes --- netbox/dcim/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/dcim/search.py b/netbox/dcim/search.py index b964421de..5dea2a09b 100644 --- a/netbox/dcim/search.py +++ b/netbox/dcim/search.py @@ -224,7 +224,7 @@ class PowerOutletIndex(SearchIndex): ('label', 200), ('description', 500), ) - display_attrs = ('device', 'label', 'type', 'description') + display_attrs = ('device', 'label', 'type', 'status', 'description') @register_search From 8efcbddb374d4fdc5510555f8b2f9db497d3c7b4 Mon Sep 17 00:00:00 2001 From: Jason Novinger Date: Wed, 26 Feb 2025 16:51:07 -0600 Subject: [PATCH 19/25] Updates PowetOutler docs to include new status field --- docs/models/dcim/poweroutlet.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/models/dcim/poweroutlet.md b/docs/models/dcim/poweroutlet.md index a99f60b23..22a7ec63e 100644 --- a/docs/models/dcim/poweroutlet.md +++ b/docs/models/dcim/poweroutlet.md @@ -29,6 +29,19 @@ An alternative physical label identifying the power outlet. The type of power outlet. +### Status + +The operational status of the power outlet. By default, the following statuses are available: + +* Enabled +* Disabled +* Faulty + +!!! tip "Custom power outlet statuses" + Additional power outlet statuses may be defined by setting `PowerOutlet.status` under the [`FIELD_CHOICES`](../../configuration/data-validation.md#field_choices) configuration parameter. + +!!! info "This field was introduced in NetBox v4.3." + ### Color !!! info "This field was introduced in NetBox v4.2." From 2dcf2d203ca7c0d89b54e4093d7ff2189fdc6c96 Mon Sep 17 00:00:00 2001 From: Jason Novinger Date: Wed, 26 Feb 2025 17:09:56 -0600 Subject: [PATCH 20/25] Extend filterset/model tests to cover PowerOutlet.status --- netbox/dcim/tests/test_filtersets.py | 20 ++++++++++++++++++++ netbox/dcim/tests/test_models.py | 3 ++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/netbox/dcim/tests/test_filtersets.py b/netbox/dcim/tests/test_filtersets.py index ede1e2a09..7c9b8adc6 100644 --- a/netbox/dcim/tests/test_filtersets.py +++ b/netbox/dcim/tests/test_filtersets.py @@ -3684,6 +3684,7 @@ class PowerOutletTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedF feed_leg=PowerOutletFeedLegChoices.FEED_LEG_A, description='First', color='ff0000', + status=PowerOutletStatusChoices.STATUS_ENABLED, ), PowerOutlet( device=devices[1], @@ -3693,6 +3694,7 @@ class PowerOutletTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedF feed_leg=PowerOutletFeedLegChoices.FEED_LEG_B, description='Second', color='00ff00', + status=PowerOutletStatusChoices.STATUS_DISABLED, ), PowerOutlet( device=devices[2], @@ -3702,6 +3704,7 @@ class PowerOutletTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedF feed_leg=PowerOutletFeedLegChoices.FEED_LEG_C, description='Third', color='0000ff', + status=PowerOutletStatusChoices.STATUS_FAULTY, ), ) PowerOutlet.objects.bulk_create(power_outlets) @@ -3796,6 +3799,23 @@ class PowerOutletTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedF params = {'connected': False} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + def test_status(self): + params = {'status': [PowerOutletStatusChoices.STATUS_ENABLED]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + + params = {'status': [PowerOutletStatusChoices.STATUS_DISABLED]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + + params = {'status': [PowerOutletStatusChoices.STATUS_FAULTY]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + + params = {'status': [ + PowerOutletStatusChoices.STATUS_ENABLED, + PowerOutletStatusChoices.STATUS_DISABLED, + PowerOutletStatusChoices.STATUS_FAULTY, + ]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3) + class InterfaceTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedFilterSetTests): queryset = Interface.objects.all() diff --git a/netbox/dcim/tests/test_models.py b/netbox/dcim/tests/test_models.py index ff1eddd56..bdb07d6d1 100644 --- a/netbox/dcim/tests/test_models.py +++ b/netbox/dcim/tests/test_models.py @@ -465,7 +465,8 @@ class DeviceTestCase(TestCase): device=device, name='Power Outlet 1', power_port=powerport, - feed_leg=PowerOutletFeedLegChoices.FEED_LEG_A + feed_leg=PowerOutletFeedLegChoices.FEED_LEG_A, + status=PowerOutletStatusChoices.STATUS_ENABLED, ) self.assertEqual(poweroutlet.cf['cf1'], 'foo') From cf7e2c8dc9b14cf94668eef122b9f78b145fcccd Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 27 Feb 2025 09:30:52 -0500 Subject: [PATCH 21/25] Closes #17424: Add custom visibility toggle to ViewTab --- netbox/utilities/views.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index b3334ca87..5a9830918 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -206,22 +206,30 @@ class ViewTab: Args: label: Human-friendly text + visible: A callable which determines whether the tab should be displayed. This callable must accept exactly one + argument: the object instance. If a callable is not specified, the tab's visibility will be determined by + its badge (if any) and the value of `hide_if_empty`. badge: A static value or callable to display alongside the label (optional). If a callable is used, it must accept a single argument representing the object being viewed. weight: Numeric weight to influence ordering among other tabs (default: 1000) permission: The permission required to display the tab (optional). - hide_if_empty: If true, the tab will be displayed only if its badge has a meaningful value. (Tabs without a - badge are always displayed.) + hide_if_empty: If true, the tab will be displayed only if its badge has a meaningful value. (This parameter is + evaluated only if the tab is permitted to be displayed according to the `visible` parameter.) """ - def __init__(self, label, badge=None, weight=1000, permission=None, hide_if_empty=False): + def __init__(self, label, visible=None, badge=None, weight=1000, permission=None, hide_if_empty=False): self.label = label + self.visible = visible self.badge = badge self.weight = weight self.permission = permission self.hide_if_empty = hide_if_empty def render(self, instance): - """Return the attributes needed to render a tab in HTML.""" + """ + Return the attributes needed to render a tab in HTML if the tab should be displayed. Otherwise, return None. + """ + if self.visible is not None and not self.visible(instance): + return None badge_value = self._get_badge_value(instance) if self.badge and self.hide_if_empty and not badge_value: return None From 2ae84ce9fb40ec94cd4b88269417a57bfa513099 Mon Sep 17 00:00:00 2001 From: Jason Novinger Date: Thu, 27 Feb 2025 15:02:14 -0600 Subject: [PATCH 22/25] Adds initial PowerOutletForm tests --- netbox/dcim/tests/test_forms.py | 54 ++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/netbox/dcim/tests/test_forms.py b/netbox/dcim/tests/test_forms.py index 89b7508f3..0067acaaf 100644 --- a/netbox/dcim/tests/test_forms.py +++ b/netbox/dcim/tests/test_forms.py @@ -1,6 +1,8 @@ from django.test import TestCase -from dcim.choices import DeviceFaceChoices, DeviceStatusChoices, InterfaceTypeChoices, InterfaceModeChoices +from dcim.choices import ( + DeviceFaceChoices, DeviceStatusChoices, InterfaceTypeChoices, InterfaceModeChoices, PowerOutletStatusChoices +) from dcim.forms import * from dcim.models import * from ipam.models import VLAN @@ -12,6 +14,56 @@ def get_id(model, slug): return model.objects.get(slug=slug).id +class PowerOutletFormTestCase(TestCase): + @classmethod + def setUpTestData(cls): + cls.site = site = Site.objects.create(name='Site 1', slug='site-1') + cls.manufacturer = manufacturer = Manufacturer.objects.create(name='Manufacturer 1', slug='manufacturer-1') + cls.role = role = DeviceRole.objects.create( + name='Device Role 1', slug='device-role-1', color='ff0000' + ) + cls.device_type = device_type = DeviceType.objects.create( + manufacturer=manufacturer, model='Device Type 1', slug='device-type-1', u_height=1 + ) + cls.rack = rack = Rack.objects.create(name='Rack 1', site=site) + cls.device = Device.objects.create( + name='Device 1', device_type=device_type, role=role, site=site, rack=rack, position=1 + ) + + def test_status_is_required(self): + form = PowerOutletForm(data={ + 'device': self.device, + 'module': None, + 'name': 'New Enabled Outlet', + }) + self.assertFalse(form.is_valid()) + self.assertIn('status', form.errors) + + def test_status_must_be_defined_choice(self): + form = PowerOutletForm(data={ + 'device': self.device, + 'module': None, + 'name': 'New Enabled Outlet', + 'status': 'this isn\'t a defined choice', + }) + self.assertFalse(form.is_valid()) + self.assertIn('status', form.errors) + self.assertTrue(form.errors['status'][-1].startswith('Select a valid choice.')) + + def test_status_recognizes_choices(self): + for index, choice in enumerate(PowerOutletStatusChoices.CHOICES): + form = PowerOutletForm(data={ + 'device': self.device, + 'module': None, + 'name': f'New Enabled Outlet {index + 1}', + 'status': choice[0], + }) + self.assertEqual({}, form.errors) + self.assertTrue(form.is_valid()) + instance = form.save() + self.assertEqual(instance.status, choice[0]) + + class DeviceTestCase(TestCase): @classmethod From 77b98205776efd01ea86846d4882e1415e85841a Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 3 Mar 2025 09:29:40 -0500 Subject: [PATCH 23/25] Closes #18287: Enable periodic synchronization for data sources (#18747) * Add sync_interval to DataSource * Enqueue a SyncDataSourceJob when needed after saving a DataSource * Fix logic for clearing pending jobs on interval change * Fix lingering background tasks after modifying DataSource --- docs/models/core/datasource.md | 6 +++++ netbox/core/api/serializers_/data.py | 4 ++-- netbox/core/filtersets.py | 4 ++++ netbox/core/forms/bulk_edit.py | 10 ++++++-- netbox/core/forms/bulk_import.py | 3 ++- netbox/core/forms/filtersets.py | 7 +++++- netbox/core/forms/model_forms.py | 7 ++++-- .../0013_datasource_sync_interval.py | 18 ++++++++++++++ netbox/core/models/data.py | 6 +++++ netbox/core/signals.py | 24 ++++++++++++++++--- netbox/core/tables/data.py | 9 ++++--- netbox/core/tests/test_filtersets.py | 13 +++++++--- netbox/templates/core/datasource.html | 4 ++++ 13 files changed, 98 insertions(+), 17 deletions(-) create mode 100644 netbox/core/migrations/0013_datasource_sync_interval.py diff --git a/docs/models/core/datasource.md b/docs/models/core/datasource.md index 0e18a2aae..527d93939 100644 --- a/docs/models/core/datasource.md +++ b/docs/models/core/datasource.md @@ -44,6 +44,12 @@ A set of rules (one per line) identifying filenames to ignore during synchroniza | `*.txt` | Ignore any files with a `.txt` extension | | `data???.json` | Ignore e.g. `data123.json` | +### Sync Interval + +!!! info "This field was introduced in NetBox v4.3." + +The interval at which the data source should automatically synchronize. If not set, the data source must be synchronized manually. + ### Last Synced The date and time at which the source was most recently synchronized successfully. diff --git a/netbox/core/api/serializers_/data.py b/netbox/core/api/serializers_/data.py index 2c155ba6b..3f2ddb2a0 100644 --- a/netbox/core/api/serializers_/data.py +++ b/netbox/core/api/serializers_/data.py @@ -26,8 +26,8 @@ class DataSourceSerializer(NetBoxModelSerializer): model = DataSource fields = [ 'id', 'url', 'display_url', 'display', 'name', 'type', 'source_url', 'enabled', 'status', 'description', - 'parameters', 'ignore_rules', 'comments', 'custom_fields', 'created', 'last_updated', 'last_synced', - 'file_count', + 'sync_interval', 'parameters', 'ignore_rules', 'comments', 'custom_fields', 'created', 'last_updated', + 'last_synced', 'file_count', ] brief_fields = ('id', 'url', 'display', 'name', 'description') diff --git a/netbox/core/filtersets.py b/netbox/core/filtersets.py index 21fdaa4ab..42ec22350 100644 --- a/netbox/core/filtersets.py +++ b/netbox/core/filtersets.py @@ -29,6 +29,10 @@ class DataSourceFilterSet(NetBoxModelFilterSet): choices=DataSourceStatusChoices, null_value=None ) + sync_interval = django_filters.MultipleChoiceFilter( + choices=JobIntervalChoices, + null_value=None + ) class Meta: model = DataSource diff --git a/netbox/core/forms/bulk_edit.py b/netbox/core/forms/bulk_edit.py index c1f1fca4d..73618826d 100644 --- a/netbox/core/forms/bulk_edit.py +++ b/netbox/core/forms/bulk_edit.py @@ -1,6 +1,7 @@ from django import forms from django.utils.translation import gettext_lazy as _ +from core.choices import JobIntervalChoices from core.models import * from netbox.forms import NetBoxModelBulkEditForm from netbox.utils import get_data_backend_choices @@ -29,6 +30,11 @@ class DataSourceBulkEditForm(NetBoxModelBulkEditForm): max_length=200, required=False ) + sync_interval = forms.ChoiceField( + choices=JobIntervalChoices, + required=False, + label=_('Sync interval') + ) comments = CommentField() parameters = forms.JSONField( label=_('Parameters'), @@ -42,8 +48,8 @@ class DataSourceBulkEditForm(NetBoxModelBulkEditForm): model = DataSource fieldsets = ( - FieldSet('type', 'enabled', 'description', 'comments', 'parameters', 'ignore_rules'), + FieldSet('type', 'enabled', 'description', 'sync_interval', 'parameters', 'ignore_rules', 'comments'), ) nullable_fields = ( - 'description', 'description', 'parameters', 'comments', 'parameters', 'ignore_rules', + 'description', 'description', 'sync_interval', 'parameters', 'parameters', 'ignore_rules' 'comments', ) diff --git a/netbox/core/forms/bulk_import.py b/netbox/core/forms/bulk_import.py index 78a859dcb..a5791c945 100644 --- a/netbox/core/forms/bulk_import.py +++ b/netbox/core/forms/bulk_import.py @@ -11,5 +11,6 @@ class DataSourceImportForm(NetBoxModelImportForm): class Meta: model = DataSource fields = ( - 'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'parameters', 'ignore_rules', + 'name', 'type', 'source_url', 'enabled', 'description', 'sync_interval', 'parameters', 'ignore_rules', + 'comments', ) diff --git a/netbox/core/forms/filtersets.py b/netbox/core/forms/filtersets.py index ab4b869b7..4e7286737 100644 --- a/netbox/core/forms/filtersets.py +++ b/netbox/core/forms/filtersets.py @@ -27,7 +27,7 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm): model = DataSource fieldsets = ( FieldSet('q', 'filter_id'), - FieldSet('type', 'status', name=_('Data Source')), + FieldSet('type', 'status', 'enabled', 'sync_interval', name=_('Data Source')), ) type = forms.MultipleChoiceField( label=_('Type'), @@ -46,6 +46,11 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm): choices=BOOLEAN_WITH_BLANK_CHOICES ) ) + sync_interval = forms.ChoiceField( + label=_('Sync interval'), + choices=JobIntervalChoices, + required=False + ) class DataFileFilterForm(NetBoxModelFilterSetForm): diff --git a/netbox/core/forms/model_forms.py b/netbox/core/forms/model_forms.py index a05377597..0a683a381 100644 --- a/netbox/core/forms/model_forms.py +++ b/netbox/core/forms/model_forms.py @@ -36,7 +36,7 @@ class DataSourceForm(NetBoxModelForm): class Meta: model = DataSource fields = [ - 'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags', + 'name', 'type', 'source_url', 'enabled', 'description', 'sync_interval', 'ignore_rules', 'comments', 'tags', ] widgets = { 'ignore_rules': forms.Textarea( @@ -51,7 +51,10 @@ class DataSourceForm(NetBoxModelForm): @property def fieldsets(self): fieldsets = [ - FieldSet('name', 'type', 'source_url', 'enabled', 'description', 'tags', 'ignore_rules', name=_('Source')), + FieldSet( + 'name', 'type', 'source_url', 'description', 'tags', 'ignore_rules', name=_('Source') + ), + FieldSet('enabled', 'sync_interval', name=_('Sync')), ] if self.backend_fields: fieldsets.append( diff --git a/netbox/core/migrations/0013_datasource_sync_interval.py b/netbox/core/migrations/0013_datasource_sync_interval.py new file mode 100644 index 000000000..ec3d2a5d6 --- /dev/null +++ b/netbox/core/migrations/0013_datasource_sync_interval.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.6 on 2025-02-26 19:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0012_job_object_type_optional'), + ] + + operations = [ + migrations.AddField( + model_name='datasource', + name='sync_interval', + field=models.PositiveSmallIntegerField(blank=True, null=True), + ), + ] diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py index 39ee8fa57..3f97fb003 100644 --- a/netbox/core/models/data.py +++ b/netbox/core/models/data.py @@ -59,6 +59,12 @@ class DataSource(JobsMixin, PrimaryModel): verbose_name=_('enabled'), default=True ) + sync_interval = models.PositiveSmallIntegerField( + verbose_name=_('sync interval'), + choices=JobIntervalChoices, + blank=True, + null=True + ) ignore_rules = models.TextField( verbose_name=_('ignore rules'), blank=True, diff --git a/netbox/core/signals.py b/netbox/core/signals.py index 06432bf4c..bdaa60f97 100644 --- a/netbox/core/signals.py +++ b/netbox/core/signals.py @@ -8,16 +8,15 @@ from django.dispatch import receiver, Signal from django.utils.translation import gettext_lazy as _ from django_prometheus.models import model_deletes, model_inserts, model_updates -from core.choices import ObjectChangeActionChoices +from core.choices import JobStatusChoices, ObjectChangeActionChoices from core.events import * -from core.models import ObjectChange from extras.events import enqueue_event from extras.utils import run_validators from netbox.config import get_config from netbox.context import current_request, events_queue from netbox.models.features import ChangeLoggingMixin from utilities.exceptions import AbortRequest -from .models import ConfigRevision +from .models import ConfigRevision, DataSource, ObjectChange __all__ = ( 'clear_events', @@ -182,6 +181,25 @@ def clear_events_queue(sender, **kwargs): # DataSource handlers # +@receiver(post_save, sender=DataSource) +def enqueue_sync_job(instance, created, **kwargs): + """ + When a DataSource is saved, check its sync_interval and enqueue a sync job if appropriate. + """ + from .jobs import SyncDataSourceJob + + if instance.enabled and instance.sync_interval: + SyncDataSourceJob.enqueue_once(instance, interval=instance.sync_interval) + elif not created: + # Delete any previously scheduled recurring jobs for this DataSource + for job in SyncDataSourceJob.get_jobs(instance).defer('data').filter( + interval__isnull=False, + status=JobStatusChoices.STATUS_SCHEDULED + ): + # Call delete() per instance to ensure the associated background task is deleted as well + job.delete() + + @receiver(post_sync) def auto_sync(instance, **kwargs): """ diff --git a/netbox/core/tables/data.py b/netbox/core/tables/data.py index 4059ea9bc..5d237c689 100644 --- a/netbox/core/tables/data.py +++ b/netbox/core/tables/data.py @@ -25,6 +25,9 @@ class DataSourceTable(NetBoxTable): enabled = columns.BooleanColumn( verbose_name=_('Enabled'), ) + sync_interval = columns.ChoiceFieldColumn( + verbose_name=_('Sync interval'), + ) tags = columns.TagColumn( url_name='core:datasource_list' ) @@ -35,10 +38,10 @@ class DataSourceTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = DataSource fields = ( - 'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters', - 'created', 'last_updated', 'file_count', + 'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'sync_interval', 'comments', + 'parameters', 'created', 'last_updated', 'file_count', ) - default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count') + default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'sync_interval', 'file_count') class DataFileTable(NetBoxTable): diff --git a/netbox/core/tests/test_filtersets.py b/netbox/core/tests/test_filtersets.py index 310be1d0e..b7dfd516e 100644 --- a/netbox/core/tests/test_filtersets.py +++ b/netbox/core/tests/test_filtersets.py @@ -27,7 +27,8 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests): source_url='file:///var/tmp/source1/', status=DataSourceStatusChoices.NEW, enabled=True, - description='foobar1' + description='foobar1', + sync_interval=JobIntervalChoices.INTERVAL_HOURLY ), DataSource( name='Data Source 2', @@ -35,14 +36,16 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests): source_url='file:///var/tmp/source2/', status=DataSourceStatusChoices.SYNCING, enabled=True, - description='foobar2' + description='foobar2', + sync_interval=JobIntervalChoices.INTERVAL_DAILY ), DataSource( name='Data Source 3', type='git', source_url='https://example.com/git/source3', status=DataSourceStatusChoices.COMPLETED, - enabled=False + enabled=False, + sync_interval=JobIntervalChoices.INTERVAL_WEEKLY ), ) DataSource.objects.bulk_create(data_sources) @@ -73,6 +76,10 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests): params = {'status': [DataSourceStatusChoices.NEW, DataSourceStatusChoices.SYNCING]} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + def test_sync_interval(self): + params = {'sync_interval': [JobIntervalChoices.INTERVAL_HOURLY, JobIntervalChoices.INTERVAL_DAILY]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + class DataFileTestCase(TestCase, ChangeLoggedFilterSetTests): queryset = DataFile.objects.all() diff --git a/netbox/templates/core/datasource.html b/netbox/templates/core/datasource.html index a5afedec6..0d56c4087 100644 --- a/netbox/templates/core/datasource.html +++ b/netbox/templates/core/datasource.html @@ -46,6 +46,10 @@ {% trans "Status" %} {% badge object.get_status_display bg_color=object.get_status_color %} + + {% trans "Sync interval" %} + {{ object.get_sync_interval_display|placeholder }} + {% trans "Last synced" %} {{ object.last_synced|placeholder }} From 913405a3ae93ec28b8970a2dbdd81c99508dd557 Mon Sep 17 00:00:00 2001 From: Jason Novinger Date: Mon, 3 Mar 2025 12:22:34 -0600 Subject: [PATCH 24/25] Adds PowerOutlet.status to detail view Also fixes color display in list table and detail template --- netbox/dcim/models/device_components.py | 3 +++ netbox/dcim/tables/devices.py | 3 +++ netbox/templates/dcim/poweroutlet.html | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 49b709944..6a994d770 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -498,6 +498,9 @@ class PowerOutlet(ModularComponentModel, CabledObjectModel, PathEndpoint, Tracki _("Parent power port ({power_port}) must belong to the same device").format(power_port=self.power_port) ) + def get_status_color(self): + return PowerOutletStatusChoices.colors.get(self.status) + # # Interfaces diff --git a/netbox/dcim/tables/devices.py b/netbox/dcim/tables/devices.py index f69c58994..06f6469d3 100644 --- a/netbox/dcim/tables/devices.py +++ b/netbox/dcim/tables/devices.py @@ -520,6 +520,9 @@ class PowerOutletTable(ModularDeviceComponentTable, PathEndpointTable): verbose_name=_('Power Port'), linkify=True ) + status = columns.ChoiceFieldColumn( + verbose_name=_('Status'), + ) color = columns.ColorColumn() tags = columns.TagColumn( url_name='dcim:poweroutlet_list' diff --git a/netbox/templates/dcim/poweroutlet.html b/netbox/templates/dcim/poweroutlet.html index 146f6d580..8e44df88e 100644 --- a/netbox/templates/dcim/poweroutlet.html +++ b/netbox/templates/dcim/poweroutlet.html @@ -36,6 +36,10 @@ {% trans "Type" %} {{ object.get_type_display }} + + {% trans "Status" %} + {% badge object.get_status_display bg_color=object.get_status_color %} + {% trans "Description" %} {{ object.description|placeholder }} From 4e65117e7c77c3a07b90c448b7af645e41f08c71 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 4 Mar 2025 08:24:54 -0500 Subject: [PATCH 25/25] Closes #18627: Proxy routing (#18681) * Introduce proxy routing * Misc cleanup * Document PROXY_ROUTERS parameter --- docs/configuration/system.md | 14 ++++- netbox/core/data_backends.py | 26 ++++----- netbox/core/jobs.py | 3 +- netbox/core/plugins.py | 6 +- netbox/extras/dashboard/widgets.py | 3 +- .../management/commands/housekeeping.py | 3 +- netbox/extras/webhooks.py | 8 ++- netbox/netbox/settings.py | 13 ++++- netbox/utilities/proxy.py | 55 +++++++++++++++++++ 9 files changed, 108 insertions(+), 23 deletions(-) create mode 100644 netbox/utilities/proxy.py diff --git a/docs/configuration/system.md b/docs/configuration/system.md index af3a6f5e6..81c1a6a94 100644 --- a/docs/configuration/system.md +++ b/docs/configuration/system.md @@ -64,7 +64,7 @@ Email is sent from NetBox only for critical events or if configured for [logging ## HTTP_PROXIES -Default: None +Default: Empty A dictionary of HTTP proxies to use for outbound requests originating from NetBox (e.g. when sending webhook requests). Proxies should be specified by schema (HTTP and HTTPS) as per the [Python requests library documentation](https://requests.readthedocs.io/en/latest/user/advanced/#proxies). For example: @@ -75,6 +75,8 @@ HTTP_PROXIES = { } ``` +If more flexibility is needed in determining which proxy to use for a given request, consider implementing one or more custom proxy routers via the [`PROXY_ROUTERS`](#proxy_routers) parameter. + --- ## INTERNAL_IPS @@ -160,6 +162,16 @@ The file path to the location where media files (such as image attachments) are --- +## PROXY_ROUTERS + +Default: `["utilities.proxy.DefaultProxyRouter"]` + +A list of Python classes responsible for determining which proxy server(s) to use for outbound HTTP requests. Each item in the list can be the class itself or the dotted path to the class. + +The `route()` method on each class must return a dictionary of candidate proxies arranged by protocol (e.g. `http` and/or `https`), or None if no viable proxy can be determined. The default class, `DefaultProxyRouter`, simply returns the content of [`HTTP_PROXIES`](#http_proxies). + +--- + ## REPORTS_ROOT Default: `$INSTALL_ROOT/netbox/reports/` diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index 770a3b258..9ba1d5dfd 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -7,13 +7,13 @@ from pathlib import Path from urllib.parse import urlparse from django import forms -from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.utils.translation import gettext as _ from netbox.data_backends import DataBackend from netbox.utils import register_data_backend from utilities.constants import HTTP_PROXY_SUPPORTED_SCHEMAS, HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS +from utilities.proxy import resolve_proxies from utilities.socks import ProxyPoolManager from .exceptions import SyncError @@ -70,18 +70,18 @@ class GitBackend(DataBackend): # Initialize backend config config = ConfigDict() - self.use_socks = False + self.socks_proxy = None # Apply HTTP proxy (if configured) - if settings.HTTP_PROXIES: - if proxy := settings.HTTP_PROXIES.get(self.url_scheme, None): - if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS: - raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}") + proxies = resolve_proxies(url=self.url, context={'client': self}) or {} + if proxy := proxies.get(self.url_scheme): + if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS: + raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}") - if self.url_scheme in ('http', 'https'): - config.set("http", "proxy", proxy) - if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS: - self.use_socks = True + if self.url_scheme in ('http', 'https'): + config.set("http", "proxy", proxy) + if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS: + self.socks_proxy = proxy return config @@ -98,8 +98,8 @@ class GitBackend(DataBackend): } # check if using socks for proxy - if so need to use custom pool_manager - if self.use_socks: - clone_args['pool_manager'] = ProxyPoolManager(settings.HTTP_PROXIES.get(self.url_scheme)) + if self.socks_proxy: + clone_args['pool_manager'] = ProxyPoolManager(self.socks_proxy) if self.url_scheme in ('http', 'https'): if self.params.get('username'): @@ -147,7 +147,7 @@ class S3Backend(DataBackend): # Initialize backend config return Boto3Config( - proxies=settings.HTTP_PROXIES, + proxies=resolve_proxies(url=self.url, context={'client': self}), ) @contextmanager diff --git a/netbox/core/jobs.py b/netbox/core/jobs.py index 891b1cbdb..b3dfaf1e7 100644 --- a/netbox/core/jobs.py +++ b/netbox/core/jobs.py @@ -5,6 +5,7 @@ import sys from django.conf import settings from netbox.jobs import JobRunner, system_job from netbox.search.backends import search_backend +from utilities.proxy import resolve_proxies from .choices import DataSourceStatusChoices, JobIntervalChoices from .exceptions import SyncError from .models import DataSource @@ -71,7 +72,7 @@ class SystemHousekeepingJob(JobRunner): url=settings.CENSUS_URL, params=census_data, timeout=3, - proxies=settings.HTTP_PROXIES + proxies=resolve_proxies(url=settings.CENSUS_URL) ) except requests.exceptions.RequestException: pass diff --git a/netbox/core/plugins.py b/netbox/core/plugins.py index e6d09711f..d31a699e4 100644 --- a/netbox/core/plugins.py +++ b/netbox/core/plugins.py @@ -11,6 +11,7 @@ from django.core.cache import cache from netbox.plugins import PluginConfig from netbox.registry import registry from utilities.datetime import datetime_from_timestamp +from utilities.proxy import resolve_proxies USER_AGENT_STRING = f'NetBox/{settings.RELEASE.version} {settings.RELEASE.edition}' CACHE_KEY_CATALOG_FEED = 'plugins-catalog-feed' @@ -120,10 +121,11 @@ def get_catalog_plugins(): def get_pages(): # TODO: pagination is currently broken in API payload = {'page': '1', 'per_page': '50'} + proxies = resolve_proxies(url=settings.PLUGIN_CATALOG_URL) first_page = session.get( settings.PLUGIN_CATALOG_URL, headers={'User-Agent': USER_AGENT_STRING}, - proxies=settings.HTTP_PROXIES, + proxies=proxies, timeout=3, params=payload ).json() @@ -135,7 +137,7 @@ def get_catalog_plugins(): next_page = session.get( settings.PLUGIN_CATALOG_URL, headers={'User-Agent': USER_AGENT_STRING}, - proxies=settings.HTTP_PROXIES, + proxies=proxies, timeout=3, params=payload ).json() diff --git a/netbox/extras/dashboard/widgets.py b/netbox/extras/dashboard/widgets.py index eeed5414f..c04f8f423 100644 --- a/netbox/extras/dashboard/widgets.py +++ b/netbox/extras/dashboard/widgets.py @@ -17,6 +17,7 @@ from core.models import ObjectType from extras.choices import BookmarkOrderingChoices from utilities.object_types import object_type_identifier, object_type_name from utilities.permissions import get_permission_for_model +from utilities.proxy import resolve_proxies from utilities.querydict import dict_to_querydict from utilities.templatetags.builtins.filters import render_markdown from utilities.views import get_viewname @@ -330,7 +331,7 @@ class RSSFeedWidget(DashboardWidget): response = requests.get( url=self.config['feed_url'], headers={'User-Agent': f'NetBox/{settings.RELEASE.version}'}, - proxies=settings.HTTP_PROXIES, + proxies=resolve_proxies(url=self.config['feed_url'], context={'client': self}), timeout=3 ) response.raise_for_status() diff --git a/netbox/extras/management/commands/housekeeping.py b/netbox/extras/management/commands/housekeeping.py index ade486fc0..ade20a118 100644 --- a/netbox/extras/management/commands/housekeeping.py +++ b/netbox/extras/management/commands/housekeeping.py @@ -11,6 +11,7 @@ from packaging import version from core.models import Job, ObjectChange from netbox.config import Config +from utilities.proxy import resolve_proxies class Command(BaseCommand): @@ -107,7 +108,7 @@ class Command(BaseCommand): response = requests.get( url=settings.RELEASE_CHECK_URL, headers=headers, - proxies=settings.HTTP_PROXIES + proxies=resolve_proxies(url=settings.RELEASE_CHECK_URL) ) response.raise_for_status() diff --git a/netbox/extras/webhooks.py b/netbox/extras/webhooks.py index 889c97ac2..368075217 100644 --- a/netbox/extras/webhooks.py +++ b/netbox/extras/webhooks.py @@ -3,10 +3,10 @@ import hmac import logging import requests -from django.conf import settings from django_rq import job from jinja2.exceptions import TemplateError +from utilities.proxy import resolve_proxies from .constants import WEBHOOK_EVENT_TYPES logger = logging.getLogger('netbox.webhooks') @@ -63,9 +63,10 @@ def send_webhook(event_rule, model_name, event_type, data, timestamp, username, raise e # Prepare the HTTP request + url = webhook.render_payload_url(context) params = { 'method': webhook.http_method, - 'url': webhook.render_payload_url(context), + 'url': url, 'headers': headers, 'data': body.encode('utf8'), } @@ -88,7 +89,8 @@ def send_webhook(event_rule, model_name, event_type, data, timestamp, username, session.verify = webhook.ssl_verification if webhook.ca_file_path: session.verify = webhook.ca_file_path - response = session.send(prepared_request, proxies=settings.HTTP_PROXIES) + proxies = resolve_proxies(url=url, context={'client': webhook}) + response = session.send(prepared_request, proxies=proxies) if 200 <= response.status_code <= 299: logger.info(f"Request succeeded; response status {response.status_code}") diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index a17bb7730..0ad46b21e 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -9,6 +9,7 @@ import warnings from django.contrib.messages import constants as messages from django.core.exceptions import ImproperlyConfigured, ValidationError from django.core.validators import URLValidator +from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ from netbox.config import PARAMS as CONFIG_PARAMS @@ -116,7 +117,7 @@ EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', []) FIELD_CHOICES = getattr(configuration, 'FIELD_CHOICES', {}) FILE_UPLOAD_MAX_MEMORY_SIZE = getattr(configuration, 'FILE_UPLOAD_MAX_MEMORY_SIZE', 2621440) GRAPHQL_MAX_ALIASES = getattr(configuration, 'GRAPHQL_MAX_ALIASES', 10) -HTTP_PROXIES = getattr(configuration, 'HTTP_PROXIES', None) +HTTP_PROXIES = getattr(configuration, 'HTTP_PROXIES', {}) INTERNAL_IPS = getattr(configuration, 'INTERNAL_IPS', ('127.0.0.1', '::1')) ISOLATED_DEPLOYMENT = getattr(configuration, 'ISOLATED_DEPLOYMENT', False) JINJA2_FILTERS = getattr(configuration, 'JINJA2_FILTERS', {}) @@ -131,6 +132,7 @@ MEDIA_ROOT = getattr(configuration, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media' METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False) PLUGINS = getattr(configuration, 'PLUGINS', []) PLUGINS_CONFIG = getattr(configuration, 'PLUGINS_CONFIG', {}) +PROXY_ROUTERS = getattr(configuration, 'PROXY_ROUTERS', ['utilities.proxy.DefaultProxyRouter']) QUEUE_MAPPINGS = getattr(configuration, 'QUEUE_MAPPINGS', {}) REDIS = getattr(configuration, 'REDIS') # Required RELEASE_CHECK_URL = getattr(configuration, 'RELEASE_CHECK_URL', None) @@ -201,6 +203,14 @@ if RELEASE_CHECK_URL: "RELEASE_CHECK_URL must be a valid URL. Example: https://api.github.com/repos/netbox-community/netbox" ) +# Validate configured proxy routers +for path in PROXY_ROUTERS: + if type(path) is str: + try: + import_string(path) + except ImportError: + raise ImproperlyConfigured(f"Invalid path in PROXY_ROUTERS: {path}") + # # Database @@ -577,6 +587,7 @@ if SENTRY_ENABLED: sample_rate=SENTRY_SAMPLE_RATE, traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE, send_default_pii=SENTRY_SEND_DEFAULT_PII, + # TODO: Support proxy routing http_proxy=HTTP_PROXIES.get('http') if HTTP_PROXIES else None, https_proxy=HTTP_PROXIES.get('https') if HTTP_PROXIES else None ) diff --git a/netbox/utilities/proxy.py b/netbox/utilities/proxy.py new file mode 100644 index 000000000..8c9e3d196 --- /dev/null +++ b/netbox/utilities/proxy.py @@ -0,0 +1,55 @@ +from django.conf import settings +from django.utils.module_loading import import_string +from urllib.parse import urlparse + +__all__ = ( + 'DefaultProxyRouter', + 'resolve_proxies', +) + + +class DefaultProxyRouter: + """ + Base class for a proxy router. + """ + @staticmethod + def _get_protocol_from_url(url): + """ + Determine the applicable protocol (e.g. HTTP or HTTPS) from the given URL. + """ + return urlparse(url).scheme + + def route(self, url=None, protocol=None, context=None): + """ + Returns the appropriate proxy given a URL or protocol. Arbitrary context data may also be passed where + available. + + Args: + url: The specific request URL for which the proxy will be used (if known) + protocol: The protocol in use (e.g. http or https) (if known) + context: Additional context to aid in proxy selection. May include e.g. the requesting client. + """ + if url and protocol is None: + protocol = self._get_protocol_from_url(url) + if protocol and protocol in settings.HTTP_PROXIES: + return { + protocol: settings.HTTP_PROXIES[protocol] + } + return settings.HTTP_PROXIES + + +def resolve_proxies(url=None, protocol=None, context=None): + """ + Return a dictionary of candidate proxies (compatible with the requests module), or None. + + Args: + url: The specific request URL for which the proxy will be used (optional) + protocol: The protocol in use (e.g. http or https) (optional) + context: Arbitrary additional context to aid in proxy selection (optional) + """ + context = context or {} + + for item in settings.PROXY_ROUTERS: + router = import_string(item) if type(item) is str else item + if proxies := router().route(url=url, protocol=protocol, context=context): + return proxies