From 8767fd818610bd5efc48ada5aec0760d26eae467 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 31 Oct 2024 14:17:06 -0400 Subject: [PATCH 1/3] Closes #13428: Q-in-Q VLANs (#17822) * Initial work on #13428 (QinQ) * Misc cleanup; add tests for Q-in-Q fields * Address PR feedback --- docs/models/dcim/interface.md | 4 ++ docs/models/ipam/vlan.md | 8 ++++ docs/models/virtualization/vminterface.md | 4 ++ .../api/serializers_/device_components.py | 9 ++-- netbox/dcim/choices.py | 2 + netbox/dcim/filtersets.py | 6 ++- netbox/dcim/forms/common.py | 2 + netbox/dcim/forms/model_forms.py | 18 ++++++- netbox/dcim/graphql/types.py | 1 + netbox/dcim/migrations/0196_qinq_svlan.py | 28 +++++++++++ netbox/dcim/models/device_components.py | 47 +++++++++++++------ netbox/dcim/tables/devices.py | 16 ++++--- netbox/dcim/tests/test_api.py | 6 +++ netbox/dcim/tests/test_filtersets.py | 29 ++++++++++-- netbox/ipam/api/serializers_/nested.py | 8 ++++ netbox/ipam/api/serializers_/vlans.py | 7 ++- netbox/ipam/choices.py | 11 +++++ netbox/ipam/filtersets.py | 11 +++++ netbox/ipam/forms/bulk_edit.py | 16 ++++++- netbox/ipam/forms/bulk_import.py | 18 ++++++- netbox/ipam/forms/filtersets.py | 12 +++++ netbox/ipam/forms/model_forms.py | 12 ++++- netbox/ipam/graphql/types.py | 6 ++- netbox/ipam/migrations/0075_vlan_qinq.py | 30 ++++++++++++ netbox/ipam/models/vlans.py | 40 +++++++++++++++- netbox/ipam/tables/vlans.py | 9 +++- netbox/ipam/tests/test_api.py | 7 +++ netbox/ipam/tests/test_filtersets.py | 24 ++++++++++ netbox/ipam/tests/test_models.py | 21 +++++++++ netbox/templates/ipam/vlan.html | 31 ++++++++++++ netbox/templates/ipam/vlan_edit.html | 8 ++++ .../api/serializers_/virtualmachines.py | 7 +-- netbox/virtualization/forms/model_forms.py | 20 ++++++-- netbox/virtualization/graphql/types.py | 1 + ...042_vminterface_vlan_translation_policy.py | 2 - .../migrations/0043_qinq_svlan.py | 28 +++++++++++ .../virtualization/models/virtualmachines.py | 14 ------ .../virtualization/tables/virtualmachines.py | 7 +-- netbox/virtualization/tests/test_api.py | 8 ++++ .../virtualization/tests/test_filtersets.py | 24 ++++++++-- 40 files changed, 492 insertions(+), 70 deletions(-) create mode 100644 netbox/dcim/migrations/0196_qinq_svlan.py create mode 100644 netbox/ipam/migrations/0075_vlan_qinq.py create mode 100644 netbox/virtualization/migrations/0043_qinq_svlan.py diff --git a/docs/models/dcim/interface.md b/docs/models/dcim/interface.md index 869cb8510..fb7198682 100644 --- a/docs/models/dcim/interface.md +++ b/docs/models/dcim/interface.md @@ -120,6 +120,10 @@ The "native" (untagged) VLAN for the interface. Valid only when one of the above The tagged VLANs which are configured to be carried by this interface. Valid only for the "tagged" 802.1Q mode above. +### Q-in-Q SVLAN + +The assigned service VLAN (for Q-in-Q/802.1ad interfaces). + ### Wireless Role Indicates the configured role for wireless interfaces (access point or station). diff --git a/docs/models/ipam/vlan.md b/docs/models/ipam/vlan.md index 2dd5ec2d3..dc547ddbc 100644 --- a/docs/models/ipam/vlan.md +++ b/docs/models/ipam/vlan.md @@ -26,3 +26,11 @@ The user-defined functional [role](./role.md) assigned to the VLAN. ### VLAN Group or Site The [VLAN group](./vlangroup.md) or [site](../dcim/site.md) to which the VLAN is assigned. + +### Q-in-Q Role + +For VLANs which comprise a Q-in-Q/IEEE 802.1ad topology, this field indicates whether the VLAN is treated as a service or customer VLAN. + +### Q-in-Q Service VLAN + +The designated parent service VLAN for a Q-in-Q customer VLAN. This may be set only for Q-in-Q custom VLANs. diff --git a/docs/models/virtualization/vminterface.md b/docs/models/virtualization/vminterface.md index 1e022b091..4a0c474f9 100644 --- a/docs/models/virtualization/vminterface.md +++ b/docs/models/virtualization/vminterface.md @@ -53,6 +53,10 @@ The "native" (untagged) VLAN for the interface. Valid only when one of the above The tagged VLANs which are configured to be carried by this interface. Valid only for the "tagged" 802.1Q mode above. +### Q-in-Q SVLAN + +The assigned service VLAN (for Q-in-Q/802.1ad interfaces). + ### VRF The [virtual routing and forwarding](../ipam/vrf.md) instance to which this interface is assigned. diff --git a/netbox/dcim/api/serializers_/device_components.py b/netbox/dcim/api/serializers_/device_components.py index 3be19bb58..57111c2af 100644 --- a/netbox/dcim/api/serializers_/device_components.py +++ b/netbox/dcim/api/serializers_/device_components.py @@ -196,6 +196,7 @@ class InterfaceSerializer(NetBoxModelSerializer, CabledObjectSerializer, Connect required=False, many=True ) + qinq_svlan = VLANSerializer(nested=True, required=False, allow_null=True) vlan_translation_policy = VLANTranslationPolicySerializer(nested=True, required=False, allow_null=True) vrf = VRFSerializer(nested=True, required=False, allow_null=True) l2vpn_termination = L2VPNTerminationSerializer(nested=True, read_only=True, allow_null=True) @@ -223,10 +224,10 @@ class InterfaceSerializer(NetBoxModelSerializer, CabledObjectSerializer, Connect 'id', 'url', 'display_url', 'display', 'device', 'vdcs', 'module', 'name', 'label', 'type', 'enabled', 'parent', 'bridge', 'lag', 'mtu', 'mac_address', 'speed', 'duplex', 'wwn', 'mgmt_only', 'description', 'mode', 'rf_role', 'rf_channel', 'poe_mode', 'poe_type', 'rf_channel_frequency', 'rf_channel_width', - 'tx_power', 'untagged_vlan', 'tagged_vlans', 'mark_connected', 'cable', 'cable_end', 'wireless_link', - 'link_peers', 'link_peers_type', 'wireless_lans', 'vrf', 'l2vpn_termination', 'connected_endpoints', - 'connected_endpoints_type', 'connected_endpoints_reachable', 'tags', 'custom_fields', 'created', - 'last_updated', 'count_ipaddresses', 'count_fhrp_groups', '_occupied', 'vlan_translation_policy' + 'tx_power', 'untagged_vlan', 'tagged_vlans', 'qinq_svlan', 'vlan_translation_policy', 'mark_connected', + 'cable', 'cable_end', 'wireless_link', 'link_peers', 'link_peers_type', 'wireless_lans', 'vrf', + 'l2vpn_termination', 'connected_endpoints', 'connected_endpoints_type', 'connected_endpoints_reachable', + 'tags', 'custom_fields', 'created', 'last_updated', 'count_ipaddresses', 'count_fhrp_groups', '_occupied', ] brief_fields = ('id', 'url', 'display', 'device', 'name', 'description', 'cable', '_occupied') diff --git a/netbox/dcim/choices.py b/netbox/dcim/choices.py index fee587e6b..e1a99e0db 100644 --- a/netbox/dcim/choices.py +++ b/netbox/dcim/choices.py @@ -1258,11 +1258,13 @@ class InterfaceModeChoices(ChoiceSet): MODE_ACCESS = 'access' MODE_TAGGED = 'tagged' MODE_TAGGED_ALL = 'tagged-all' + MODE_Q_IN_Q = 'q-in-q' CHOICES = ( (MODE_ACCESS, _('Access')), (MODE_TAGGED, _('Tagged')), (MODE_TAGGED_ALL, _('Tagged (All)')), + (MODE_Q_IN_Q, _('Q-in-Q (802.1ad)')), ) diff --git a/netbox/dcim/filtersets.py b/netbox/dcim/filtersets.py index c11a7ef00..0371f882b 100644 --- a/netbox/dcim/filtersets.py +++ b/netbox/dcim/filtersets.py @@ -1647,7 +1647,8 @@ class CommonInterfaceFilterSet(django_filters.FilterSet): return queryset return queryset.filter( Q(untagged_vlan_id=value) | - Q(tagged_vlans=value) + Q(tagged_vlans=value) | + Q(qinq_svlan=value) ) def filter_vlan(self, queryset, name, value): @@ -1656,7 +1657,8 @@ class CommonInterfaceFilterSet(django_filters.FilterSet): return queryset return queryset.filter( Q(untagged_vlan_id__vid=value) | - Q(tagged_vlans__vid=value) + Q(tagged_vlans__vid=value) | + Q(qinq_svlan__vid=value) ) diff --git a/netbox/dcim/forms/common.py b/netbox/dcim/forms/common.py index 4341ec041..bae7fd222 100644 --- a/netbox/dcim/forms/common.py +++ b/netbox/dcim/forms/common.py @@ -37,6 +37,8 @@ class InterfaceCommonForm(forms.Form): del self.fields['vlan_group'] del self.fields['untagged_vlan'] del self.fields['tagged_vlans'] + if interface_mode != InterfaceModeChoices.MODE_Q_IN_Q: + del self.fields['qinq_svlan'] def clean(self): super().clean() diff --git a/netbox/dcim/forms/model_forms.py b/netbox/dcim/forms/model_forms.py index 1ab9f138b..2fcdbe5fd 100644 --- a/netbox/dcim/forms/model_forms.py +++ b/netbox/dcim/forms/model_forms.py @@ -7,6 +7,7 @@ from dcim.choices import * from dcim.constants import * from dcim.models import * from extras.models import ConfigTemplate +from ipam.choices import VLANQinQRoleChoices from ipam.models import ASN, IPAddress, VLAN, VLANGroup, VLANTranslationPolicy, VRF from netbox.forms import NetBoxModelForm from tenancy.forms import TenancyForm @@ -1372,6 +1373,16 @@ class InterfaceForm(InterfaceCommonForm, ModularDeviceComponentForm): 'available_on_device': '$device', } ) + qinq_svlan = DynamicModelMultipleChoiceField( + queryset=VLAN.objects.all(), + required=False, + label=_('Q-in-Q Service VLAN'), + query_params={ + 'group_id': '$vlan_group', + 'available_on_device': '$device', + 'qinq_role': VLANQinQRoleChoices.ROLE_SERVICE, + } + ) vrf = DynamicModelChoiceField( queryset=VRF.objects.all(), required=False, @@ -1396,7 +1407,10 @@ class InterfaceForm(InterfaceCommonForm, ModularDeviceComponentForm): FieldSet('vdcs', 'mtu', 'tx_power', 'enabled', 'mgmt_only', 'mark_connected', name=_('Operation')), FieldSet('parent', 'bridge', 'lag', name=_('Related Interfaces')), FieldSet('poe_mode', 'poe_type', name=_('PoE')), - FieldSet('mode', 'vlan_group', 'untagged_vlan', 'tagged_vlans', 'vlan_translation_policy', name=_('802.1Q Switching')), + FieldSet( + 'mode', 'vlan_group', 'untagged_vlan', 'tagged_vlans', 'qinq_svlan', 'vlan_translation_policy', + name=_('802.1Q Switching') + ), FieldSet( 'rf_role', 'rf_channel', 'rf_channel_frequency', 'rf_channel_width', 'wireless_lan_group', 'wireless_lans', name=_('Wireless') @@ -1409,7 +1423,7 @@ class InterfaceForm(InterfaceCommonForm, ModularDeviceComponentForm): 'device', 'module', 'vdcs', 'name', 'label', 'type', 'speed', 'duplex', 'enabled', 'parent', 'bridge', 'lag', 'mac_address', 'wwn', 'mtu', 'mgmt_only', 'mark_connected', 'description', 'poe_mode', 'poe_type', 'mode', 'rf_role', 'rf_channel', 'rf_channel_frequency', 'rf_channel_width', 'tx_power', 'wireless_lans', - 'untagged_vlan', 'tagged_vlans', 'vrf', 'tags', 'vlan_translation_policy', + 'untagged_vlan', 'tagged_vlans', 'qinq_svlan', 'vlan_translation_policy', 'vrf', 'tags', ] widgets = { 'speed': NumberWithOptions( diff --git a/netbox/dcim/graphql/types.py b/netbox/dcim/graphql/types.py index da7a0af25..5965fdcec 100644 --- a/netbox/dcim/graphql/types.py +++ b/netbox/dcim/graphql/types.py @@ -385,6 +385,7 @@ class InterfaceType(IPAddressesMixin, ModularComponentType, CabledObjectMixin, P wireless_link: Annotated["WirelessLinkType", strawberry.lazy('wireless.graphql.types')] | None untagged_vlan: Annotated["VLANType", strawberry.lazy('ipam.graphql.types')] | None vrf: Annotated["VRFType", strawberry.lazy('ipam.graphql.types')] | None + qinq_svlan: Annotated["VLANType", strawberry.lazy('ipam.graphql.types')] | None vlan_translation_policy: Annotated["VLANTranslationPolicyType", strawberry.lazy('ipam.graphql.types')] | None vdcs: List[Annotated["VirtualDeviceContextType", strawberry.lazy('dcim.graphql.types')]] diff --git a/netbox/dcim/migrations/0196_qinq_svlan.py b/netbox/dcim/migrations/0196_qinq_svlan.py new file mode 100644 index 000000000..9012d74f3 --- /dev/null +++ b/netbox/dcim/migrations/0196_qinq_svlan.py @@ -0,0 +1,28 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dcim', '0195_interface_vlan_translation_policy'), + ('ipam', '0075_vlan_qinq'), + ] + + operations = [ + migrations.AddField( + model_name='interface', + name='qinq_svlan', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)ss_svlan', to='ipam.vlan'), + ), + migrations.AlterField( + model_name='interface', + name='tagged_vlans', + field=models.ManyToManyField(blank=True, related_name='%(class)ss_as_tagged', to='ipam.vlan'), + ), + migrations.AlterField( + model_name='interface', + name='untagged_vlan', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)ss_as_untagged', to='ipam.vlan'), + ), + ] diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 14f4120b5..36fd02add 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -547,17 +547,48 @@ class BaseInterface(models.Model): blank=True, verbose_name=_('bridge interface') ) + untagged_vlan = models.ForeignKey( + to='ipam.VLAN', + on_delete=models.SET_NULL, + related_name='%(class)ss_as_untagged', + null=True, + blank=True, + verbose_name=_('untagged VLAN') + ) + tagged_vlans = models.ManyToManyField( + to='ipam.VLAN', + related_name='%(class)ss_as_tagged', + blank=True, + verbose_name=_('tagged VLANs') + ) + qinq_svlan = models.ForeignKey( + to='ipam.VLAN', + on_delete=models.SET_NULL, + related_name='%(class)ss_svlan', + null=True, + blank=True, + verbose_name=_('Q-in-Q SVLAN') + ) vlan_translation_policy = models.ForeignKey( to='ipam.VLANTranslationPolicy', on_delete=models.PROTECT, null=True, blank=True, - verbose_name=_('VLAN Translation Policy'), + verbose_name=_('VLAN Translation Policy') ) class Meta: abstract = True + def clean(self): + super().clean() + + # SVLAN can be defined only for Q-in-Q interfaces + if self.qinq_svlan and self.mode != InterfaceModeChoices.MODE_Q_IN_Q: + raise ValidationError({ + 'qinq_svlan': _("Only Q-in-Q interfaces may specify a service VLAN.") + }) + def save(self, *args, **kwargs): # Remove untagged VLAN assignment for non-802.1Q interfaces @@ -697,20 +728,6 @@ class Interface(ModularComponentModel, BaseInterface, CabledObjectModel, PathEnd blank=True, verbose_name=_('wireless LANs') ) - untagged_vlan = models.ForeignKey( - to='ipam.VLAN', - on_delete=models.SET_NULL, - related_name='interfaces_as_untagged', - null=True, - blank=True, - verbose_name=_('untagged VLAN') - ) - tagged_vlans = models.ManyToManyField( - to='ipam.VLAN', - related_name='interfaces_as_tagged', - blank=True, - verbose_name=_('tagged VLANs') - ) vrf = models.ForeignKey( to='ipam.VRF', on_delete=models.SET_NULL, diff --git a/netbox/dcim/tables/devices.py b/netbox/dcim/tables/devices.py index b39a2b87f..fed33401c 100644 --- a/netbox/dcim/tables/devices.py +++ b/netbox/dcim/tables/devices.py @@ -585,6 +585,10 @@ class BaseInterfaceTable(NetBoxTable): orderable=False, verbose_name=_('Tagged VLANs') ) + qinq_svlan = tables.Column( + verbose_name=_('Q-in-Q SVLAN'), + linkify=True + ) def value_ip_addresses(self, value): return ",".join([str(obj.address) for obj in value.all()]) @@ -635,11 +639,11 @@ class InterfaceTable(ModularDeviceComponentTable, BaseInterfaceTable, PathEndpoi model = models.Interface fields = ( 'pk', 'id', 'name', 'device', 'module_bay', 'module', 'label', 'enabled', 'type', 'mgmt_only', 'mtu', - 'speed', 'speed_formatted', 'duplex', 'mode', 'mac_address', 'wwn', 'poe_mode', 'poe_type', 'rf_role', 'rf_channel', - 'rf_channel_frequency', 'rf_channel_width', 'tx_power', 'description', 'mark_connected', 'cable', - 'cable_color', 'wireless_link', 'wireless_lans', 'link_peer', 'connection', 'tags', 'vdcs', 'vrf', 'l2vpn', - 'tunnel', 'ip_addresses', 'fhrp_groups', 'untagged_vlan', 'tagged_vlans', 'inventory_items', 'created', - 'last_updated', + 'speed', 'speed_formatted', 'duplex', 'mode', 'mac_address', 'wwn', 'poe_mode', 'poe_type', 'rf_role', + 'rf_channel', 'rf_channel_frequency', 'rf_channel_width', 'tx_power', 'description', 'mark_connected', + 'cable', 'cable_color', 'wireless_link', 'wireless_lans', 'link_peer', 'connection', 'tags', 'vdcs', 'vrf', + 'l2vpn', 'tunnel', 'ip_addresses', 'fhrp_groups', 'untagged_vlan', 'tagged_vlans', 'qinq_svlan', + 'inventory_items', 'created', 'last_updated', ) default_columns = ('pk', 'name', 'device', 'label', 'enabled', 'type', 'description') @@ -676,7 +680,7 @@ class DeviceInterfaceTable(InterfaceTable): 'mgmt_only', 'mtu', 'mode', 'mac_address', 'wwn', 'rf_role', 'rf_channel', 'rf_channel_frequency', 'rf_channel_width', 'tx_power', 'description', 'mark_connected', 'cable', 'cable_color', 'wireless_link', 'wireless_lans', 'link_peer', 'connection', 'tags', 'vdcs', 'vrf', 'l2vpn', 'tunnel', 'ip_addresses', - 'fhrp_groups', 'untagged_vlan', 'tagged_vlans', 'actions', + 'fhrp_groups', 'untagged_vlan', 'tagged_vlans', 'qinq_svlan', 'actions', ) default_columns = ( 'pk', 'name', 'label', 'enabled', 'type', 'parent', 'lag', 'mtu', 'mode', 'description', 'ip_addresses', diff --git a/netbox/dcim/tests/test_api.py b/netbox/dcim/tests/test_api.py index 1b460cd59..f78722b67 100644 --- a/netbox/dcim/tests/test_api.py +++ b/netbox/dcim/tests/test_api.py @@ -7,6 +7,7 @@ from dcim.choices import * from dcim.constants import * from dcim.models import * from extras.models import ConfigTemplate +from ipam.choices import VLANQinQRoleChoices from ipam.models import ASN, RIR, VLAN, VRF from netbox.api.serializers import GenericObjectSerializer from tenancy.models import Tenant @@ -1618,6 +1619,7 @@ class InterfaceTest(Mixins.ComponentTraceMixin, APIViewTestCases.APIViewTestCase VLAN(name='VLAN 1', vid=1), VLAN(name='VLAN 2', vid=2), VLAN(name='VLAN 3', vid=3), + VLAN(name='SVLAN 1', vid=1001, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE), ) VLAN.objects.bulk_create(vlans) @@ -1676,18 +1678,22 @@ class InterfaceTest(Mixins.ComponentTraceMixin, APIViewTestCases.APIViewTestCase 'vdcs': [vdcs[1].pk], 'name': 'Interface 7', 'type': InterfaceTypeChoices.TYPE_80211A, + 'mode': InterfaceModeChoices.MODE_Q_IN_Q, 'tx_power': 10, 'wireless_lans': [wireless_lans[0].pk, wireless_lans[1].pk], 'rf_channel': WirelessChannelChoices.CHANNEL_5G_32, + 'qinq_svlan': vlans[3].pk, }, { 'device': device.pk, 'vdcs': [vdcs[1].pk], 'name': 'Interface 8', 'type': InterfaceTypeChoices.TYPE_80211A, + 'mode': InterfaceModeChoices.MODE_Q_IN_Q, 'tx_power': 10, 'wireless_lans': [wireless_lans[0].pk, wireless_lans[1].pk], 'rf_channel': "", + 'qinq_svlan': vlans[3].pk, }, ] diff --git a/netbox/dcim/tests/test_filtersets.py b/netbox/dcim/tests/test_filtersets.py index c5ca01db2..993c2fa4e 100644 --- a/netbox/dcim/tests/test_filtersets.py +++ b/netbox/dcim/tests/test_filtersets.py @@ -4,7 +4,8 @@ from circuits.models import Circuit, CircuitTermination, CircuitType, Provider from dcim.choices import * from dcim.filtersets import * from dcim.models import * -from ipam.models import ASN, IPAddress, RIR, VLANTranslationPolicy, VRF +from ipam.choices import VLANQinQRoleChoices +from ipam.models import ASN, IPAddress, RIR, VLAN, VLANTranslationPolicy, VRF from netbox.choices import ColorChoices, WeightUnitChoices from tenancy.models import Tenant, TenantGroup from users.models import User @@ -3520,7 +3521,7 @@ class PowerOutletTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedF class InterfaceTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedFilterSetTests): queryset = Interface.objects.all() filterset = InterfaceFilterSet - ignore_fields = ('tagged_vlans', 'untagged_vlan', 'vdcs') + ignore_fields = ('tagged_vlans', 'untagged_vlan', 'qinq_svlan', 'vdcs') @classmethod def setUpTestData(cls): @@ -3669,6 +3670,13 @@ class InterfaceTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedFil ) VirtualDeviceContext.objects.bulk_create(vdcs) + vlans = ( + VLAN(name='SVLAN 1', vid=1001, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE), + VLAN(name='SVLAN 2', vid=1002, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE), + VLAN(name='SVLAN 3', vid=1003, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE), + ) + VLAN.objects.bulk_create(vlans) + vlan_translation_policies = ( VLANTranslationPolicy(name='Policy 1'), VLANTranslationPolicy(name='Policy 2'), @@ -3753,6 +3761,8 @@ class InterfaceTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedFil duplex='full', poe_mode=InterfacePoEModeChoices.MODE_PD, poe_type=InterfacePoETypeChoices.TYPE_2_8023AT, + mode=InterfaceModeChoices.MODE_Q_IN_Q, + qinq_svlan=vlans[0], vlan_translation_policy=vlan_translation_policies[1], ), Interface( @@ -3762,7 +3772,9 @@ class InterfaceTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedFil type=InterfaceTypeChoices.TYPE_OTHER, enabled=True, mgmt_only=True, - tx_power=40 + tx_power=40, + mode=InterfaceModeChoices.MODE_Q_IN_Q, + qinq_svlan=vlans[1] ), Interface( device=devices[4], @@ -3771,7 +3783,9 @@ class InterfaceTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedFil type=InterfaceTypeChoices.TYPE_OTHER, enabled=False, mgmt_only=False, - tx_power=40 + tx_power=40, + mode=InterfaceModeChoices.MODE_Q_IN_Q, + qinq_svlan=vlans[2] ), Interface( device=devices[4], @@ -4027,6 +4041,13 @@ class InterfaceTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedFil params = {'vdc_identifier': vdc.values_list('identifier', flat=True)} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3) + def test_vlan(self): + vlan = VLAN.objects.filter(qinq_role=VLANQinQRoleChoices.ROLE_SERVICE).first() + params = {'vlan_id': vlan.pk} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + params = {'vlan': vlan.vid} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + def test_vlan_translation_policy(self): vlan_translation_policies = VLANTranslationPolicy.objects.all()[:2] params = {'vlan_translation_policy_id': [vlan_translation_policies[0].pk, vlan_translation_policies[1].pk]} diff --git a/netbox/ipam/api/serializers_/nested.py b/netbox/ipam/api/serializers_/nested.py index 5297565bb..b56b15984 100644 --- a/netbox/ipam/api/serializers_/nested.py +++ b/netbox/ipam/api/serializers_/nested.py @@ -6,6 +6,7 @@ from ..field_serializers import IPAddressField __all__ = ( 'NestedIPAddressSerializer', + 'NestedVLANSerializer', ) @@ -16,3 +17,10 @@ class NestedIPAddressSerializer(WritableNestedSerializer): class Meta: model = models.IPAddress fields = ['id', 'url', 'display_url', 'display', 'family', 'address'] + + +class NestedVLANSerializer(WritableNestedSerializer): + + class Meta: + model = models.VLAN + fields = ['id', 'url', 'display', 'vid', 'name', 'description'] diff --git a/netbox/ipam/api/serializers_/vlans.py b/netbox/ipam/api/serializers_/vlans.py index ee06357a5..05fdd5813 100644 --- a/netbox/ipam/api/serializers_/vlans.py +++ b/netbox/ipam/api/serializers_/vlans.py @@ -11,6 +11,7 @@ from netbox.api.serializers import NetBoxModelSerializer from tenancy.api.serializers_.tenants import TenantSerializer from utilities.api import get_serializer_for_model from vpn.api.serializers_.l2vpn import L2VPNTerminationSerializer +from .nested import NestedVLANSerializer from .roles import RoleSerializer __all__ = ( @@ -64,6 +65,8 @@ class VLANSerializer(NetBoxModelSerializer): tenant = TenantSerializer(nested=True, required=False, allow_null=True) status = ChoiceField(choices=VLANStatusChoices, required=False) role = RoleSerializer(nested=True, required=False, allow_null=True) + qinq_role = ChoiceField(choices=VLANQinQRoleChoices, required=False) + qinq_svlan = NestedVLANSerializer(required=False, allow_null=True, default=None) l2vpn_termination = L2VPNTerminationSerializer(nested=True, read_only=True, allow_null=True) # Related object counts @@ -73,8 +76,8 @@ class VLANSerializer(NetBoxModelSerializer): model = VLAN fields = [ 'id', 'url', 'display_url', 'display', 'site', 'group', 'vid', 'name', 'tenant', 'status', 'role', - 'description', 'comments', 'l2vpn_termination', 'tags', 'custom_fields', 'created', 'last_updated', - 'prefix_count', + 'description', 'qinq_role', 'qinq_svlan', 'comments', 'l2vpn_termination', 'tags', 'custom_fields', + 'created', 'last_updated', 'prefix_count', ] brief_fields = ('id', 'url', 'display', 'vid', 'name', 'description') diff --git a/netbox/ipam/choices.py b/netbox/ipam/choices.py index 017fd0430..4d9c0bdd4 100644 --- a/netbox/ipam/choices.py +++ b/netbox/ipam/choices.py @@ -157,6 +157,17 @@ class VLANStatusChoices(ChoiceSet): ] +class VLANQinQRoleChoices(ChoiceSet): + + ROLE_SERVICE = 's-vlan' + ROLE_CUSTOMER = 'c-vlan' + + CHOICES = [ + (ROLE_SERVICE, _('Service'), 'blue'), + (ROLE_CUSTOMER, _('Customer'), 'orange'), + ] + + # # Services # diff --git a/netbox/ipam/filtersets.py b/netbox/ipam/filtersets.py index 017a34ac4..88c869a50 100644 --- a/netbox/ipam/filtersets.py +++ b/netbox/ipam/filtersets.py @@ -1041,6 +1041,17 @@ class VLANFilterSet(NetBoxModelFilterSet, TenancyFilterSet): queryset=VirtualMachine.objects.all(), method='get_for_virtualmachine' ) + qinq_role = django_filters.MultipleChoiceFilter( + choices=VLANQinQRoleChoices + ) + qinq_svlan_id = django_filters.ModelMultipleChoiceFilter( + queryset=VLAN.objects.all(), + label=_('Q-in-Q SVLAN (ID)'), + ) + qinq_svlan_vid = MultiValueNumberFilter( + field_name='qinq_svlan__vid', + label=_('Q-in-Q SVLAN number (1-4094)'), + ) l2vpn_id = django_filters.ModelMultipleChoiceFilter( field_name='l2vpn_terminations__l2vpn', queryset=L2VPN.objects.all(), diff --git a/netbox/ipam/forms/bulk_edit.py b/netbox/ipam/forms/bulk_edit.py index 49a79623c..c323a41c1 100644 --- a/netbox/ipam/forms/bulk_edit.py +++ b/netbox/ipam/forms/bulk_edit.py @@ -527,15 +527,29 @@ class VLANBulkEditForm(NetBoxModelBulkEditForm): max_length=200, required=False ) + qinq_role = forms.ChoiceField( + label=_('Q-in-Q role'), + choices=add_blank_choice(VLANQinQRoleChoices), + required=False + ) + qinq_svlan = DynamicModelChoiceField( + label=_('Q-in-Q SVLAN'), + queryset=VLAN.objects.all(), + required=False, + query_params={ + 'qinq_role': VLANQinQRoleChoices.ROLE_SERVICE, + } + ) comments = CommentField() model = VLAN fieldsets = ( FieldSet('status', 'role', 'tenant', 'description'), + FieldSet('qinq_role', 'qinq_svlan', name=_('Q-in-Q')), FieldSet('region', 'site_group', 'site', 'group', name=_('Site & Group')), ) nullable_fields = ( - 'site', 'group', 'tenant', 'role', 'description', 'comments', + 'site', 'group', 'tenant', 'role', 'description', 'qinq_role', 'qinq_svlan', 'comments', ) diff --git a/netbox/ipam/forms/bulk_import.py b/netbox/ipam/forms/bulk_import.py index cd34a6d84..3be4ccc59 100644 --- a/netbox/ipam/forms/bulk_import.py +++ b/netbox/ipam/forms/bulk_import.py @@ -461,10 +461,26 @@ class VLANImportForm(NetBoxModelImportForm): to_field_name='name', help_text=_('Functional role') ) + qinq_role = CSVChoiceField( + label=_('Q-in-Q role'), + choices=VLANStatusChoices, + required=False, + help_text=_('Operational status') + ) + qinq_svlan = CSVModelChoiceField( + label=_('Q-in-Q SVLAN'), + queryset=VLAN.objects.all(), + required=False, + to_field_name='vid', + help_text=_("Service VLAN (for Q-in-Q/802.1ad customer VLANs)") + ) class Meta: model = VLAN - fields = ('site', 'group', 'vid', 'name', 'tenant', 'status', 'role', 'description', 'comments', 'tags') + fields = ( + 'site', 'group', 'vid', 'name', 'tenant', 'status', 'role', 'description', 'qinq_role', 'qinq_svlan', + 'comments', 'tags', + ) class VLANTranslationPolicyImportForm(NetBoxModelImportForm): diff --git a/netbox/ipam/forms/filtersets.py b/netbox/ipam/forms/filtersets.py index b9bee6d97..3f951512b 100644 --- a/netbox/ipam/forms/filtersets.py +++ b/netbox/ipam/forms/filtersets.py @@ -506,6 +506,7 @@ class VLANFilterForm(TenancyFilterForm, NetBoxModelFilterSetForm): FieldSet('q', 'filter_id', 'tag'), FieldSet('region_id', 'site_group_id', 'site_id', name=_('Location')), FieldSet('group_id', 'status', 'role_id', 'vid', 'l2vpn_id', name=_('Attributes')), + FieldSet('qinq_role', 'qinq_svlan_id', name=_('Q-in-Q/802.1ad')), FieldSet('tenant_group_id', 'tenant_id', name=_('Tenant')), ) selector_fields = ('filter_id', 'q', 'site_id') @@ -552,6 +553,17 @@ class VLANFilterForm(TenancyFilterForm, NetBoxModelFilterSetForm): required=False, label=_('VLAN ID') ) + qinq_role = forms.MultipleChoiceField( + label=_('Q-in-Q role'), + choices=VLANQinQRoleChoices, + required=False + ) + qinq_svlan_id = DynamicModelMultipleChoiceField( + queryset=VLAN.objects.all(), + required=False, + null_option='None', + label=_('Q-in-Q SVLAN') + ) l2vpn_id = DynamicModelMultipleChoiceField( queryset=L2VPN.objects.all(), required=False, diff --git a/netbox/ipam/forms/model_forms.py b/netbox/ipam/forms/model_forms.py index 629c1a481..3d0cd3dd1 100644 --- a/netbox/ipam/forms/model_forms.py +++ b/netbox/ipam/forms/model_forms.py @@ -683,13 +683,21 @@ class VLANForm(TenancyForm, NetBoxModelForm): queryset=Role.objects.all(), required=False ) + qinq_svlan = DynamicModelChoiceField( + label=_('Q-in-Q SVLAN'), + queryset=VLAN.objects.all(), + required=False, + query_params={ + 'qinq_role': VLANQinQRoleChoices.ROLE_SERVICE, + } + ) comments = CommentField() class Meta: model = VLAN fields = [ - 'site', 'group', 'vid', 'name', 'status', 'role', 'tenant_group', 'tenant', 'description', 'comments', - 'tags', + 'site', 'group', 'vid', 'name', 'status', 'role', 'tenant_group', 'tenant', 'qinq_role', 'qinq_svlan', + 'description', 'comments', 'tags', ] diff --git a/netbox/ipam/graphql/types.py b/netbox/ipam/graphql/types.py index ef50138c2..2ef63cf0c 100644 --- a/netbox/ipam/graphql/types.py +++ b/netbox/ipam/graphql/types.py @@ -236,7 +236,7 @@ class ServiceTemplateType(NetBoxObjectType): @strawberry_django.type( models.VLAN, - fields='__all__', + exclude=('qinq_svlan',), filters=VLANFilter ) class VLANType(NetBoxObjectType): @@ -252,6 +252,10 @@ class VLANType(NetBoxObjectType): interfaces_as_tagged: List[Annotated["InterfaceType", strawberry.lazy('dcim.graphql.types')]] vminterfaces_as_tagged: List[Annotated["VMInterfaceType", strawberry.lazy('virtualization.graphql.types')]] + @strawberry_django.field + def qinq_svlan(self) -> Annotated["VLANType", strawberry.lazy('ipam.graphql.types')] | None: + return self.qinq_svlan + @strawberry_django.type( models.VLANGroup, diff --git a/netbox/ipam/migrations/0075_vlan_qinq.py b/netbox/ipam/migrations/0075_vlan_qinq.py new file mode 100644 index 000000000..8a3b8a39a --- /dev/null +++ b/netbox/ipam/migrations/0075_vlan_qinq.py @@ -0,0 +1,30 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ipam', '0074_vlantranslationpolicy_vlantranslationrule'), + ] + + operations = [ + migrations.AddField( + model_name='vlan', + name='qinq_role', + field=models.CharField(blank=True, max_length=50, null=True), + ), + migrations.AddField( + model_name='vlan', + name='qinq_svlan', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='qinq_cvlans', to='ipam.vlan'), + ), + migrations.AddConstraint( + model_name='vlan', + constraint=models.UniqueConstraint(fields=('qinq_svlan', 'vid'), name='ipam_vlan_unique_qinq_svlan_vid'), + ), + migrations.AddConstraint( + model_name='vlan', + constraint=models.UniqueConstraint(fields=('qinq_svlan', 'name'), name='ipam_vlan_unique_qinq_svlan_name'), + ), + ] diff --git a/netbox/ipam/models/vlans.py b/netbox/ipam/models/vlans.py index ff8394839..7832cfc67 100644 --- a/netbox/ipam/models/vlans.py +++ b/netbox/ipam/models/vlans.py @@ -204,6 +204,21 @@ class VLAN(PrimaryModel): null=True, help_text=_("The primary function of this VLAN") ) + qinq_svlan = models.ForeignKey( + to='self', + on_delete=models.PROTECT, + related_name='qinq_cvlans', + blank=True, + null=True + ) + qinq_role = models.CharField( + verbose_name=_('Q-in-Q role'), + max_length=50, + choices=VLANQinQRoleChoices, + blank=True, + null=True, + help_text=_("Customer/service VLAN designation (for Q-in-Q/IEEE 802.1ad)") + ) l2vpn_terminations = GenericRelation( to='vpn.L2VPNTermination', content_type_field='assigned_object_type', @@ -214,7 +229,7 @@ class VLAN(PrimaryModel): objects = VLANQuerySet.as_manager() clone_fields = [ - 'site', 'group', 'tenant', 'status', 'role', 'description', + 'site', 'group', 'tenant', 'status', 'role', 'description', 'qinq_role', 'qinq_svlan', ] class Meta: @@ -228,6 +243,14 @@ class VLAN(PrimaryModel): fields=('group', 'name'), name='%(app_label)s_%(class)s_unique_group_name' ), + models.UniqueConstraint( + fields=('qinq_svlan', 'vid'), + name='%(app_label)s_%(class)s_unique_qinq_svlan_vid' + ), + models.UniqueConstraint( + fields=('qinq_svlan', 'name'), + name='%(app_label)s_%(class)s_unique_qinq_svlan_name' + ), ) verbose_name = _('VLAN') verbose_name_plural = _('VLANs') @@ -255,9 +278,24 @@ class VLAN(PrimaryModel): ).format(ranges=ranges_to_string(self.group.vid_ranges), group=self.group) }) + # Only Q-in-Q customer VLANs may be assigned to a service VLAN + if self.qinq_svlan and self.qinq_role != VLANQinQRoleChoices.ROLE_CUSTOMER: + raise ValidationError({ + 'qinq_svlan': _("Only Q-in-Q customer VLANs maybe assigned to a service VLAN.") + }) + + # A Q-in-Q customer VLAN must be assigned to a service VLAN + if self.qinq_role == VLANQinQRoleChoices.ROLE_CUSTOMER and not self.qinq_svlan: + raise ValidationError({ + 'qinq_role': _("A Q-in-Q customer VLAN must be assigned to a service VLAN.") + }) + def get_status_color(self): return VLANStatusChoices.colors.get(self.status) + def get_qinq_role_color(self): + return VLANQinQRoleChoices.colors.get(self.qinq_role) + def get_interfaces(self): # Return all device interfaces assigned to this VLAN return Interface.objects.filter( diff --git a/netbox/ipam/tables/vlans.py b/netbox/ipam/tables/vlans.py index 1a06bb700..d34ff5f45 100644 --- a/netbox/ipam/tables/vlans.py +++ b/netbox/ipam/tables/vlans.py @@ -132,6 +132,13 @@ class VLANTable(TenancyColumnsMixin, NetBoxTable): verbose_name=_('Role'), linkify=True ) + qinq_role = columns.ChoiceFieldColumn( + verbose_name=_('Q-in-Q role') + ) + qinq_svlan = tables.Column( + verbose_name=_('Q-in-Q SVLAN'), + linkify=True + ) l2vpn = tables.Column( accessor=tables.A('l2vpn_termination__l2vpn'), linkify=True, @@ -154,7 +161,7 @@ class VLANTable(TenancyColumnsMixin, NetBoxTable): model = VLAN fields = ( 'pk', 'id', 'vid', 'name', 'site', 'group', 'prefixes', 'tenant', 'tenant_group', 'status', 'role', - 'description', 'comments', 'tags', 'l2vpn', 'created', 'last_updated', + 'qinq_role', 'qinq_svlan', 'description', 'comments', 'tags', 'l2vpn', 'created', 'last_updated', ) default_columns = ('pk', 'vid', 'name', 'site', 'group', 'prefixes', 'tenant', 'status', 'role', 'description') row_attrs = { diff --git a/netbox/ipam/tests/test_api.py b/netbox/ipam/tests/test_api.py index cd3e47342..4e8456e5a 100644 --- a/netbox/ipam/tests/test_api.py +++ b/netbox/ipam/tests/test_api.py @@ -980,6 +980,7 @@ class VLANTest(APIViewTestCases.APIViewTestCase): VLAN(name='VLAN 1', vid=1, group=vlan_groups[0]), VLAN(name='VLAN 2', vid=2, group=vlan_groups[0]), VLAN(name='VLAN 3', vid=3, group=vlan_groups[0]), + VLAN(name='SVLAN 1', vid=1001, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE), ) VLAN.objects.bulk_create(vlans) @@ -999,6 +1000,12 @@ class VLANTest(APIViewTestCases.APIViewTestCase): 'name': 'VLAN 6', 'group': vlan_groups[1].pk, }, + { + 'vid': 2001, + 'name': 'CVLAN 1', + 'qinq_role': VLANQinQRoleChoices.ROLE_CUSTOMER, + 'qinq_svlan': vlans[3].pk, + }, ] def test_delete_vlan_with_prefix(self): diff --git a/netbox/ipam/tests/test_filtersets.py b/netbox/ipam/tests/test_filtersets.py index b402a8426..f651c970d 100644 --- a/netbox/ipam/tests/test_filtersets.py +++ b/netbox/ipam/tests/test_filtersets.py @@ -1630,6 +1630,7 @@ class VLANTestCase(TestCase, ChangeLoggedFilterSetTests): Site(name='Site 4', slug='site-4', region=regions[0], group=site_groups[0]), Site(name='Site 5', slug='site-5', region=regions[1], group=site_groups[1]), Site(name='Site 6', slug='site-6', region=regions[2], group=site_groups[2]), + Site(name='Site 7', slug='site-7'), ) Site.objects.bulk_create(sites) @@ -1784,9 +1785,21 @@ class VLANTestCase(TestCase, ChangeLoggedFilterSetTests): # Create one globally available VLAN VLAN(vid=1000, name='Global VLAN'), + + # Create some Q-in-Q service VLANs + VLAN(vid=2001, name='SVLAN 1', site=sites[6], qinq_role=VLANQinQRoleChoices.ROLE_SERVICE), + VLAN(vid=2002, name='SVLAN 2', site=sites[6], qinq_role=VLANQinQRoleChoices.ROLE_SERVICE), + VLAN(vid=2003, name='SVLAN 3', site=sites[6], qinq_role=VLANQinQRoleChoices.ROLE_SERVICE), ) VLAN.objects.bulk_create(vlans) + # Create Q-in-Q customer VLANs + VLAN.objects.bulk_create([ + VLAN(vid=3001, name='CVLAN 1', site=sites[6], qinq_svlan=vlans[29], qinq_role=VLANQinQRoleChoices.ROLE_CUSTOMER), + VLAN(vid=3002, name='CVLAN 2', site=sites[6], qinq_svlan=vlans[30], qinq_role=VLANQinQRoleChoices.ROLE_CUSTOMER), + VLAN(vid=3003, name='CVLAN 3', site=sites[6], qinq_svlan=vlans[31], qinq_role=VLANQinQRoleChoices.ROLE_CUSTOMER), + ]) + # Assign VLANs to device interfaces interfaces[0].untagged_vlan = vlans[0] interfaces[0].tagged_vlans.add(vlans[1]) @@ -1897,6 +1910,17 @@ class VLANTestCase(TestCase, ChangeLoggedFilterSetTests): params = {'vminterface_id': vminterface_id} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + def test_qinq_role(self): + params = {'qinq_role': [VLANQinQRoleChoices.ROLE_SERVICE, VLANQinQRoleChoices.ROLE_CUSTOMER]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 6) + + def test_qinq_svlan(self): + vlans = VLAN.objects.filter(qinq_role=VLANQinQRoleChoices.ROLE_SERVICE)[:2] + params = {'qinq_svlan_id': [vlans[0].pk, vlans[1].pk]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + params = {'qinq_svlan_vid': [vlans[0].vid, vlans[1].vid]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + class VLANTranslationPolicyTestCase(TestCase, ChangeLoggedFilterSetTests): queryset = VLANTranslationPolicy.objects.all() diff --git a/netbox/ipam/tests/test_models.py b/netbox/ipam/tests/test_models.py index d14fa0657..917b50f33 100644 --- a/netbox/ipam/tests/test_models.py +++ b/netbox/ipam/tests/test_models.py @@ -586,3 +586,24 @@ class TestVLANGroup(TestCase): vlangroup.vid_ranges = string_to_ranges('2-2') vlangroup.full_clean() vlangroup.save() + + +class TestVLAN(TestCase): + + @classmethod + def setUpTestData(cls): + VLAN.objects.bulk_create(( + VLAN(name='VLAN 1', vid=1, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE), + )) + + def test_qinq_role(self): + svlan = VLAN.objects.filter(qinq_role=VLANQinQRoleChoices.ROLE_SERVICE).first() + + vlan = VLAN( + name='VLAN X', + vid=999, + qinq_role=VLANQinQRoleChoices.ROLE_SERVICE, + qinq_svlan=svlan + ) + with self.assertRaises(ValidationError): + vlan.full_clean() diff --git a/netbox/templates/ipam/vlan.html b/netbox/templates/ipam/vlan.html index 95a4f7856..a10a1439a 100644 --- a/netbox/templates/ipam/vlan.html +++ b/netbox/templates/ipam/vlan.html @@ -62,6 +62,22 @@ {% trans "Description" %} {{ object.description|placeholder }} + + {% trans "Q-in-Q Role" %} + + {% if object.qinq_role %} + {% badge object.get_qinq_role_display bg_color=object.get_qinq_role_color %} + {% else %} + {{ ''|placeholder }} + {% endif %} + + + {% if object.qinq_role == 'c-vlan' %} + + {% trans "Q-in-Q SVLAN" %} + {{ object.qinq_svlan|linkify|placeholder }} + + {% endif %} {% trans "L2VPN" %} {{ object.l2vpn_termination.l2vpn|linkify|placeholder }} @@ -92,6 +108,21 @@ {% htmx_table 'ipam:prefix_list' vlan_id=object.pk %} + {% if object.qinq_role == 's-vlan' %} +
+

+ {% trans "Customer VLANs" %} + {% if perms.ipam.add_vlan %} +
+ + {% trans "Add a VLAN" %} + +
+ {% endif %} +

+ {% htmx_table 'ipam:vlan_list' qinq_svlan_id=object.pk %} +
+ {% endif %} {% plugin_full_width_page object %} diff --git a/netbox/templates/ipam/vlan_edit.html b/netbox/templates/ipam/vlan_edit.html index 814fc6b78..885844580 100644 --- a/netbox/templates/ipam/vlan_edit.html +++ b/netbox/templates/ipam/vlan_edit.html @@ -17,6 +17,14 @@ {% render_field form.tags %} +
+
+

{% trans "Q-in-Q (802.1ad)" %}

+
+ {% render_field form.qinq_role %} + {% render_field form.qinq_svlan %} +
+

{% trans "Tenancy" %}

diff --git a/netbox/virtualization/api/serializers_/virtualmachines.py b/netbox/virtualization/api/serializers_/virtualmachines.py index 2c00cac96..9b7000def 100644 --- a/netbox/virtualization/api/serializers_/virtualmachines.py +++ b/netbox/virtualization/api/serializers_/virtualmachines.py @@ -89,6 +89,7 @@ class VMInterfaceSerializer(NetBoxModelSerializer): required=False, many=True ) + qinq_svlan = VLANSerializer(nested=True, required=False, allow_null=True) vlan_translation_policy = VLANTranslationPolicySerializer(nested=True, required=False, allow_null=True) vrf = VRFSerializer(nested=True, required=False, allow_null=True) l2vpn_termination = L2VPNTerminationSerializer(nested=True, read_only=True, allow_null=True) @@ -104,9 +105,9 @@ class VMInterfaceSerializer(NetBoxModelSerializer): model = VMInterface fields = [ 'id', 'url', 'display_url', 'display', 'virtual_machine', 'name', 'enabled', 'parent', 'bridge', 'mtu', - 'mac_address', 'description', 'mode', 'untagged_vlan', 'tagged_vlans', 'vrf', 'l2vpn_termination', - 'tags', 'custom_fields', 'created', 'last_updated', 'count_ipaddresses', 'count_fhrp_groups', - 'vlan_translation_policy', + 'mac_address', 'description', 'mode', 'untagged_vlan', 'tagged_vlans', 'qinq_svlan', + 'vlan_translation_policy', 'vrf', 'l2vpn_termination', 'tags', 'custom_fields', 'created', 'last_updated', + 'count_ipaddresses', 'count_fhrp_groups', ] brief_fields = ('id', 'url', 'display', 'virtual_machine', 'name', 'description') diff --git a/netbox/virtualization/forms/model_forms.py b/netbox/virtualization/forms/model_forms.py index 5971fc894..4527e7f4c 100644 --- a/netbox/virtualization/forms/model_forms.py +++ b/netbox/virtualization/forms/model_forms.py @@ -6,6 +6,7 @@ from django.utils.translation import gettext_lazy as _ from dcim.forms.common import InterfaceCommonForm from dcim.models import Device, DeviceRole, Platform, Rack, Region, Site, SiteGroup from extras.models import ConfigTemplate +from ipam.choices import VLANQinQRoleChoices from ipam.models import IPAddress, VLAN, VLANGroup, VLANTranslationPolicy, VRF from netbox.forms import NetBoxModelForm from tenancy.forms import TenancyForm @@ -338,6 +339,16 @@ class VMInterfaceForm(InterfaceCommonForm, VMComponentForm): 'available_on_virtualmachine': '$virtual_machine', } ) + qinq_svlan = DynamicModelChoiceField( + queryset=VLAN.objects.all(), + required=False, + label=_('Q-in-Q Service VLAN'), + query_params={ + 'group_id': '$vlan_group', + 'available_on_virtualmachine': '$virtual_machine', + 'qinq_role': VLANQinQRoleChoices.ROLE_SERVICE, + } + ) vrf = DynamicModelChoiceField( queryset=VRF.objects.all(), required=False, @@ -354,17 +365,20 @@ class VMInterfaceForm(InterfaceCommonForm, VMComponentForm): FieldSet('vrf', 'mac_address', name=_('Addressing')), FieldSet('mtu', 'enabled', name=_('Operation')), FieldSet('parent', 'bridge', name=_('Related Interfaces')), - FieldSet('mode', 'vlan_group', 'untagged_vlan', 'tagged_vlans', 'vlan_translation_policy', name=_('802.1Q Switching')), + FieldSet( + 'mode', 'vlan_group', 'untagged_vlan', 'tagged_vlans', 'qinq_svlan', 'vlan_translation_policy', + name=_('802.1Q Switching') + ), ) class Meta: model = VMInterface fields = [ 'virtual_machine', 'name', 'parent', 'bridge', 'enabled', 'mac_address', 'mtu', 'description', 'mode', - 'vlan_group', 'untagged_vlan', 'tagged_vlans', 'vrf', 'tags', 'vlan_translation_policy', + 'vlan_group', 'untagged_vlan', 'tagged_vlans', 'qinq_svlan', 'vlan_translation_policy', 'vrf', 'tags', ] labels = { - 'mode': '802.1Q Mode', + 'mode': _('802.1Q Mode'), } widgets = { 'mode': HTMXSelect(), diff --git a/netbox/virtualization/graphql/types.py b/netbox/virtualization/graphql/types.py index bed65a3b3..79b5cb216 100644 --- a/netbox/virtualization/graphql/types.py +++ b/netbox/virtualization/graphql/types.py @@ -100,6 +100,7 @@ class VMInterfaceType(IPAddressesMixin, ComponentType): bridge: Annotated["VMInterfaceType", strawberry.lazy('virtualization.graphql.types')] | None untagged_vlan: Annotated["VLANType", strawberry.lazy('ipam.graphql.types')] | None vrf: Annotated["VRFType", strawberry.lazy('ipam.graphql.types')] | None + qinq_svlan: Annotated["VLANType", strawberry.lazy('ipam.graphql.types')] | None vlan_translation_policy: Annotated["VLANTranslationPolicyType", strawberry.lazy('ipam.graphql.types')] | None tagged_vlans: List[Annotated["VLANType", strawberry.lazy('ipam.graphql.types')]] diff --git a/netbox/virtualization/migrations/0042_vminterface_vlan_translation_policy.py b/netbox/virtualization/migrations/0042_vminterface_vlan_translation_policy.py index e0992c9c8..3a6d5e481 100644 --- a/netbox/virtualization/migrations/0042_vminterface_vlan_translation_policy.py +++ b/netbox/virtualization/migrations/0042_vminterface_vlan_translation_policy.py @@ -1,5 +1,3 @@ -# Generated by Django 5.0.9 on 2024-10-11 19:45 - import django.db.models.deletion from django.db import migrations, models diff --git a/netbox/virtualization/migrations/0043_qinq_svlan.py b/netbox/virtualization/migrations/0043_qinq_svlan.py new file mode 100644 index 000000000..422289fb7 --- /dev/null +++ b/netbox/virtualization/migrations/0043_qinq_svlan.py @@ -0,0 +1,28 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ipam', '0075_vlan_qinq'), + ('virtualization', '0042_vminterface_vlan_translation_policy'), + ] + + operations = [ + migrations.AddField( + model_name='vminterface', + name='qinq_svlan', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)ss_svlan', to='ipam.vlan'), + ), + migrations.AlterField( + model_name='vminterface', + name='tagged_vlans', + field=models.ManyToManyField(blank=True, related_name='%(class)ss_as_tagged', to='ipam.vlan'), + ), + migrations.AlterField( + model_name='vminterface', + name='untagged_vlan', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)ss_as_untagged', to='ipam.vlan'), + ), + ] diff --git a/netbox/virtualization/models/virtualmachines.py b/netbox/virtualization/models/virtualmachines.py index 0767b2c13..da8419e88 100644 --- a/netbox/virtualization/models/virtualmachines.py +++ b/netbox/virtualization/models/virtualmachines.py @@ -322,20 +322,6 @@ class VMInterface(ComponentModel, BaseInterface, TrackingModelMixin): max_length=100, blank=True ) - untagged_vlan = models.ForeignKey( - to='ipam.VLAN', - on_delete=models.SET_NULL, - related_name='vminterfaces_as_untagged', - null=True, - blank=True, - verbose_name=_('untagged VLAN') - ) - tagged_vlans = models.ManyToManyField( - to='ipam.VLAN', - related_name='vminterfaces_as_tagged', - blank=True, - verbose_name=_('tagged VLANs') - ) ip_addresses = GenericRelation( to='ipam.IPAddress', content_type_field='assigned_object_type', diff --git a/netbox/virtualization/tables/virtualmachines.py b/netbox/virtualization/tables/virtualmachines.py index 8a2a26bb9..4a3138711 100644 --- a/netbox/virtualization/tables/virtualmachines.py +++ b/netbox/virtualization/tables/virtualmachines.py @@ -151,8 +151,8 @@ class VMInterfaceTable(BaseInterfaceTable): model = VMInterface fields = ( 'pk', 'id', 'name', 'virtual_machine', 'enabled', 'mac_address', 'mtu', 'mode', 'description', 'tags', - 'vrf', 'l2vpn', 'tunnel', 'ip_addresses', 'fhrp_groups', 'untagged_vlan', 'tagged_vlans', 'created', - 'last_updated', + 'vrf', 'l2vpn', 'tunnel', 'ip_addresses', 'fhrp_groups', 'untagged_vlan', 'tagged_vlans', 'qinq_svlan', + 'created', 'last_updated', ) default_columns = ('pk', 'name', 'virtual_machine', 'enabled', 'description') @@ -175,7 +175,8 @@ class VirtualMachineVMInterfaceTable(VMInterfaceTable): model = VMInterface fields = ( 'pk', 'id', 'name', 'enabled', 'parent', 'bridge', 'mac_address', 'mtu', 'mode', 'description', 'tags', - 'vrf', 'l2vpn', 'tunnel', 'ip_addresses', 'fhrp_groups', 'untagged_vlan', 'tagged_vlans', 'actions', + 'vrf', 'l2vpn', 'tunnel', 'ip_addresses', 'fhrp_groups', 'untagged_vlan', 'tagged_vlans', 'qinq_svlan', + 'actions', ) default_columns = ('pk', 'name', 'enabled', 'mac_address', 'mtu', 'mode', 'description', 'ip_addresses') row_attrs = { diff --git a/netbox/virtualization/tests/test_api.py b/netbox/virtualization/tests/test_api.py index 69728f67c..521064fc6 100644 --- a/netbox/virtualization/tests/test_api.py +++ b/netbox/virtualization/tests/test_api.py @@ -4,6 +4,7 @@ from rest_framework import status from dcim.choices import InterfaceModeChoices from dcim.models import Site from extras.models import ConfigTemplate +from ipam.choices import VLANQinQRoleChoices from ipam.models import VLAN, VRF from utilities.testing import APITestCase, APIViewTestCases, create_test_device, create_test_virtualmachine from virtualization.choices import * @@ -270,6 +271,7 @@ class VMInterfaceTest(APIViewTestCases.APIViewTestCase): VLAN(name='VLAN 1', vid=1), VLAN(name='VLAN 2', vid=2), VLAN(name='VLAN 3', vid=3), + VLAN(name='SVLAN 1', vid=1001, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE), ) VLAN.objects.bulk_create(vlans) @@ -307,6 +309,12 @@ class VMInterfaceTest(APIViewTestCases.APIViewTestCase): 'untagged_vlan': vlans[2].pk, 'vrf': vrfs[2].pk, }, + { + 'virtual_machine': virtualmachine.pk, + 'name': 'Interface 7', + 'mode': InterfaceModeChoices.MODE_Q_IN_Q, + 'qinq_svlan': vlans[3].pk, + }, ] def test_bulk_delete_child_interfaces(self): diff --git a/netbox/virtualization/tests/test_filtersets.py b/netbox/virtualization/tests/test_filtersets.py index cd598274f..0c7079bba 100644 --- a/netbox/virtualization/tests/test_filtersets.py +++ b/netbox/virtualization/tests/test_filtersets.py @@ -1,7 +1,9 @@ from django.test import TestCase +from dcim.choices import InterfaceModeChoices from dcim.models import Device, DeviceRole, Platform, Region, Site, SiteGroup -from ipam.models import IPAddress, VLANTranslationPolicy, VRF +from ipam.choices import VLANQinQRoleChoices +from ipam.models import IPAddress, VLAN, VLANTranslationPolicy, VRF from tenancy.models import Tenant, TenantGroup from utilities.testing import ChangeLoggedFilterSetTests, create_test_device from virtualization.choices import * @@ -528,7 +530,7 @@ class VirtualMachineTestCase(TestCase, ChangeLoggedFilterSetTests): class VMInterfaceTestCase(TestCase, ChangeLoggedFilterSetTests): queryset = VMInterface.objects.all() filterset = VMInterfaceFilterSet - ignore_fields = ('tagged_vlans', 'untagged_vlan',) + ignore_fields = ('tagged_vlans', 'untagged_vlan', 'qinq_svlan') @classmethod def setUpTestData(cls): @@ -554,6 +556,13 @@ class VMInterfaceTestCase(TestCase, ChangeLoggedFilterSetTests): ) VRF.objects.bulk_create(vrfs) + vlans = ( + VLAN(name='SVLAN 1', vid=1001, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE), + VLAN(name='SVLAN 2', vid=1002, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE), + VLAN(name='SVLAN 3', vid=1003, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE), + ) + VLAN.objects.bulk_create(vlans) + vms = ( VirtualMachine(name='Virtual Machine 1', cluster=clusters[0]), VirtualMachine(name='Virtual Machine 2', cluster=clusters[1]), @@ -596,7 +605,9 @@ class VMInterfaceTestCase(TestCase, ChangeLoggedFilterSetTests): mtu=300, mac_address='00-00-00-00-00-03', vrf=vrfs[2], - description='foobar3' + description='foobar3', + mode=InterfaceModeChoices.MODE_Q_IN_Q, + qinq_svlan=vlans[0] ), ) VMInterface.objects.bulk_create(interfaces) @@ -667,6 +678,13 @@ class VMInterfaceTestCase(TestCase, ChangeLoggedFilterSetTests): params = {'description': ['foobar1', 'foobar2']} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + def test_vlan(self): + vlan = VLAN.objects.filter(qinq_role=VLANQinQRoleChoices.ROLE_SERVICE).first() + params = {'vlan_id': vlan.pk} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + params = {'vlan': vlan.vid} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + def test_vlan_translation_policy(self): vlan_translation_policies = VLANTranslationPolicy.objects.all()[:2] params = {'vlan_translation_policy_id': [vlan_translation_policies[0].pk, vlan_translation_policies[1].pk]} From 6dc75d8db1e7750f37ba5f50d65e05c430352146 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Fri, 1 Nov 2024 11:18:23 -0700 Subject: [PATCH 2/3] 7699 Add Scope to Cluster (#17848) * 7699 Add Scope to Cluster * 7699 Serializer * 7699 filterset * 7699 bulk_edit * 7699 bulk_import * 7699 model_form * 7699 graphql, tables * 7699 fixes * 7699 fixes * 7699 fixes * 7699 fixes * 7699 fix tests * 7699 fix graphql tests for clusters reference * 7699 fix dcim tests * 7699 fix ipam tests * 7699 fix tests * 7699 use mixin for model * 7699 change mixin name * 7699 scope form * 7699 scope form * 7699 scoped form, fitlerset * 7699 review changes * 7699 move ScopedFilterset * 7699 move CachedScopeMixin * 7699 review changes * 7699 review changes * 7699 refactor mixins * 7699 _sitegroup -> _site_group * 7699 update docstring * Misc cleanup * Update migrations --------- Co-authored-by: Jeremy Stretch --- docs/models/virtualization/cluster.md | 4 +- netbox/dcim/constants.py | 5 + netbox/dcim/filtersets.py | 58 ++++++++++ netbox/dcim/forms/mixins.py | 105 ++++++++++++++++++ netbox/dcim/graphql/types.py | 17 +++ netbox/dcim/models/devices.py | 11 +- netbox/dcim/models/mixins.py | 85 ++++++++++++++ netbox/dcim/tests/test_models.py | 9 +- netbox/extras/tests/test_models.py | 4 +- netbox/ipam/querysets.py | 2 +- netbox/ipam/tests/test_filtersets.py | 9 +- netbox/templates/virtualization/cluster.html | 8 +- .../api/serializers_/clusters.py | 31 +++++- netbox/virtualization/apps.py | 2 +- netbox/virtualization/filtersets.py | 42 +------ netbox/virtualization/forms/bulk_edit.py | 28 +---- netbox/virtualization/forms/bulk_import.py | 8 +- netbox/virtualization/forms/filtersets.py | 19 ++-- netbox/virtualization/forms/model_forms.py | 14 +-- netbox/virtualization/graphql/types.py | 15 ++- .../migrations/0044_cluster_scope.py | 51 +++++++++ .../0045_clusters_cached_relations.py | 94 ++++++++++++++++ netbox/virtualization/models/clusters.py | 48 +++++--- .../virtualization/models/virtualmachines.py | 4 +- netbox/virtualization/tables/clusters.py | 11 +- netbox/virtualization/tests/test_api.py | 10 +- .../virtualization/tests/test_filtersets.py | 18 +-- netbox/virtualization/tests/test_models.py | 9 +- netbox/virtualization/tests/test_views.py | 23 ++-- 29 files changed, 588 insertions(+), 156 deletions(-) create mode 100644 netbox/dcim/forms/mixins.py create mode 100644 netbox/virtualization/migrations/0044_cluster_scope.py create mode 100644 netbox/virtualization/migrations/0045_clusters_cached_relations.py diff --git a/docs/models/virtualization/cluster.md b/docs/models/virtualization/cluster.md index 50b5dbd1d..9acdb2bc4 100644 --- a/docs/models/virtualization/cluster.md +++ b/docs/models/virtualization/cluster.md @@ -23,6 +23,6 @@ The cluster's operational status. !!! tip Additional statuses may be defined by setting `Cluster.status` under the [`FIELD_CHOICES`](../../configuration/data-validation.md#field_choices) configuration parameter. -### Site +### Scope -The [site](../dcim/site.md) with which the cluster is associated. +The [region](../dcim/region.md), [site](../dcim/site.md), [site group](../dcim/sitegroup.md) or [location](../dcim/location.md) with which this cluster is associated. diff --git a/netbox/dcim/constants.py b/netbox/dcim/constants.py index ba3e6464b..4927b0198 100644 --- a/netbox/dcim/constants.py +++ b/netbox/dcim/constants.py @@ -123,3 +123,8 @@ COMPATIBLE_TERMINATION_TYPES = { 'powerport': ['poweroutlet', 'powerfeed'], 'rearport': ['consoleport', 'consoleserverport', 'interface', 'frontport', 'rearport', 'circuittermination'], } + +# Models which can serve to scope an object by location +LOCATION_SCOPE_TYPES = ( + 'region', 'sitegroup', 'site', 'location', +) diff --git a/netbox/dcim/filtersets.py b/netbox/dcim/filtersets.py index 0371f882b..df66ad77b 100644 --- a/netbox/dcim/filtersets.py +++ b/netbox/dcim/filtersets.py @@ -73,6 +73,7 @@ __all__ = ( 'RearPortFilterSet', 'RearPortTemplateFilterSet', 'RegionFilterSet', + 'ScopedFilterSet', 'SiteFilterSet', 'SiteGroupFilterSet', 'VirtualChassisFilterSet', @@ -2344,3 +2345,60 @@ class InterfaceConnectionFilterSet(ConnectionFilterSet): class Meta: model = Interface fields = tuple() + + +class ScopedFilterSet(BaseFilterSet): + """ + Provides additional filtering functionality for location, site, etc.. for Scoped models. + """ + scope_type = ContentTypeFilter() + region_id = TreeNodeMultipleChoiceFilter( + queryset=Region.objects.all(), + field_name='_region', + lookup_expr='in', + label=_('Region (ID)'), + ) + region = TreeNodeMultipleChoiceFilter( + queryset=Region.objects.all(), + field_name='_region', + lookup_expr='in', + to_field_name='slug', + label=_('Region (slug)'), + ) + site_group_id = TreeNodeMultipleChoiceFilter( + queryset=SiteGroup.objects.all(), + field_name='_site_group', + lookup_expr='in', + label=_('Site group (ID)'), + ) + site_group = TreeNodeMultipleChoiceFilter( + queryset=SiteGroup.objects.all(), + field_name='_site_group', + lookup_expr='in', + to_field_name='slug', + label=_('Site group (slug)'), + ) + site_id = django_filters.ModelMultipleChoiceFilter( + queryset=Site.objects.all(), + field_name='_site', + label=_('Site (ID)'), + ) + site = django_filters.ModelMultipleChoiceFilter( + field_name='_site__slug', + queryset=Site.objects.all(), + to_field_name='slug', + label=_('Site (slug)'), + ) + location_id = TreeNodeMultipleChoiceFilter( + queryset=Location.objects.all(), + field_name='_location', + lookup_expr='in', + label=_('Location (ID)'), + ) + location = TreeNodeMultipleChoiceFilter( + queryset=Location.objects.all(), + field_name='_location', + lookup_expr='in', + to_field_name='slug', + label=_('Location (slug)'), + ) diff --git a/netbox/dcim/forms/mixins.py b/netbox/dcim/forms/mixins.py new file mode 100644 index 000000000..98862af10 --- /dev/null +++ b/netbox/dcim/forms/mixins.py @@ -0,0 +1,105 @@ +from django import forms +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ObjectDoesNotExist +from django.utils.translation import gettext_lazy as _ + +from dcim.constants import LOCATION_SCOPE_TYPES +from dcim.models import Site +from utilities.forms import get_field_value +from utilities.forms.fields import ( + ContentTypeChoiceField, CSVContentTypeField, DynamicModelChoiceField, +) +from utilities.templatetags.builtins.filters import bettertitle +from utilities.forms.widgets import HTMXSelect + +__all__ = ( + 'ScopedBulkEditForm', + 'ScopedForm', + 'ScopedImportForm', +) + + +class ScopedForm(forms.Form): + scope_type = ContentTypeChoiceField( + queryset=ContentType.objects.filter(model__in=LOCATION_SCOPE_TYPES), + widget=HTMXSelect(), + required=False, + label=_('Scope type') + ) + scope = DynamicModelChoiceField( + label=_('Scope'), + queryset=Site.objects.none(), # Initial queryset + required=False, + disabled=True, + selector=True + ) + + def __init__(self, *args, **kwargs): + instance = kwargs.get('instance') + initial = kwargs.get('initial', {}) + + if instance is not None and instance.scope: + initial['scope'] = instance.scope + kwargs['initial'] = initial + + super().__init__(*args, **kwargs) + self._set_scoped_values() + + def clean(self): + super().clean() + + # Assign the selected scope (if any) + self.instance.scope = self.cleaned_data.get('scope') + + def _set_scoped_values(self): + if scope_type_id := get_field_value(self, 'scope_type'): + try: + scope_type = ContentType.objects.get(pk=scope_type_id) + model = scope_type.model_class() + self.fields['scope'].queryset = model.objects.all() + self.fields['scope'].widget.attrs['selector'] = model._meta.label_lower + self.fields['scope'].disabled = False + self.fields['scope'].label = _(bettertitle(model._meta.verbose_name)) + except ObjectDoesNotExist: + pass + + if self.instance and scope_type_id != self.instance.scope_type_id: + self.initial['scope'] = None + + +class ScopedBulkEditForm(forms.Form): + scope_type = ContentTypeChoiceField( + queryset=ContentType.objects.filter(model__in=LOCATION_SCOPE_TYPES), + widget=HTMXSelect(method='post', attrs={'hx-select': '#form_fields'}), + required=False, + label=_('Scope type') + ) + scope = DynamicModelChoiceField( + label=_('Scope'), + queryset=Site.objects.none(), # Initial queryset + required=False, + disabled=True, + selector=True + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + if scope_type_id := get_field_value(self, 'scope_type'): + try: + scope_type = ContentType.objects.get(pk=scope_type_id) + model = scope_type.model_class() + self.fields['scope'].queryset = model.objects.all() + self.fields['scope'].widget.attrs['selector'] = model._meta.label_lower + self.fields['scope'].disabled = False + self.fields['scope'].label = _(bettertitle(model._meta.verbose_name)) + except ObjectDoesNotExist: + pass + + +class ScopedImportForm(forms.Form): + scope_type = CSVContentTypeField( + queryset=ContentType.objects.filter(model__in=LOCATION_SCOPE_TYPES), + required=False, + label=_('Scope type (app & model)') + ) diff --git a/netbox/dcim/graphql/types.py b/netbox/dcim/graphql/types.py index 5965fdcec..6493ec6b1 100644 --- a/netbox/dcim/graphql/types.py +++ b/netbox/dcim/graphql/types.py @@ -463,6 +463,10 @@ class LocationType(VLANGroupsMixin, ImageAttachmentsMixin, ContactsMixin, Organi devices: List[Annotated["DeviceType", strawberry.lazy('dcim.graphql.types')]] children: List[Annotated["LocationType", strawberry.lazy('dcim.graphql.types')]] + @strawberry_django.field + def clusters(self) -> List[Annotated["ClusterType", strawberry.lazy('virtualization.graphql.types')]]: + return self._clusters.all() + @strawberry_django.field def circuit_terminations(self) -> List[Annotated["CircuitTerminationType", strawberry.lazy('circuits.graphql.types')]]: return self.circuit_terminations.all() @@ -710,6 +714,10 @@ class RegionType(VLANGroupsMixin, ContactsMixin, OrganizationalObjectType): def parent(self) -> Annotated["RegionType", strawberry.lazy('dcim.graphql.types')] | None: return self.parent + @strawberry_django.field + def clusters(self) -> List[Annotated["ClusterType", strawberry.lazy('virtualization.graphql.types')]]: + return self._clusters.all() + @strawberry_django.field def circuit_terminations(self) -> List[Annotated["CircuitTerminationType", strawberry.lazy('circuits.graphql.types')]]: return self.circuit_terminations.all() @@ -735,9 +743,14 @@ class SiteType(VLANGroupsMixin, ImageAttachmentsMixin, ContactsMixin, NetBoxObje devices: List[Annotated["DeviceType", strawberry.lazy('dcim.graphql.types')]] locations: List[Annotated["LocationType", strawberry.lazy('dcim.graphql.types')]] asns: List[Annotated["ASNType", strawberry.lazy('ipam.graphql.types')]] + circuit_terminations: List[Annotated["CircuitTerminationType", strawberry.lazy('circuits.graphql.types')]] clusters: List[Annotated["ClusterType", strawberry.lazy('virtualization.graphql.types')]] vlans: List[Annotated["VLANType", strawberry.lazy('ipam.graphql.types')]] + @strawberry_django.field + def clusters(self) -> List[Annotated["ClusterType", strawberry.lazy('virtualization.graphql.types')]]: + return self._clusters.all() + @strawberry_django.field def circuit_terminations(self) -> List[Annotated["CircuitTerminationType", strawberry.lazy('circuits.graphql.types')]]: return self.circuit_terminations.all() @@ -758,6 +771,10 @@ class SiteGroupType(VLANGroupsMixin, ContactsMixin, OrganizationalObjectType): def parent(self) -> Annotated["SiteGroupType", strawberry.lazy('dcim.graphql.types')] | None: return self.parent + @strawberry_django.field + def clusters(self) -> List[Annotated["ClusterType", strawberry.lazy('virtualization.graphql.types')]]: + return self._clusters.all() + @strawberry_django.field def circuit_terminations(self) -> List[Annotated["CircuitTerminationType", strawberry.lazy('circuits.graphql.types')]]: return self.circuit_terminations.all() diff --git a/netbox/dcim/models/devices.py b/netbox/dcim/models/devices.py index b9ba2bb64..47f4ee6c9 100644 --- a/netbox/dcim/models/devices.py +++ b/netbox/dcim/models/devices.py @@ -958,10 +958,17 @@ class Device( }) # A Device can only be assigned to a Cluster in the same Site (or no Site) - if self.cluster and self.cluster.site is not None and self.cluster.site != self.site: + if self.cluster and self.cluster._site is not None and self.cluster._site != self.site: raise ValidationError({ 'cluster': _("The assigned cluster belongs to a different site ({site})").format( - site=self.cluster.site + site=self.cluster._site + ) + }) + + if self.cluster and self.cluster._location is not None and self.cluster._location != self.location: + raise ValidationError({ + 'cluster': _("The assigned cluster belongs to a different location ({location})").format( + site=self.cluster._location ) }) diff --git a/netbox/dcim/models/mixins.py b/netbox/dcim/models/mixins.py index c9be451a0..1df3364c4 100644 --- a/netbox/dcim/models/mixins.py +++ b/netbox/dcim/models/mixins.py @@ -1,6 +1,10 @@ +from django.apps import apps +from django.contrib.contenttypes.fields import GenericForeignKey from django.db import models +from dcim.constants import LOCATION_SCOPE_TYPES __all__ = ( + 'CachedScopeMixin', 'RenderConfigMixin', ) @@ -27,3 +31,84 @@ class RenderConfigMixin(models.Model): return self.role.config_template if self.platform and self.platform.config_template: return self.platform.config_template + + +class CachedScopeMixin(models.Model): + """ + Mixin for adding a GenericForeignKey scope to a model that can point to a Region, SiteGroup, Site, or Location. + Includes cached fields for each to allow efficient filtering. Appropriate validation must be done in the clean() + method as this does not have any as validation is generally model-specific. + """ + scope_type = models.ForeignKey( + to='contenttypes.ContentType', + on_delete=models.PROTECT, + limit_choices_to=models.Q(model__in=LOCATION_SCOPE_TYPES), + related_name='+', + blank=True, + null=True + ) + scope_id = models.PositiveBigIntegerField( + blank=True, + null=True + ) + scope = GenericForeignKey( + ct_field='scope_type', + fk_field='scope_id' + ) + + _location = models.ForeignKey( + to='dcim.Location', + on_delete=models.CASCADE, + related_name='_%(class)ss', + blank=True, + null=True + ) + _site = models.ForeignKey( + to='dcim.Site', + on_delete=models.CASCADE, + related_name='_%(class)ss', + blank=True, + null=True + ) + _region = models.ForeignKey( + to='dcim.Region', + on_delete=models.CASCADE, + related_name='_%(class)ss', + blank=True, + null=True + ) + _site_group = models.ForeignKey( + to='dcim.SiteGroup', + on_delete=models.CASCADE, + related_name='_%(class)ss', + blank=True, + null=True + ) + + class Meta: + abstract = True + + def save(self, *args, **kwargs): + # Cache objects associated with the terminating object (for filtering) + self.cache_related_objects() + + super().save(*args, **kwargs) + + def cache_related_objects(self): + self._region = self._site_group = self._site = self._location = None + if self.scope_type: + scope_type = self.scope_type.model_class() + if scope_type == apps.get_model('dcim', 'region'): + self._region = self.scope + elif scope_type == apps.get_model('dcim', 'sitegroup'): + self._site_group = self.scope + elif scope_type == apps.get_model('dcim', 'site'): + self._region = self.scope.region + self._site_group = self.scope.group + self._site = self.scope + elif scope_type == apps.get_model('dcim', 'location'): + self._region = self.scope.site.region + self._site_group = self.scope.site.group + self._site = self.scope.site + self._location = self.scope + cache_related_objects.alters_data = True diff --git a/netbox/dcim/tests/test_models.py b/netbox/dcim/tests/test_models.py index f0fe4da3b..8d43d67ea 100644 --- a/netbox/dcim/tests/test_models.py +++ b/netbox/dcim/tests/test_models.py @@ -601,11 +601,12 @@ class DeviceTestCase(TestCase): Site.objects.bulk_create(sites) clusters = ( - Cluster(name='Cluster 1', type=cluster_type, site=sites[0]), - Cluster(name='Cluster 2', type=cluster_type, site=sites[1]), - Cluster(name='Cluster 3', type=cluster_type, site=None), + Cluster(name='Cluster 1', type=cluster_type, scope=sites[0]), + Cluster(name='Cluster 2', type=cluster_type, scope=sites[1]), + Cluster(name='Cluster 3', type=cluster_type, scope=None), ) - Cluster.objects.bulk_create(clusters) + for cluster in clusters: + cluster.save() device_type = DeviceType.objects.first() device_role = DeviceRole.objects.first() diff --git a/netbox/extras/tests/test_models.py b/netbox/extras/tests/test_models.py index 188a06a3f..c90390dd1 100644 --- a/netbox/extras/tests/test_models.py +++ b/netbox/extras/tests/test_models.py @@ -274,7 +274,7 @@ class ConfigContextTest(TestCase): name="Cluster", group=cluster_group, type=cluster_type, - site=site, + scope=site, ) region_context = ConfigContext.objects.create( @@ -366,7 +366,7 @@ class ConfigContextTest(TestCase): """ site = Site.objects.first() cluster_type = ClusterType.objects.create(name="Cluster Type") - cluster = Cluster.objects.create(name="Cluster", type=cluster_type, site=site) + cluster = Cluster.objects.create(name="Cluster", type=cluster_type, scope=site) vm_role = DeviceRole.objects.first() # Create a ConfigContext associated with the site diff --git a/netbox/ipam/querysets.py b/netbox/ipam/querysets.py index 771e9b3b9..77ab8194a 100644 --- a/netbox/ipam/querysets.py +++ b/netbox/ipam/querysets.py @@ -148,7 +148,7 @@ class VLANQuerySet(RestrictedQuerySet): # Find all relevant VLANGroups q = Q() - site = vm.site or vm.cluster.site + site = vm.site or vm.cluster._site if vm.cluster: # Add VLANGroups scoped to the assigned cluster (or its group) q |= Q( diff --git a/netbox/ipam/tests/test_filtersets.py b/netbox/ipam/tests/test_filtersets.py index f651c970d..28e8cda1e 100644 --- a/netbox/ipam/tests/test_filtersets.py +++ b/netbox/ipam/tests/test_filtersets.py @@ -1675,11 +1675,12 @@ class VLANTestCase(TestCase, ChangeLoggedFilterSetTests): cluster_type = ClusterType.objects.create(name='Cluster Type 1', slug='cluster-type-1') clusters = ( - Cluster(name='Cluster 1', type=cluster_type, group=cluster_groups[0], site=sites[0]), - Cluster(name='Cluster 2', type=cluster_type, group=cluster_groups[1], site=sites[1]), - Cluster(name='Cluster 3', type=cluster_type, group=cluster_groups[2], site=sites[2]), + Cluster(name='Cluster 1', type=cluster_type, group=cluster_groups[0], scope=sites[0]), + Cluster(name='Cluster 2', type=cluster_type, group=cluster_groups[1], scope=sites[1]), + Cluster(name='Cluster 3', type=cluster_type, group=cluster_groups[2], scope=sites[2]), ) - Cluster.objects.bulk_create(clusters) + for cluster in clusters: + cluster.save() virtual_machines = ( VirtualMachine(name='Virtual Machine 1', cluster=clusters[0]), diff --git a/netbox/templates/virtualization/cluster.html b/netbox/templates/virtualization/cluster.html index d79d8075c..4155dacb2 100644 --- a/netbox/templates/virtualization/cluster.html +++ b/netbox/templates/virtualization/cluster.html @@ -39,8 +39,12 @@ - {% trans "Site" %} - {{ object.site|linkify|placeholder }} + {% trans "Scope" %} + {% if object.scope %} + {{ object.scope|linkify }} ({% trans object.scope_type.name %}) + {% else %} + {{ ''|placeholder }} + {% endif %}
diff --git a/netbox/virtualization/api/serializers_/clusters.py b/netbox/virtualization/api/serializers_/clusters.py index b64b6e7ba..101a5b5a3 100644 --- a/netbox/virtualization/api/serializers_/clusters.py +++ b/netbox/virtualization/api/serializers_/clusters.py @@ -1,9 +1,13 @@ -from dcim.api.serializers_.sites import SiteSerializer -from netbox.api.fields import ChoiceField, RelatedObjectCountField +from dcim.constants import LOCATION_SCOPE_TYPES +from django.contrib.contenttypes.models import ContentType +from drf_spectacular.utils import extend_schema_field +from rest_framework import serializers +from netbox.api.fields import ChoiceField, ContentTypeField, RelatedObjectCountField from netbox.api.serializers import NetBoxModelSerializer from tenancy.api.serializers_.tenants import TenantSerializer from virtualization.choices import * from virtualization.models import Cluster, ClusterGroup, ClusterType +from utilities.api import get_serializer_for_model __all__ = ( 'ClusterGroupSerializer', @@ -45,7 +49,16 @@ class ClusterSerializer(NetBoxModelSerializer): group = ClusterGroupSerializer(nested=True, required=False, allow_null=True, default=None) status = ChoiceField(choices=ClusterStatusChoices, required=False) tenant = TenantSerializer(nested=True, required=False, allow_null=True) - site = SiteSerializer(nested=True, required=False, allow_null=True, default=None) + scope_type = ContentTypeField( + queryset=ContentType.objects.filter( + model__in=LOCATION_SCOPE_TYPES + ), + allow_null=True, + required=False, + default=None + ) + scope_id = serializers.IntegerField(allow_null=True, required=False, default=None) + scope = serializers.SerializerMethodField(read_only=True) # Related object counts device_count = RelatedObjectCountField('devices') @@ -54,8 +67,18 @@ class ClusterSerializer(NetBoxModelSerializer): class Meta: model = Cluster fields = [ - 'id', 'url', 'display_url', 'display', 'name', 'type', 'group', 'status', 'tenant', 'site', + 'id', 'url', 'display_url', 'display', 'name', 'type', 'group', 'status', 'tenant', 'scope_type', 'scope_id', 'scope', 'description', 'comments', 'tags', 'custom_fields', 'created', 'last_updated', 'device_count', 'virtualmachine_count', ] brief_fields = ('id', 'url', 'display', 'name', 'description', 'virtualmachine_count') + + @extend_schema_field(serializers.JSONField(allow_null=True)) + def get_scope(self, obj): + if obj.scope_id is None: + return None + serializer = get_serializer_for_model(obj.scope) + context = {'request': self.context['request']} + return serializer(obj.scope, nested=True, context=context).data + + diff --git a/netbox/virtualization/apps.py b/netbox/virtualization/apps.py index ebcc591bf..65ce0f112 100644 --- a/netbox/virtualization/apps.py +++ b/netbox/virtualization/apps.py @@ -17,7 +17,7 @@ class VirtualizationConfig(AppConfig): # Register denormalized fields denormalized.register(VirtualMachine, 'cluster', { - 'site': 'site', + 'site': '_site', }) # Register counters diff --git a/netbox/virtualization/filtersets.py b/netbox/virtualization/filtersets.py index ec0831f9f..ac72bea12 100644 --- a/netbox/virtualization/filtersets.py +++ b/netbox/virtualization/filtersets.py @@ -2,7 +2,7 @@ import django_filters from django.db.models import Q from django.utils.translation import gettext as _ -from dcim.filtersets import CommonInterfaceFilterSet +from dcim.filtersets import CommonInterfaceFilterSet, ScopedFilterSet from dcim.models import Device, DeviceRole, Platform, Region, Site, SiteGroup from extras.filtersets import LocalConfigContextFilterSet from extras.models import ConfigTemplate @@ -37,43 +37,7 @@ class ClusterGroupFilterSet(OrganizationalModelFilterSet, ContactModelFilterSet) fields = ('id', 'name', 'slug', 'description') -class ClusterFilterSet(NetBoxModelFilterSet, TenancyFilterSet, ContactModelFilterSet): - region_id = TreeNodeMultipleChoiceFilter( - queryset=Region.objects.all(), - field_name='site__region', - lookup_expr='in', - label=_('Region (ID)'), - ) - region = TreeNodeMultipleChoiceFilter( - queryset=Region.objects.all(), - field_name='site__region', - lookup_expr='in', - to_field_name='slug', - label=_('Region (slug)'), - ) - site_group_id = TreeNodeMultipleChoiceFilter( - queryset=SiteGroup.objects.all(), - field_name='site__group', - lookup_expr='in', - label=_('Site group (ID)'), - ) - site_group = TreeNodeMultipleChoiceFilter( - queryset=SiteGroup.objects.all(), - field_name='site__group', - lookup_expr='in', - to_field_name='slug', - label=_('Site group (slug)'), - ) - site_id = django_filters.ModelMultipleChoiceFilter( - queryset=Site.objects.all(), - label=_('Site (ID)'), - ) - site = django_filters.ModelMultipleChoiceFilter( - field_name='site__slug', - queryset=Site.objects.all(), - to_field_name='slug', - label=_('Site (slug)'), - ) +class ClusterFilterSet(NetBoxModelFilterSet, TenancyFilterSet, ScopedFilterSet, ContactModelFilterSet): group_id = django_filters.ModelMultipleChoiceFilter( queryset=ClusterGroup.objects.all(), label=_('Parent group (ID)'), @@ -101,7 +65,7 @@ class ClusterFilterSet(NetBoxModelFilterSet, TenancyFilterSet, ContactModelFilte class Meta: model = Cluster - fields = ('id', 'name', 'description') + fields = ('id', 'name', 'description', 'scope_id') def search(self, queryset, name, value): if not value.strip(): diff --git a/netbox/virtualization/forms/bulk_edit.py b/netbox/virtualization/forms/bulk_edit.py index 2bd3434ac..aaeb259b9 100644 --- a/netbox/virtualization/forms/bulk_edit.py +++ b/netbox/virtualization/forms/bulk_edit.py @@ -3,7 +3,8 @@ from django.utils.translation import gettext_lazy as _ from dcim.choices import InterfaceModeChoices from dcim.constants import INTERFACE_MTU_MAX, INTERFACE_MTU_MIN -from dcim.models import Device, DeviceRole, Platform, Region, Site, SiteGroup +from dcim.forms.mixins import ScopedBulkEditForm +from dcim.models import Device, DeviceRole, Platform, Site from extras.models import ConfigTemplate from ipam.models import VLAN, VLANGroup, VRF from netbox.forms import NetBoxModelBulkEditForm @@ -55,7 +56,7 @@ class ClusterGroupBulkEditForm(NetBoxModelBulkEditForm): nullable_fields = ('description',) -class ClusterBulkEditForm(NetBoxModelBulkEditForm): +class ClusterBulkEditForm(ScopedBulkEditForm, NetBoxModelBulkEditForm): type = DynamicModelChoiceField( label=_('Type'), queryset=ClusterType.objects.all(), @@ -77,25 +78,6 @@ class ClusterBulkEditForm(NetBoxModelBulkEditForm): queryset=Tenant.objects.all(), required=False ) - region = DynamicModelChoiceField( - label=_('Region'), - queryset=Region.objects.all(), - required=False, - ) - site_group = DynamicModelChoiceField( - label=_('Site group'), - queryset=SiteGroup.objects.all(), - required=False, - ) - site = DynamicModelChoiceField( - label=_('Site'), - queryset=Site.objects.all(), - required=False, - query_params={ - 'region_id': '$region', - 'group_id': '$site_group', - } - ) description = forms.CharField( label=_('Description'), max_length=200, @@ -106,10 +88,10 @@ class ClusterBulkEditForm(NetBoxModelBulkEditForm): model = Cluster fieldsets = ( FieldSet('type', 'group', 'status', 'tenant', 'description'), - FieldSet('region', 'site_group', 'site', name=_('Site')), + FieldSet('scope_type', 'scope', name=_('Scope')), ) nullable_fields = ( - 'group', 'site', 'tenant', 'description', 'comments', + 'group', 'scope', 'tenant', 'description', 'comments', ) diff --git a/netbox/virtualization/forms/bulk_import.py b/netbox/virtualization/forms/bulk_import.py index 17efc567a..9ccdd68f7 100644 --- a/netbox/virtualization/forms/bulk_import.py +++ b/netbox/virtualization/forms/bulk_import.py @@ -1,6 +1,7 @@ from django.utils.translation import gettext_lazy as _ from dcim.choices import InterfaceModeChoices +from dcim.forms.mixins import ScopedImportForm from dcim.models import Device, DeviceRole, Platform, Site from extras.models import ConfigTemplate from ipam.models import VRF @@ -36,7 +37,7 @@ class ClusterGroupImportForm(NetBoxModelImportForm): fields = ('name', 'slug', 'description', 'tags') -class ClusterImportForm(NetBoxModelImportForm): +class ClusterImportForm(ScopedImportForm, NetBoxModelImportForm): type = CSVModelChoiceField( label=_('Type'), queryset=ClusterType.objects.all(), @@ -72,7 +73,10 @@ class ClusterImportForm(NetBoxModelImportForm): class Meta: model = Cluster - fields = ('name', 'type', 'group', 'status', 'site', 'tenant', 'description', 'comments', 'tags') + fields = ('name', 'type', 'group', 'status', 'scope_type', 'scope_id', 'tenant', 'description', 'comments', 'tags') + labels = { + 'scope_id': _('Scope ID'), + } class VirtualMachineImportForm(NetBoxModelImportForm): diff --git a/netbox/virtualization/forms/filtersets.py b/netbox/virtualization/forms/filtersets.py index 7c040d948..695641e4e 100644 --- a/netbox/virtualization/forms/filtersets.py +++ b/netbox/virtualization/forms/filtersets.py @@ -1,7 +1,7 @@ from django import forms from django.utils.translation import gettext_lazy as _ -from dcim.models import Device, DeviceRole, Platform, Region, Site, SiteGroup +from dcim.models import Device, DeviceRole, Location, Platform, Region, Site, SiteGroup from extras.forms import LocalConfigContextFilterForm from extras.models import ConfigTemplate from ipam.models import VRF @@ -43,7 +43,7 @@ class ClusterFilterForm(TenancyFilterForm, ContactModelFilterForm, NetBoxModelFi fieldsets = ( FieldSet('q', 'filter_id', 'tag'), FieldSet('group_id', 'type_id', 'status', name=_('Attributes')), - FieldSet('region_id', 'site_group_id', 'site_id', name=_('Location')), + FieldSet('region_id', 'site_group_id', 'site_id', 'location_id', name=_('Scope')), FieldSet('tenant_group_id', 'tenant_id', name=_('Tenant')), FieldSet('contact', 'contact_role', 'contact_group', name=_('Contacts')), ) @@ -58,11 +58,6 @@ class ClusterFilterForm(TenancyFilterForm, ContactModelFilterForm, NetBoxModelFi required=False, label=_('Region') ) - status = forms.MultipleChoiceField( - label=_('Status'), - choices=ClusterStatusChoices, - required=False - ) site_group_id = DynamicModelMultipleChoiceField( queryset=SiteGroup.objects.all(), required=False, @@ -78,6 +73,16 @@ class ClusterFilterForm(TenancyFilterForm, ContactModelFilterForm, NetBoxModelFi }, label=_('Site') ) + location_id = DynamicModelMultipleChoiceField( + queryset=Location.objects.all(), + required=False, + label=_('Location') + ) + status = forms.MultipleChoiceField( + label=_('Status'), + choices=ClusterStatusChoices, + required=False + ) group_id = DynamicModelMultipleChoiceField( queryset=ClusterGroup.objects.all(), required=False, diff --git a/netbox/virtualization/forms/model_forms.py b/netbox/virtualization/forms/model_forms.py index 4527e7f4c..44c67d389 100644 --- a/netbox/virtualization/forms/model_forms.py +++ b/netbox/virtualization/forms/model_forms.py @@ -4,6 +4,7 @@ from django.core.exceptions import ValidationError from django.utils.translation import gettext_lazy as _ from dcim.forms.common import InterfaceCommonForm +from dcim.forms.mixins import ScopedForm from dcim.models import Device, DeviceRole, Platform, Rack, Region, Site, SiteGroup from extras.models import ConfigTemplate from ipam.choices import VLANQinQRoleChoices @@ -58,7 +59,7 @@ class ClusterGroupForm(NetBoxModelForm): ) -class ClusterForm(TenancyForm, NetBoxModelForm): +class ClusterForm(TenancyForm, ScopedForm, NetBoxModelForm): type = DynamicModelChoiceField( label=_('Type'), queryset=ClusterType.objects.all() @@ -68,23 +69,18 @@ class ClusterForm(TenancyForm, NetBoxModelForm): queryset=ClusterGroup.objects.all(), required=False ) - site = DynamicModelChoiceField( - label=_('Site'), - queryset=Site.objects.all(), - required=False, - selector=True - ) comments = CommentField() fieldsets = ( - FieldSet('name', 'type', 'group', 'site', 'status', 'description', 'tags', name=_('Cluster')), + FieldSet('name', 'type', 'group', 'status', 'description', 'tags', name=_('Cluster')), + FieldSet('scope_type', 'scope', name=_('Scope')), FieldSet('tenant_group', 'tenant', name=_('Tenancy')), ) class Meta: model = Cluster fields = ( - 'name', 'type', 'group', 'status', 'tenant', 'site', 'description', 'comments', 'tags', + 'name', 'type', 'group', 'status', 'tenant', 'scope_type', 'description', 'comments', 'tags', ) diff --git a/netbox/virtualization/graphql/types.py b/netbox/virtualization/graphql/types.py index 79b5cb216..f51e0e3f5 100644 --- a/netbox/virtualization/graphql/types.py +++ b/netbox/virtualization/graphql/types.py @@ -1,4 +1,4 @@ -from typing import Annotated, List +from typing import Annotated, List, Union import strawberry import strawberry_django @@ -31,18 +31,25 @@ class ComponentType(NetBoxObjectType): @strawberry_django.type( models.Cluster, - fields='__all__', + exclude=('scope_type', 'scope_id', '_location', '_region', '_site', '_site_group'), filters=ClusterFilter ) class ClusterType(VLANGroupsMixin, NetBoxObjectType): type: Annotated["ClusterTypeType", strawberry.lazy('virtualization.graphql.types')] | None group: Annotated["ClusterGroupType", strawberry.lazy('virtualization.graphql.types')] | None tenant: Annotated["TenantType", strawberry.lazy('tenancy.graphql.types')] | None - site: Annotated["SiteType", strawberry.lazy('dcim.graphql.types')] | None - virtual_machines: List[Annotated["VirtualMachineType", strawberry.lazy('virtualization.graphql.types')]] devices: List[Annotated["DeviceType", strawberry.lazy('dcim.graphql.types')]] + @strawberry_django.field + def scope(self) -> Annotated[Union[ + Annotated["LocationType", strawberry.lazy('dcim.graphql.types')], + Annotated["RegionType", strawberry.lazy('dcim.graphql.types')], + Annotated["SiteGroupType", strawberry.lazy('dcim.graphql.types')], + Annotated["SiteType", strawberry.lazy('dcim.graphql.types')], + ], strawberry.union("ClusterScopeType")] | None: + return self.scope + @strawberry_django.type( models.ClusterGroup, diff --git a/netbox/virtualization/migrations/0044_cluster_scope.py b/netbox/virtualization/migrations/0044_cluster_scope.py new file mode 100644 index 000000000..63a888ac3 --- /dev/null +++ b/netbox/virtualization/migrations/0044_cluster_scope.py @@ -0,0 +1,51 @@ +import django.db.models.deletion +from django.db import migrations, models + + +def copy_site_assignments(apps, schema_editor): + """ + Copy site ForeignKey values to the scope GFK. + """ + ContentType = apps.get_model('contenttypes', 'ContentType') + Cluster = apps.get_model('virtualization', 'Cluster') + Site = apps.get_model('dcim', 'Site') + + Cluster.objects.filter(site__isnull=False).update( + scope_type=ContentType.objects.get_for_model(Site), + scope_id=models.F('site_id') + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('virtualization', '0043_qinq_svlan'), + ] + + operations = [ + migrations.AddField( + model_name='cluster', + name='scope_id', + field=models.PositiveBigIntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='cluster', + name='scope_type', + field=models.ForeignKey( + blank=True, + limit_choices_to=models.Q(('model__in', ('region', 'sitegroup', 'site', 'location'))), + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name='+', + to='contenttypes.contenttype', + ), + ), + + # Copy over existing site assignments + migrations.RunPython( + code=copy_site_assignments, + reverse_code=migrations.RunPython.noop + ), + + ] diff --git a/netbox/virtualization/migrations/0045_clusters_cached_relations.py b/netbox/virtualization/migrations/0045_clusters_cached_relations.py new file mode 100644 index 000000000..ff851aa7c --- /dev/null +++ b/netbox/virtualization/migrations/0045_clusters_cached_relations.py @@ -0,0 +1,94 @@ +import django.db.models.deletion +from django.db import migrations, models + + +def populate_denormalized_fields(apps, schema_editor): + """ + Copy the denormalized fields for _region, _site_group and _site from existing site field. + """ + Cluster = apps.get_model('virtualization', 'Cluster') + + clusters = Cluster.objects.filter(site__isnull=False).prefetch_related('site') + for cluster in clusters: + cluster._region_id = cluster.site.region_id + cluster._site_group_id = cluster.site.group_id + cluster._site_id = cluster.site_id + # Note: Location cannot be set prior to migration + + Cluster.objects.bulk_update(clusters, ['_region', '_site_group', '_site']) + + +class Migration(migrations.Migration): + + dependencies = [ + ('virtualization', '0044_cluster_scope'), + ] + + operations = [ + migrations.AddField( + model_name='cluster', + name='_location', + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='_%(class)ss', + to='dcim.location', + ), + ), + migrations.AddField( + model_name='cluster', + name='_region', + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='_%(class)ss', + to='dcim.region', + ), + ), + migrations.AddField( + model_name='cluster', + name='_site', + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='_%(class)ss', + to='dcim.site', + ), + ), + migrations.AddField( + model_name='cluster', + name='_site_group', + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='_%(class)ss', + to='dcim.sitegroup', + ), + ), + + # Populate denormalized FK values + migrations.RunPython( + code=populate_denormalized_fields, + reverse_code=migrations.RunPython.noop + ), + + migrations.RemoveConstraint( + model_name='cluster', + name='virtualization_cluster_unique_site_name', + ), + # Delete the site ForeignKey + migrations.RemoveField( + model_name='cluster', + name='site', + ), + migrations.AddConstraint( + model_name='cluster', + constraint=models.UniqueConstraint( + fields=('_site', 'name'), name='virtualization_cluster_unique__site_name' + ), + ), + ] diff --git a/netbox/virtualization/models/clusters.py b/netbox/virtualization/models/clusters.py index b8921c603..601ee7f23 100644 --- a/netbox/virtualization/models/clusters.py +++ b/netbox/virtualization/models/clusters.py @@ -1,9 +1,11 @@ +from django.apps import apps from django.contrib.contenttypes.fields import GenericRelation from django.core.exceptions import ValidationError from django.db import models from django.utils.translation import gettext_lazy as _ from dcim.models import Device +from dcim.models.mixins import CachedScopeMixin from netbox.models import OrganizationalModel, PrimaryModel from netbox.models.features import ContactsMixin from virtualization.choices import * @@ -42,7 +44,7 @@ class ClusterGroup(ContactsMixin, OrganizationalModel): verbose_name_plural = _('cluster groups') -class Cluster(ContactsMixin, PrimaryModel): +class Cluster(ContactsMixin, CachedScopeMixin, PrimaryModel): """ A cluster of VirtualMachines. Each Cluster may optionally be associated with one or more Devices. """ @@ -76,13 +78,6 @@ class Cluster(ContactsMixin, PrimaryModel): blank=True, null=True ) - site = models.ForeignKey( - to='dcim.Site', - on_delete=models.PROTECT, - related_name='clusters', - blank=True, - null=True - ) # Generic relations vlan_groups = GenericRelation( @@ -93,7 +88,7 @@ class Cluster(ContactsMixin, PrimaryModel): ) clone_fields = ( - 'type', 'group', 'status', 'tenant', 'site', + 'scope_type', 'scope_id', 'type', 'group', 'status', 'tenant', ) prerequisite_models = ( 'virtualization.ClusterType', @@ -107,8 +102,8 @@ class Cluster(ContactsMixin, PrimaryModel): name='%(app_label)s_%(class)s_unique_group_name' ), models.UniqueConstraint( - fields=('site', 'name'), - name='%(app_label)s_%(class)s_unique_site_name' + fields=('_site', 'name'), + name='%(app_label)s_%(class)s_unique__site_name' ), ) verbose_name = _('cluster') @@ -123,11 +118,28 @@ class Cluster(ContactsMixin, PrimaryModel): def clean(self): super().clean() + site = location = None + if self.scope_type: + scope_type = self.scope_type.model_class() + if scope_type == apps.get_model('dcim', 'site'): + site = self.scope + elif scope_type == apps.get_model('dcim', 'location'): + location = self.scope + site = location.site + # If the Cluster is assigned to a Site, verify that all host Devices belong to that Site. - if not self._state.adding and self.site: - if nonsite_devices := Device.objects.filter(cluster=self).exclude(site=self.site).count(): - raise ValidationError({ - 'site': _( - "{count} devices are assigned as hosts for this cluster but are not in site {site}" - ).format(count=nonsite_devices, site=self.site) - }) + if not self._state.adding: + if site: + if nonsite_devices := Device.objects.filter(cluster=self).exclude(site=site).count(): + raise ValidationError({ + 'scope': _( + "{count} devices are assigned as hosts for this cluster but are not in site {site}" + ).format(count=nonsite_devices, site=site) + }) + if location: + if nonlocation_devices := Device.objects.filter(cluster=self).exclude(location=location).count(): + raise ValidationError({ + 'scope': _( + "{count} devices are assigned as hosts for this cluster but are not in location {location}" + ).format(count=nonlocation_devices, location=location) + }) diff --git a/netbox/virtualization/models/virtualmachines.py b/netbox/virtualization/models/virtualmachines.py index da8419e88..4ee41e403 100644 --- a/netbox/virtualization/models/virtualmachines.py +++ b/netbox/virtualization/models/virtualmachines.py @@ -181,7 +181,7 @@ class VirtualMachine(ContactsMixin, ImageAttachmentsMixin, RenderConfigMixin, Co }) # Validate site for cluster & VM - if self.cluster and self.site and self.cluster.site and self.cluster.site != self.site: + if self.cluster and self.site and self.cluster._site and self.cluster._site != self.site: raise ValidationError({ 'cluster': _( 'The selected cluster ({cluster}) is not assigned to this site ({site}).' @@ -238,7 +238,7 @@ class VirtualMachine(ContactsMixin, ImageAttachmentsMixin, RenderConfigMixin, Co # Assign site from cluster if not set if self.cluster and not self.site: - self.site = self.cluster.site + self.site = self.cluster._site super().save(*args, **kwargs) diff --git a/netbox/virtualization/tables/clusters.py b/netbox/virtualization/tables/clusters.py index d3c799fb9..91807e35b 100644 --- a/netbox/virtualization/tables/clusters.py +++ b/netbox/virtualization/tables/clusters.py @@ -73,8 +73,11 @@ class ClusterTable(TenancyColumnsMixin, ContactsColumnMixin, NetBoxTable): status = columns.ChoiceFieldColumn( verbose_name=_('Status'), ) - site = tables.Column( - verbose_name=_('Site'), + scope_type = columns.ContentTypeColumn( + verbose_name=_('Scope Type'), + ) + scope = tables.Column( + verbose_name=_('Scope'), linkify=True ) device_count = columns.LinkedCountColumn( @@ -97,7 +100,7 @@ class ClusterTable(TenancyColumnsMixin, ContactsColumnMixin, NetBoxTable): class Meta(NetBoxTable.Meta): model = Cluster fields = ( - 'pk', 'id', 'name', 'type', 'group', 'status', 'tenant', 'tenant_group', 'site', 'description', 'comments', - 'device_count', 'vm_count', 'contacts', 'tags', 'created', 'last_updated', + 'pk', 'id', 'name', 'type', 'group', 'status', 'tenant', 'tenant_group', 'scope', 'scope_type', 'description', + 'comments', 'device_count', 'vm_count', 'contacts', 'tags', 'created', 'last_updated', ) default_columns = ('pk', 'name', 'type', 'group', 'status', 'tenant', 'site', 'device_count', 'vm_count') diff --git a/netbox/virtualization/tests/test_api.py b/netbox/virtualization/tests/test_api.py index 521064fc6..149b64684 100644 --- a/netbox/virtualization/tests/test_api.py +++ b/netbox/virtualization/tests/test_api.py @@ -113,7 +113,8 @@ class ClusterTest(APIViewTestCases.APIViewTestCase): Cluster(name='Cluster 2', type=cluster_types[0], group=cluster_groups[0], status=ClusterStatusChoices.STATUS_PLANNED), Cluster(name='Cluster 3', type=cluster_types[0], group=cluster_groups[0], status=ClusterStatusChoices.STATUS_PLANNED), ) - Cluster.objects.bulk_create(clusters) + for cluster in clusters: + cluster.save() cls.create_data = [ { @@ -157,11 +158,12 @@ class VirtualMachineTest(APIViewTestCases.APIViewTestCase): Site.objects.bulk_create(sites) clusters = ( - Cluster(name='Cluster 1', type=clustertype, site=sites[0], group=clustergroup), - Cluster(name='Cluster 2', type=clustertype, site=sites[1], group=clustergroup), + Cluster(name='Cluster 1', type=clustertype, scope=sites[0], group=clustergroup), + Cluster(name='Cluster 2', type=clustertype, scope=sites[1], group=clustergroup), Cluster(name='Cluster 3', type=clustertype), ) - Cluster.objects.bulk_create(clusters) + for cluster in clusters: + cluster.save() device1 = create_test_device('device1', site=sites[0], cluster=clusters[0]) device2 = create_test_device('device2', site=sites[1], cluster=clusters[1]) diff --git a/netbox/virtualization/tests/test_filtersets.py b/netbox/virtualization/tests/test_filtersets.py index 0c7079bba..5a5bf2325 100644 --- a/netbox/virtualization/tests/test_filtersets.py +++ b/netbox/virtualization/tests/test_filtersets.py @@ -138,7 +138,7 @@ class ClusterTestCase(TestCase, ChangeLoggedFilterSetTests): type=cluster_types[0], group=cluster_groups[0], status=ClusterStatusChoices.STATUS_PLANNED, - site=sites[0], + scope=sites[0], tenant=tenants[0], description='foobar1' ), @@ -147,7 +147,7 @@ class ClusterTestCase(TestCase, ChangeLoggedFilterSetTests): type=cluster_types[1], group=cluster_groups[1], status=ClusterStatusChoices.STATUS_STAGING, - site=sites[1], + scope=sites[1], tenant=tenants[1], description='foobar2' ), @@ -156,12 +156,13 @@ class ClusterTestCase(TestCase, ChangeLoggedFilterSetTests): type=cluster_types[2], group=cluster_groups[2], status=ClusterStatusChoices.STATUS_ACTIVE, - site=sites[2], + scope=sites[2], tenant=tenants[2], description='foobar3' ), ) - Cluster.objects.bulk_create(clusters) + for cluster in clusters: + cluster.save() def test_q(self): params = {'q': 'foobar1'} @@ -274,11 +275,12 @@ class VirtualMachineTestCase(TestCase, ChangeLoggedFilterSetTests): Site.objects.bulk_create(sites) clusters = ( - Cluster(name='Cluster 1', type=cluster_types[0], group=cluster_groups[0], site=sites[0]), - Cluster(name='Cluster 2', type=cluster_types[1], group=cluster_groups[1], site=sites[1]), - Cluster(name='Cluster 3', type=cluster_types[2], group=cluster_groups[2], site=sites[2]), + Cluster(name='Cluster 1', type=cluster_types[0], group=cluster_groups[0], scope=sites[0]), + Cluster(name='Cluster 2', type=cluster_types[1], group=cluster_groups[1], scope=sites[1]), + Cluster(name='Cluster 3', type=cluster_types[2], group=cluster_groups[2], scope=sites[2]), ) - Cluster.objects.bulk_create(clusters) + for cluster in clusters: + cluster.save() platforms = ( Platform(name='Platform 1', slug='platform-1'), diff --git a/netbox/virtualization/tests/test_models.py b/netbox/virtualization/tests/test_models.py index a4e8d7947..7be423bf1 100644 --- a/netbox/virtualization/tests/test_models.py +++ b/netbox/virtualization/tests/test_models.py @@ -54,11 +54,12 @@ class VirtualMachineTestCase(TestCase): Site.objects.bulk_create(sites) clusters = ( - Cluster(name='Cluster 1', type=cluster_type, site=sites[0]), - Cluster(name='Cluster 2', type=cluster_type, site=sites[1]), - Cluster(name='Cluster 3', type=cluster_type, site=None), + Cluster(name='Cluster 1', type=cluster_type, scope=sites[0]), + Cluster(name='Cluster 2', type=cluster_type, scope=sites[1]), + Cluster(name='Cluster 3', type=cluster_type, scope=None), ) - Cluster.objects.bulk_create(clusters) + for cluster in clusters: + cluster.save() # VM with site only should pass VirtualMachine(name='vm1', site=sites[0]).full_clean() diff --git a/netbox/virtualization/tests/test_views.py b/netbox/virtualization/tests/test_views.py index 3c6a058c9..b9cb7b437 100644 --- a/netbox/virtualization/tests/test_views.py +++ b/netbox/virtualization/tests/test_views.py @@ -1,3 +1,4 @@ +from django.contrib.contenttypes.models import ContentType from django.test import override_settings from django.urls import reverse from netaddr import EUI @@ -117,11 +118,12 @@ class ClusterTestCase(ViewTestCases.PrimaryObjectViewTestCase): ClusterType.objects.bulk_create(clustertypes) clusters = ( - Cluster(name='Cluster 1', group=clustergroups[0], type=clustertypes[0], status=ClusterStatusChoices.STATUS_ACTIVE, site=sites[0]), - Cluster(name='Cluster 2', group=clustergroups[0], type=clustertypes[0], status=ClusterStatusChoices.STATUS_ACTIVE, site=sites[0]), - Cluster(name='Cluster 3', group=clustergroups[0], type=clustertypes[0], status=ClusterStatusChoices.STATUS_ACTIVE, site=sites[0]), + Cluster(name='Cluster 1', group=clustergroups[0], type=clustertypes[0], status=ClusterStatusChoices.STATUS_ACTIVE, scope=sites[0]), + Cluster(name='Cluster 2', group=clustergroups[0], type=clustertypes[0], status=ClusterStatusChoices.STATUS_ACTIVE, scope=sites[0]), + Cluster(name='Cluster 3', group=clustergroups[0], type=clustertypes[0], status=ClusterStatusChoices.STATUS_ACTIVE, scope=sites[0]), ) - Cluster.objects.bulk_create(clusters) + for cluster in clusters: + cluster.save() tags = create_tags('Alpha', 'Bravo', 'Charlie') @@ -131,7 +133,8 @@ class ClusterTestCase(ViewTestCases.PrimaryObjectViewTestCase): 'type': clustertypes[1].pk, 'status': ClusterStatusChoices.STATUS_OFFLINE, 'tenant': None, - 'site': sites[1].pk, + 'scope_type': ContentType.objects.get_for_model(Site).pk, + 'scope': sites[1].pk, 'comments': 'Some comments', 'tags': [t.pk for t in tags], } @@ -155,7 +158,6 @@ class ClusterTestCase(ViewTestCases.PrimaryObjectViewTestCase): 'type': clustertypes[1].pk, 'status': ClusterStatusChoices.STATUS_OFFLINE, 'tenant': None, - 'site': sites[1].pk, 'comments': 'New comments', } @@ -201,10 +203,11 @@ class VirtualMachineTestCase(ViewTestCases.PrimaryObjectViewTestCase): clustertype = ClusterType.objects.create(name='Cluster Type 1', slug='cluster-type-1') clusters = ( - Cluster(name='Cluster 1', type=clustertype, site=sites[0]), - Cluster(name='Cluster 2', type=clustertype, site=sites[1]), + Cluster(name='Cluster 1', type=clustertype, scope=sites[0]), + Cluster(name='Cluster 2', type=clustertype, scope=sites[1]), ) - Cluster.objects.bulk_create(clusters) + for cluster in clusters: + cluster.save() devices = ( create_test_device('device1', site=sites[0], cluster=clusters[0]), @@ -292,7 +295,7 @@ class VMInterfaceTestCase(ViewTestCases.DeviceComponentViewTestCase): site = Site.objects.create(name='Site 1', slug='site-1') role = DeviceRole.objects.create(name='Device Role 1', slug='device-role-1') clustertype = ClusterType.objects.create(name='Cluster Type 1', slug='cluster-type-1') - cluster = Cluster.objects.create(name='Cluster 1', type=clustertype, site=site) + cluster = Cluster.objects.create(name='Cluster 1', type=clustertype, scope=site) virtualmachines = ( VirtualMachine(name='Virtual Machine 1', site=site, cluster=cluster, role=role), VirtualMachine(name='Virtual Machine 2', site=site, cluster=cluster, role=role), From 4bba92617d88f1eb63e9fc8894d54a928a34d9f2 Mon Sep 17 00:00:00 2001 From: Alexander Haase Date: Fri, 1 Nov 2024 19:56:08 +0100 Subject: [PATCH 3/3] Closes #16971: Add system jobs (#17716) * Fix check for existing jobs If a job is to be enqueued once and no specific scheduled time is specified, any scheduled time of existing jobs will be valid. Only if a specific scheduled time is specified for 'enqueue_once()' can it be evaluated. * Allow system jobs to be registered A new registry key allows background system jobs to be registered and automatically scheduled when rqworker starts. * Test scheduling of system jobs * Fix plugins scheduled job documentation The documentation reflected a non-production state of the JobRunner framework left over from development. Now a more practical example demonstrates the usage. * Allow plugins to register system jobs * Rename system job metadata To clarify which meta-attributes belong to system jobs, each of them is now prefixed with 'system_'. * Add predefined job interval choices * Remove 'system_enabled' JobRunner attribute Previously, the 'system_enabled' attribute was used to control whether a job should run or not. However, this can also be accomplished by evaluating the job's interval. * Fix test * Use a decorator to register system jobs * Specify interval when registering system job * Update documentation --------- Co-authored-by: Jeremy Stretch --- docs/plugins/development/background-jobs.md | 53 +++++++++++++++----- docs/plugins/development/data-backends.md | 2 +- netbox/core/choices.py | 14 ++++++ netbox/core/management/commands/rqworker.py | 11 ++++ netbox/netbox/jobs.py | 21 +++++++- netbox/netbox/registry.py | 1 + netbox/netbox/tests/dummy_plugin/__init__.py | 5 ++ netbox/netbox/tests/dummy_plugin/jobs.py | 9 ++++ netbox/netbox/tests/test_jobs.py | 36 +++++++++++++ netbox/netbox/tests/test_plugins.py | 9 ++++ 10 files changed, 147 insertions(+), 14 deletions(-) create mode 100644 netbox/netbox/tests/dummy_plugin/jobs.py diff --git a/docs/plugins/development/background-jobs.md b/docs/plugins/development/background-jobs.md index 873390a58..d51981b9e 100644 --- a/docs/plugins/development/background-jobs.md +++ b/docs/plugins/development/background-jobs.md @@ -29,6 +29,9 @@ class MyTestJob(JobRunner): You can schedule the background job from within your code (e.g. from a model's `save()` method or a view) by calling `MyTestJob.enqueue()`. This method passes through all arguments to `Job.enqueue()`. However, no `name` argument must be passed, as the background job name will be used instead. +!!! tip + A set of predefined intervals is available at `core.choices.JobIntervalChoices` for convenience. + ### Attributes `JobRunner` attributes are defined under a class named `Meta` within the job. These are optional, but encouraged. @@ -46,26 +49,52 @@ As described above, jobs can be scheduled for immediate execution or at any late #### Example +```python title="models.py" +from django.db import models +from core.choices import JobIntervalChoices +from netbox.models import NetBoxModel +from .jobs import MyTestJob + +class MyModel(NetBoxModel): + foo = models.CharField() + + def save(self, *args, **kwargs): + MyTestJob.enqueue_once(instance=self, interval=JobIntervalChoices.INTERVAL_HOURLY) + return super().save(*args, **kwargs) + + def sync(self): + MyTestJob.enqueue(instance=self) +``` + + +### System Jobs + +Some plugins may implement background jobs that are decoupled from the request/response cycle. Typical use cases would be housekeeping tasks or synchronization jobs. These can be registered as _system jobs_ using the `system_job()` decorator. The job interval must be passed as an integer (in minutes) when registering a system job. System jobs are scheduled automatically when the RQ worker (`manage.py rqworker`) is run. + +#### Example + ```python title="jobs.py" -from netbox.jobs import JobRunner - +from core.choices import JobIntervalChoices +from netbox.jobs import JobRunner, system_job +from .models import MyModel +# Specify a predefined choice or an integer indicating +# the number of minutes between job executions +@system_job(interval=JobIntervalChoices.INTERVAL_HOURLY) class MyHousekeepingJob(JobRunner): class Meta: - name = "Housekeeping" + name = "My Housekeeping Job" def run(self, *args, **kwargs): - # your logic goes here + MyModel.objects.filter(foo='bar').delete() + +system_jobs = ( + MyHousekeepingJob, +) ``` -```python title="__init__.py" -from netbox.plugins import PluginConfig - -class MyPluginConfig(PluginConfig): - def ready(self): - from .jobs import MyHousekeepingJob - MyHousekeepingJob.setup(interval=60) -``` +!!! note + Ensure that any system jobs are imported on initialization. Otherwise, they won't be registered. This can be achieved by extending the PluginConfig's `ready()` method. ## Task queues diff --git a/docs/plugins/development/data-backends.md b/docs/plugins/development/data-backends.md index 8b7226a41..0c6d44d95 100644 --- a/docs/plugins/development/data-backends.md +++ b/docs/plugins/development/data-backends.md @@ -18,6 +18,6 @@ backends = [MyDataBackend] ``` !!! tip - The path to the list of search indexes can be modified by setting `data_backends` in the PluginConfig instance. + The path to the list of data backends can be modified by setting `data_backends` in the PluginConfig instance. ::: netbox.data_backends.DataBackend diff --git a/netbox/core/choices.py b/netbox/core/choices.py index 01a072ce1..442acc26b 100644 --- a/netbox/core/choices.py +++ b/netbox/core/choices.py @@ -72,6 +72,20 @@ class JobStatusChoices(ChoiceSet): ) +class JobIntervalChoices(ChoiceSet): + INTERVAL_MINUTELY = 1 + INTERVAL_HOURLY = 60 + INTERVAL_DAILY = 60 * 24 + INTERVAL_WEEKLY = 60 * 24 * 7 + + CHOICES = ( + (INTERVAL_MINUTELY, _('Minutely')), + (INTERVAL_HOURLY, _('Hourly')), + (INTERVAL_DAILY, _('Daily')), + (INTERVAL_WEEKLY, _('Weekly')), + ) + + # # ObjectChanges # diff --git a/netbox/core/management/commands/rqworker.py b/netbox/core/management/commands/rqworker.py index e1fb6fd11..b2879c3d8 100644 --- a/netbox/core/management/commands/rqworker.py +++ b/netbox/core/management/commands/rqworker.py @@ -2,6 +2,8 @@ import logging from django_rq.management.commands.rqworker import Command as _Command +from netbox.registry import registry + DEFAULT_QUEUES = ('high', 'default', 'low') @@ -14,6 +16,15 @@ class Command(_Command): of only the 'default' queue). """ def handle(self, *args, **options): + # Setup system jobs. + for job, kwargs in registry['system_jobs'].items(): + try: + interval = kwargs['interval'] + except KeyError: + raise TypeError("System job must specify an interval (in minutes).") + logger.debug(f"Scheduling system job {job.name} (interval={interval})") + job.enqueue_once(**kwargs) + # Run the worker with scheduler functionality options['with_scheduler'] = True diff --git a/netbox/netbox/jobs.py b/netbox/netbox/jobs.py index ae8f2f109..965ebc9e9 100644 --- a/netbox/netbox/jobs.py +++ b/netbox/netbox/jobs.py @@ -2,6 +2,7 @@ import logging from abc import ABC, abstractmethod from datetime import timedelta +from django.core.exceptions import ImproperlyConfigured from django.utils.functional import classproperty from django_pglocks import advisory_lock from rq.timeouts import JobTimeoutException @@ -9,12 +10,30 @@ from rq.timeouts import JobTimeoutException from core.choices import JobStatusChoices from core.models import Job, ObjectType from netbox.constants import ADVISORY_LOCK_KEYS +from netbox.registry import registry __all__ = ( 'JobRunner', + 'system_job', ) +def system_job(interval): + """ + Decorator for registering a `JobRunner` class as system background job. + """ + if type(interval) is not int: + raise ImproperlyConfigured("System job interval must be an integer (minutes).") + + def _wrapper(cls): + registry['system_jobs'][cls] = { + 'interval': interval + } + return cls + + return _wrapper + + class JobRunner(ABC): """ Background Job helper class. @@ -129,7 +148,7 @@ class JobRunner(ABC): if job: # If the job parameters haven't changed, don't schedule a new job and keep the current schedule. Otherwise, # delete the existing job and schedule a new job instead. - if (schedule_at and job.scheduled == schedule_at) and (job.interval == interval): + if (not schedule_at or job.scheduled == schedule_at) and (job.interval == interval): return job job.delete() diff --git a/netbox/netbox/registry.py b/netbox/netbox/registry.py index 0920cbccf..48d7921f2 100644 --- a/netbox/netbox/registry.py +++ b/netbox/netbox/registry.py @@ -30,6 +30,7 @@ registry = Registry({ 'models': collections.defaultdict(set), 'plugins': dict(), 'search': dict(), + 'system_jobs': dict(), 'tables': collections.defaultdict(dict), 'views': collections.defaultdict(dict), 'widgets': dict(), diff --git a/netbox/netbox/tests/dummy_plugin/__init__.py b/netbox/netbox/tests/dummy_plugin/__init__.py index 6ab62d638..2ca7c290c 100644 --- a/netbox/netbox/tests/dummy_plugin/__init__.py +++ b/netbox/netbox/tests/dummy_plugin/__init__.py @@ -21,5 +21,10 @@ class DummyPluginConfig(PluginConfig): 'netbox.tests.dummy_plugin.events.process_events_queue' ] + def ready(self): + super().ready() + + from . import jobs # noqa: F401 + config = DummyPluginConfig diff --git a/netbox/netbox/tests/dummy_plugin/jobs.py b/netbox/netbox/tests/dummy_plugin/jobs.py new file mode 100644 index 000000000..3b9dc7a5f --- /dev/null +++ b/netbox/netbox/tests/dummy_plugin/jobs.py @@ -0,0 +1,9 @@ +from core.choices import JobIntervalChoices +from netbox.jobs import JobRunner, system_job + + +@system_job(interval=JobIntervalChoices.INTERVAL_HOURLY) +class DummySystemJob(JobRunner): + + def run(self, *args, **kwargs): + pass diff --git a/netbox/netbox/tests/test_jobs.py b/netbox/netbox/tests/test_jobs.py index 52a7bd97a..e3e24a235 100644 --- a/netbox/netbox/tests/test_jobs.py +++ b/netbox/netbox/tests/test_jobs.py @@ -90,6 +90,15 @@ class EnqueueTest(JobRunnerTestCase): self.assertEqual(job1, job2) self.assertEqual(TestJobRunner.get_jobs(instance).count(), 1) + def test_enqueue_once_twice_same_no_schedule_at(self): + instance = DataSource() + schedule_at = self.get_schedule_at() + job1 = TestJobRunner.enqueue_once(instance, schedule_at=schedule_at) + job2 = TestJobRunner.enqueue_once(instance) + + self.assertEqual(job1, job2) + self.assertEqual(TestJobRunner.get_jobs(instance).count(), 1) + def test_enqueue_once_twice_different_schedule_at(self): instance = DataSource() job1 = TestJobRunner.enqueue_once(instance, schedule_at=self.get_schedule_at()) @@ -127,3 +136,30 @@ class EnqueueTest(JobRunnerTestCase): self.assertNotEqual(job1, job2) self.assertRaises(Job.DoesNotExist, job1.refresh_from_db) self.assertEqual(TestJobRunner.get_jobs(instance).count(), 1) + + +class SystemJobTest(JobRunnerTestCase): + """ + Test that system jobs can be scheduled. + + General functionality already tested by `JobRunnerTest` and `EnqueueTest`. + """ + + def test_scheduling(self): + # Can job be enqueued? + job = TestJobRunner.enqueue(schedule_at=self.get_schedule_at()) + self.assertIsInstance(job, Job) + self.assertEqual(TestJobRunner.get_jobs().count(), 1) + + # Can job be deleted again? + job.delete() + self.assertRaises(Job.DoesNotExist, job.refresh_from_db) + self.assertEqual(TestJobRunner.get_jobs().count(), 0) + + def test_enqueue_once(self): + schedule_at = self.get_schedule_at() + job1 = TestJobRunner.enqueue_once(schedule_at=schedule_at) + job2 = TestJobRunner.enqueue_once(schedule_at=schedule_at) + + self.assertEqual(job1, job2) + self.assertEqual(TestJobRunner.get_jobs().count(), 1) diff --git a/netbox/netbox/tests/test_plugins.py b/netbox/netbox/tests/test_plugins.py index 16778667d..db82d0a75 100644 --- a/netbox/netbox/tests/test_plugins.py +++ b/netbox/netbox/tests/test_plugins.py @@ -5,8 +5,10 @@ from django.core.exceptions import ImproperlyConfigured from django.test import Client, TestCase, override_settings from django.urls import reverse +from core.choices import JobIntervalChoices from netbox.tests.dummy_plugin import config as dummy_config from netbox.tests.dummy_plugin.data_backends import DummyBackend +from netbox.tests.dummy_plugin.jobs import DummySystemJob from netbox.plugins.navigation import PluginMenu from netbox.plugins.utils import get_plugin_config from netbox.graphql.schema import Query @@ -130,6 +132,13 @@ class PluginTest(TestCase): self.assertIn('dummy', registry['data_backends']) self.assertIs(registry['data_backends']['dummy'], DummyBackend) + def test_system_jobs(self): + """ + Check registered system jobs. + """ + self.assertIn(DummySystemJob, registry['system_jobs']) + self.assertEqual(registry['system_jobs'][DummySystemJob]['interval'], JobIntervalChoices.INTERVAL_HOURLY) + def test_queues(self): """ Check that plugin queues are registered with the accurate name.