Closes #13428: Q-in-Q VLANs (#17822)

* Initial work on #13428 (QinQ)

* Misc cleanup; add tests for Q-in-Q fields

* Address PR feedback

netbox/dcim/api/serializers_/device_components.py

@@ -196,6 +196,7 @@ class InterfaceSerializer(NetBoxModelSerializer, CabledObjectSerializer, Connect
         required=False,
         many=True
     )
+    qinq_svlan = VLANSerializer(nested=True, required=False, allow_null=True)
     vlan_translation_policy = VLANTranslationPolicySerializer(nested=True, required=False, allow_null=True)
     vrf = VRFSerializer(nested=True, required=False, allow_null=True)
     l2vpn_termination = L2VPNTerminationSerializer(nested=True, read_only=True, allow_null=True)
@@ -223,10 +224,10 @@ class InterfaceSerializer(NetBoxModelSerializer, CabledObjectSerializer, Connect
             'id', 'url', 'display_url', 'display', 'device', 'vdcs', 'module', 'name', 'label', 'type', 'enabled',
             'parent', 'bridge', 'lag', 'mtu', 'mac_address', 'speed', 'duplex', 'wwn', 'mgmt_only', 'description',
             'mode', 'rf_role', 'rf_channel', 'poe_mode', 'poe_type', 'rf_channel_frequency', 'rf_channel_width',
-            'tx_power', 'untagged_vlan', 'tagged_vlans', 'mark_connected', 'cable', 'cable_end', 'wireless_link',
-            'link_peers', 'link_peers_type', 'wireless_lans', 'vrf', 'l2vpn_termination', 'connected_endpoints',
-            'connected_endpoints_type', 'connected_endpoints_reachable', 'tags', 'custom_fields', 'created',
-            'last_updated', 'count_ipaddresses', 'count_fhrp_groups', '_occupied', 'vlan_translation_policy'
+            'tx_power', 'untagged_vlan', 'tagged_vlans', 'qinq_svlan', 'vlan_translation_policy', 'mark_connected',
+            'cable', 'cable_end', 'wireless_link', 'link_peers', 'link_peers_type', 'wireless_lans', 'vrf',
+            'l2vpn_termination', 'connected_endpoints', 'connected_endpoints_type', 'connected_endpoints_reachable',
+            'tags', 'custom_fields', 'created', 'last_updated', 'count_ipaddresses', 'count_fhrp_groups', '_occupied',
         ]
         brief_fields = ('id', 'url', 'display', 'device', 'name', 'description', 'cable', '_occupied')
 

netbox/dcim/choices.py

@@ -1258,11 +1258,13 @@ class InterfaceModeChoices(ChoiceSet):
     MODE_ACCESS = 'access'
     MODE_TAGGED = 'tagged'
     MODE_TAGGED_ALL = 'tagged-all'
+    MODE_Q_IN_Q = 'q-in-q'
 
     CHOICES = (
         (MODE_ACCESS, _('Access')),
         (MODE_TAGGED, _('Tagged')),
         (MODE_TAGGED_ALL, _('Tagged (All)')),
+        (MODE_Q_IN_Q, _('Q-in-Q (802.1ad)')),
     )
 
 

netbox/dcim/filtersets.py

@@ -1647,7 +1647,8 @@ class CommonInterfaceFilterSet(django_filters.FilterSet):
             return queryset
         return queryset.filter(
             Q(untagged_vlan_id=value) |
-            Q(tagged_vlans=value)
+            Q(tagged_vlans=value) |
+            Q(qinq_svlan=value)
         )
 
     def filter_vlan(self, queryset, name, value):
@@ -1656,7 +1657,8 @@ class CommonInterfaceFilterSet(django_filters.FilterSet):
             return queryset
         return queryset.filter(
             Q(untagged_vlan_id__vid=value) |
-            Q(tagged_vlans__vid=value)
+            Q(tagged_vlans__vid=value) |
+            Q(qinq_svlan__vid=value)
         )
 
 

netbox/dcim/forms/common.py

@@ -37,6 +37,8 @@ class InterfaceCommonForm(forms.Form):
             del self.fields['vlan_group']
             del self.fields['untagged_vlan']
             del self.fields['tagged_vlans']
+        if interface_mode != InterfaceModeChoices.MODE_Q_IN_Q:
+            del self.fields['qinq_svlan']
 
     def clean(self):
         super().clean()

netbox/dcim/forms/model_forms.py

@@ -7,6 +7,7 @@ from dcim.choices import *
 from dcim.constants import *
 from dcim.models import *
 from extras.models import ConfigTemplate
+from ipam.choices import VLANQinQRoleChoices
 from ipam.models import ASN, IPAddress, VLAN, VLANGroup, VLANTranslationPolicy, VRF
 from netbox.forms import NetBoxModelForm
 from tenancy.forms import TenancyForm
@@ -1372,6 +1373,16 @@ class InterfaceForm(InterfaceCommonForm, ModularDeviceComponentForm):
             'available_on_device': '$device',
         }
     )
+    qinq_svlan = DynamicModelMultipleChoiceField(
+        queryset=VLAN.objects.all(),
+        required=False,
+        label=_('Q-in-Q Service VLAN'),
+        query_params={
+            'group_id': '$vlan_group',
+            'available_on_device': '$device',
+            'qinq_role': VLANQinQRoleChoices.ROLE_SERVICE,
+        }
+    )
     vrf = DynamicModelChoiceField(
         queryset=VRF.objects.all(),
         required=False,
@@ -1396,7 +1407,10 @@ class InterfaceForm(InterfaceCommonForm, ModularDeviceComponentForm):
         FieldSet('vdcs', 'mtu', 'tx_power', 'enabled', 'mgmt_only', 'mark_connected', name=_('Operation')),
         FieldSet('parent', 'bridge', 'lag', name=_('Related Interfaces')),
         FieldSet('poe_mode', 'poe_type', name=_('PoE')),
-        FieldSet('mode', 'vlan_group', 'untagged_vlan', 'tagged_vlans', 'vlan_translation_policy', name=_('802.1Q Switching')),
+        FieldSet(
+            'mode', 'vlan_group', 'untagged_vlan', 'tagged_vlans', 'qinq_svlan', 'vlan_translation_policy',
+            name=_('802.1Q Switching')
+        ),
         FieldSet(
             'rf_role', 'rf_channel', 'rf_channel_frequency', 'rf_channel_width', 'wireless_lan_group', 'wireless_lans',
             name=_('Wireless')
@@ -1409,7 +1423,7 @@ class InterfaceForm(InterfaceCommonForm, ModularDeviceComponentForm):
             'device', 'module', 'vdcs', 'name', 'label', 'type', 'speed', 'duplex', 'enabled', 'parent', 'bridge', 'lag',
             'mac_address', 'wwn', 'mtu', 'mgmt_only', 'mark_connected', 'description', 'poe_mode', 'poe_type', 'mode',
             'rf_role', 'rf_channel', 'rf_channel_frequency', 'rf_channel_width', 'tx_power', 'wireless_lans',
-            'untagged_vlan', 'tagged_vlans', 'vrf', 'tags', 'vlan_translation_policy',
+            'untagged_vlan', 'tagged_vlans', 'qinq_svlan', 'vlan_translation_policy', 'vrf', 'tags',
         ]
         widgets = {
             'speed': NumberWithOptions(

netbox/dcim/graphql/types.py

@@ -385,6 +385,7 @@ class InterfaceType(IPAddressesMixin, ModularComponentType, CabledObjectMixin, P
     wireless_link: Annotated["WirelessLinkType", strawberry.lazy('wireless.graphql.types')] | None
     untagged_vlan: Annotated["VLANType", strawberry.lazy('ipam.graphql.types')] | None
     vrf: Annotated["VRFType", strawberry.lazy('ipam.graphql.types')] | None
+    qinq_svlan: Annotated["VLANType", strawberry.lazy('ipam.graphql.types')] | None
     vlan_translation_policy: Annotated["VLANTranslationPolicyType", strawberry.lazy('ipam.graphql.types')] | None
 
     vdcs: List[Annotated["VirtualDeviceContextType", strawberry.lazy('dcim.graphql.types')]]

netbox/dcim/migrations/0196_qinq_svlan.py

@@ -0,0 +1,28 @@
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dcim', '0195_interface_vlan_translation_policy'),
+        ('ipam', '0075_vlan_qinq'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='interface',
+            name='qinq_svlan',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)ss_svlan', to='ipam.vlan'),
+        ),
+        migrations.AlterField(
+            model_name='interface',
+            name='tagged_vlans',
+            field=models.ManyToManyField(blank=True, related_name='%(class)ss_as_tagged', to='ipam.vlan'),
+        ),
+        migrations.AlterField(
+            model_name='interface',
+            name='untagged_vlan',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)ss_as_untagged', to='ipam.vlan'),
+        ),
+    ]

netbox/dcim/models/device_components.py

@@ -547,17 +547,48 @@ class BaseInterface(models.Model):
         blank=True,
         verbose_name=_('bridge interface')
     )
+    untagged_vlan = models.ForeignKey(
+        to='ipam.VLAN',
+        on_delete=models.SET_NULL,
+        related_name='%(class)ss_as_untagged',
+        null=True,
+        blank=True,
+        verbose_name=_('untagged VLAN')
+    )
+    tagged_vlans = models.ManyToManyField(
+        to='ipam.VLAN',
+        related_name='%(class)ss_as_tagged',
+        blank=True,
+        verbose_name=_('tagged VLANs')
+    )
+    qinq_svlan = models.ForeignKey(
+        to='ipam.VLAN',
+        on_delete=models.SET_NULL,
+        related_name='%(class)ss_svlan',
+        null=True,
+        blank=True,
+        verbose_name=_('Q-in-Q SVLAN')
+    )
     vlan_translation_policy = models.ForeignKey(
         to='ipam.VLANTranslationPolicy',
         on_delete=models.PROTECT,
         null=True,
         blank=True,
-        verbose_name=_('VLAN Translation Policy'),
+        verbose_name=_('VLAN Translation Policy')
     )
 
     class Meta:
         abstract = True
 
+    def clean(self):
+        super().clean()
+
+        # SVLAN can be defined only for Q-in-Q interfaces
+        if self.qinq_svlan and self.mode != InterfaceModeChoices.MODE_Q_IN_Q:
+            raise ValidationError({
+                'qinq_svlan': _("Only Q-in-Q interfaces may specify a service VLAN.")
+            })
+
     def save(self, *args, **kwargs):
 
         # Remove untagged VLAN assignment for non-802.1Q interfaces
@@ -697,20 +728,6 @@ class Interface(ModularComponentModel, BaseInterface, CabledObjectModel, PathEnd
         blank=True,
         verbose_name=_('wireless LANs')
     )
-    untagged_vlan = models.ForeignKey(
-        to='ipam.VLAN',
-        on_delete=models.SET_NULL,
-        related_name='interfaces_as_untagged',
-        null=True,
-        blank=True,
-        verbose_name=_('untagged VLAN')
-    )
-    tagged_vlans = models.ManyToManyField(
-        to='ipam.VLAN',
-        related_name='interfaces_as_tagged',
-        blank=True,
-        verbose_name=_('tagged VLANs')
-    )
     vrf = models.ForeignKey(
         to='ipam.VRF',
         on_delete=models.SET_NULL,

netbox/dcim/tables/devices.py

@@ -585,6 +585,10 @@ class BaseInterfaceTable(NetBoxTable):
         orderable=False,
         verbose_name=_('Tagged VLANs')
     )
+    qinq_svlan = tables.Column(
+        verbose_name=_('Q-in-Q SVLAN'),
+        linkify=True
+    )
 
     def value_ip_addresses(self, value):
         return ",".join([str(obj.address) for obj in value.all()])
@@ -635,11 +639,11 @@ class InterfaceTable(ModularDeviceComponentTable, BaseInterfaceTable, PathEndpoi
         model = models.Interface
         fields = (
             'pk', 'id', 'name', 'device', 'module_bay', 'module', 'label', 'enabled', 'type', 'mgmt_only', 'mtu',
-            'speed', 'speed_formatted', 'duplex', 'mode', 'mac_address', 'wwn', 'poe_mode', 'poe_type', 'rf_role', 'rf_channel',
-            'rf_channel_frequency', 'rf_channel_width', 'tx_power', 'description', 'mark_connected', 'cable',
-            'cable_color', 'wireless_link', 'wireless_lans', 'link_peer', 'connection', 'tags', 'vdcs', 'vrf', 'l2vpn',
-            'tunnel', 'ip_addresses', 'fhrp_groups', 'untagged_vlan', 'tagged_vlans', 'inventory_items', 'created',
-            'last_updated',
+            'speed', 'speed_formatted', 'duplex', 'mode', 'mac_address', 'wwn', 'poe_mode', 'poe_type', 'rf_role',
+            'rf_channel', 'rf_channel_frequency', 'rf_channel_width', 'tx_power', 'description', 'mark_connected',
+            'cable', 'cable_color', 'wireless_link', 'wireless_lans', 'link_peer', 'connection', 'tags', 'vdcs', 'vrf',
+            'l2vpn', 'tunnel', 'ip_addresses', 'fhrp_groups', 'untagged_vlan', 'tagged_vlans', 'qinq_svlan',
+            'inventory_items', 'created', 'last_updated',
         )
         default_columns = ('pk', 'name', 'device', 'label', 'enabled', 'type', 'description')
 
@@ -676,7 +680,7 @@ class DeviceInterfaceTable(InterfaceTable):
             'mgmt_only', 'mtu', 'mode', 'mac_address', 'wwn', 'rf_role', 'rf_channel', 'rf_channel_frequency',
             'rf_channel_width', 'tx_power', 'description', 'mark_connected', 'cable', 'cable_color', 'wireless_link',
             'wireless_lans', 'link_peer', 'connection', 'tags', 'vdcs', 'vrf', 'l2vpn', 'tunnel', 'ip_addresses',
-            'fhrp_groups', 'untagged_vlan', 'tagged_vlans', 'actions',
+            'fhrp_groups', 'untagged_vlan', 'tagged_vlans', 'qinq_svlan', 'actions',
         )
         default_columns = (
             'pk', 'name', 'label', 'enabled', 'type', 'parent', 'lag', 'mtu', 'mode', 'description', 'ip_addresses',

netbox/dcim/tests/test_api.py

@@ -7,6 +7,7 @@ from dcim.choices import *
 from dcim.constants import *
 from dcim.models import *
 from extras.models import ConfigTemplate
+from ipam.choices import VLANQinQRoleChoices
 from ipam.models import ASN, RIR, VLAN, VRF
 from netbox.api.serializers import GenericObjectSerializer
 from tenancy.models import Tenant
@@ -1618,6 +1619,7 @@ class InterfaceTest(Mixins.ComponentTraceMixin, APIViewTestCases.APIViewTestCase
             VLAN(name='VLAN 1', vid=1),
             VLAN(name='VLAN 2', vid=2),
             VLAN(name='VLAN 3', vid=3),
+            VLAN(name='SVLAN 1', vid=1001, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE),
         )
         VLAN.objects.bulk_create(vlans)
 
@@ -1676,18 +1678,22 @@ class InterfaceTest(Mixins.ComponentTraceMixin, APIViewTestCases.APIViewTestCase
                 'vdcs': [vdcs[1].pk],
                 'name': 'Interface 7',
                 'type': InterfaceTypeChoices.TYPE_80211A,
+                'mode': InterfaceModeChoices.MODE_Q_IN_Q,
                 'tx_power': 10,
                 'wireless_lans': [wireless_lans[0].pk, wireless_lans[1].pk],
                 'rf_channel': WirelessChannelChoices.CHANNEL_5G_32,
+                'qinq_svlan': vlans[3].pk,
             },
             {
                 'device': device.pk,
                 'vdcs': [vdcs[1].pk],
                 'name': 'Interface 8',
                 'type': InterfaceTypeChoices.TYPE_80211A,
+                'mode': InterfaceModeChoices.MODE_Q_IN_Q,
                 'tx_power': 10,
                 'wireless_lans': [wireless_lans[0].pk, wireless_lans[1].pk],
                 'rf_channel': "",
+                'qinq_svlan': vlans[3].pk,
             },
         ]
 

netbox/dcim/tests/test_filtersets.py

@@ -4,7 +4,8 @@ from circuits.models import Circuit, CircuitTermination, CircuitType, Provider
 from dcim.choices import *
 from dcim.filtersets import *
 from dcim.models import *
-from ipam.models import ASN, IPAddress, RIR, VLANTranslationPolicy, VRF
+from ipam.choices import VLANQinQRoleChoices
+from ipam.models import ASN, IPAddress, RIR, VLAN, VLANTranslationPolicy, VRF
 from netbox.choices import ColorChoices, WeightUnitChoices
 from tenancy.models import Tenant, TenantGroup
 from users.models import User
@@ -3520,7 +3521,7 @@ class PowerOutletTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedF
 class InterfaceTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedFilterSetTests):
     queryset = Interface.objects.all()
     filterset = InterfaceFilterSet
-    ignore_fields = ('tagged_vlans', 'untagged_vlan', 'vdcs')
+    ignore_fields = ('tagged_vlans', 'untagged_vlan', 'qinq_svlan', 'vdcs')
 
     @classmethod
     def setUpTestData(cls):
@@ -3669,6 +3670,13 @@ class InterfaceTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedFil
         )
         VirtualDeviceContext.objects.bulk_create(vdcs)
 
+        vlans = (
+            VLAN(name='SVLAN 1', vid=1001, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE),
+            VLAN(name='SVLAN 2', vid=1002, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE),
+            VLAN(name='SVLAN 3', vid=1003, qinq_role=VLANQinQRoleChoices.ROLE_SERVICE),
+        )
+        VLAN.objects.bulk_create(vlans)
+
         vlan_translation_policies = (
             VLANTranslationPolicy(name='Policy 1'),
             VLANTranslationPolicy(name='Policy 2'),
@@ -3753,6 +3761,8 @@ class InterfaceTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedFil
                 duplex='full',
                 poe_mode=InterfacePoEModeChoices.MODE_PD,
                 poe_type=InterfacePoETypeChoices.TYPE_2_8023AT,
+                mode=InterfaceModeChoices.MODE_Q_IN_Q,
+                qinq_svlan=vlans[0],
                 vlan_translation_policy=vlan_translation_policies[1],
             ),
             Interface(
@@ -3762,7 +3772,9 @@ class InterfaceTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedFil
                 type=InterfaceTypeChoices.TYPE_OTHER,
                 enabled=True,
                 mgmt_only=True,
-                tx_power=40
+                tx_power=40,
+                mode=InterfaceModeChoices.MODE_Q_IN_Q,
+                qinq_svlan=vlans[1]
             ),
             Interface(
                 device=devices[4],
@@ -3771,7 +3783,9 @@ class InterfaceTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedFil
                 type=InterfaceTypeChoices.TYPE_OTHER,
                 enabled=False,
                 mgmt_only=False,
-                tx_power=40
+                tx_power=40,
+                mode=InterfaceModeChoices.MODE_Q_IN_Q,
+                qinq_svlan=vlans[2]
             ),
             Interface(
                 device=devices[4],
@@ -4027,6 +4041,13 @@ class InterfaceTestCase(TestCase, DeviceComponentFilterSetTests, ChangeLoggedFil
         params = {'vdc_identifier': vdc.values_list('identifier', flat=True)}
         self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3)
 
+    def test_vlan(self):
+        vlan = VLAN.objects.filter(qinq_role=VLANQinQRoleChoices.ROLE_SERVICE).first()
+        params = {'vlan_id': vlan.pk}
+        self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1)
+        params = {'vlan': vlan.vid}
+        self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1)
+
     def test_vlan_translation_policy(self):
         vlan_translation_policies = VLANTranslationPolicy.objects.all()[:2]
         params = {'vlan_translation_policy_id': [vlan_translation_policies[0].pk, vlan_translation_policies[1].pk]}