diff --git a/docs/models/ipam/vlangroup.md b/docs/models/ipam/vlangroup.md index a2920fb70..67050ab4c 100644 --- a/docs/models/ipam/vlangroup.md +++ b/docs/models/ipam/vlangroup.md @@ -14,9 +14,9 @@ A unique human-friendly name. A unique URL-friendly identifier. (This value can be used for filtering.) -### Minimum & Maximum VLAN IDs +### VLAN ID Ranges -A minimum and maximum child VLAN ID must be set for each group. (These default to 1 and 4094 respectively.) VLANs created within a group must have a VID that falls between these values (inclusive). +The set of VLAN IDs which are encompassed by the group. By default, this will be the entire range of valid IEEE 802.1Q VLAN IDs (1 to 4094, inclusive). VLANs created within a group must have a VID that falls within one of these ranges. Ranges may not overlap. ### Scope diff --git a/netbox/ipam/api/serializers_/vlans.py b/netbox/ipam/api/serializers_/vlans.py index 5525545a8..608fcf0b4 100644 --- a/netbox/ipam/api/serializers_/vlans.py +++ b/netbox/ipam/api/serializers_/vlans.py @@ -6,7 +6,7 @@ from dcim.api.serializers_.sites import SiteSerializer from ipam.choices import * from ipam.constants import VLANGROUP_SCOPE_TYPES from ipam.models import VLAN, VLANGroup -from netbox.api.fields import ChoiceField, ContentTypeField, RelatedObjectCountField +from netbox.api.fields import ChoiceField, ContentTypeField, IntegerRangeSerializer, RelatedObjectCountField from netbox.api.serializers import NetBoxModelSerializer from tenancy.api.serializers_.tenants import TenantSerializer from utilities.api import get_serializer_for_model @@ -32,6 +32,7 @@ class VLANGroupSerializer(NetBoxModelSerializer): ) scope_id = serializers.IntegerField(allow_null=True, required=False, default=None) scope = serializers.SerializerMethodField(read_only=True) + vid_ranges = IntegerRangeSerializer(many=True, required=False) utilization = serializers.CharField(read_only=True) # Related object counts @@ -40,8 +41,8 @@ class VLANGroupSerializer(NetBoxModelSerializer): class Meta: model = VLANGroup fields = [ - 'id', 'url', 'display_url', 'display', 'name', 'slug', 'scope_type', 'scope_id', 'scope', 'min_vid', - 'max_vid', 'description', 'tags', 'custom_fields', 'created', 'last_updated', 'vlan_count', 'utilization' + 'id', 'url', 'display_url', 'display', 'name', 'slug', 'scope_type', 'scope_id', 'scope', 'vid_ranges', + 'description', 'tags', 'custom_fields', 'created', 'last_updated', 'vlan_count', 'utilization' ] brief_fields = ('id', 'url', 'display', 'name', 'slug', 'description', 'vlan_count') validators = [] diff --git a/netbox/ipam/filtersets.py b/netbox/ipam/filtersets.py index 5cdfac34e..30634850a 100644 --- a/netbox/ipam/filtersets.py +++ b/netbox/ipam/filtersets.py @@ -911,10 +911,13 @@ class VLANGroupFilterSet(OrganizationalModelFilterSet): cluster = django_filters.NumberFilter( method='filter_scope' ) + contains_vid = django_filters.NumberFilter( + method='filter_contains_vid' + ) class Meta: model = VLANGroup - fields = ('id', 'name', 'slug', 'min_vid', 'max_vid', 'description', 'scope_id') + fields = ('id', 'name', 'slug', 'description', 'scope_id') def search(self, queryset, name, value): if not value.strip(): @@ -932,6 +935,21 @@ class VLANGroupFilterSet(OrganizationalModelFilterSet): scope_id=value ) + def filter_contains_vid(self, queryset, name, value): + """ + Return all VLANGroups which contain the given VLAN ID. + """ + table_name = VLANGroup._meta.db_table + # TODO: See if this can be optimized without compromising queryset integrity + # Expand VLAN ID ranges to query by integer + groups = VLANGroup.objects.raw( + f'SELECT id FROM {table_name}, unnest(vid_ranges) vid_range WHERE %s <@ vid_range', + params=(value,) + ) + return queryset.filter( + pk__in=[g.id for g in groups] + ) + class VLANFilterSet(NetBoxModelFilterSet, TenancyFilterSet): region_id = TreeNodeMultipleChoiceFilter( diff --git a/netbox/ipam/forms/bulk_edit.py b/netbox/ipam/forms/bulk_edit.py index c7f64ab1d..2f59c564f 100644 --- a/netbox/ipam/forms/bulk_edit.py +++ b/netbox/ipam/forms/bulk_edit.py @@ -12,6 +12,7 @@ from tenancy.models import Tenant from utilities.forms import add_blank_choice from utilities.forms.fields import ( CommentField, ContentTypeChoiceField, DynamicModelChoiceField, DynamicModelMultipleChoiceField, NumericArrayField, + NumericRangeArrayField, ) from utilities.forms.rendering import FieldSet from utilities.forms.widgets import BulkEditNullBooleanSelect @@ -408,18 +409,6 @@ class FHRPGroupBulkEditForm(NetBoxModelBulkEditForm): class VLANGroupBulkEditForm(NetBoxModelBulkEditForm): - min_vid = forms.IntegerField( - min_value=VLAN_VID_MIN, - max_value=VLAN_VID_MAX, - required=False, - label=_('Minimum child VLAN VID') - ) - max_vid = forms.IntegerField( - min_value=VLAN_VID_MIN, - max_value=VLAN_VID_MAX, - required=False, - label=_('Maximum child VLAN VID') - ) description = forms.CharField( label=_('Description'), max_length=200, @@ -483,10 +472,14 @@ class VLANGroupBulkEditForm(NetBoxModelBulkEditForm): 'group_id': '$clustergroup', } ) + vid_ranges = NumericRangeArrayField( + label=_('VLAN ID ranges'), + required=False + ) model = VLANGroup fieldsets = ( - FieldSet('site', 'min_vid', 'max_vid', 'description'), + FieldSet('site', 'vid_ranges', 'description'), FieldSet( 'scope_type', 'region', 'sitegroup', 'site', 'location', 'rack', 'clustergroup', 'cluster', name=_('Scope') ), diff --git a/netbox/ipam/forms/bulk_import.py b/netbox/ipam/forms/bulk_import.py index bfff1f4f4..dea250c79 100644 --- a/netbox/ipam/forms/bulk_import.py +++ b/netbox/ipam/forms/bulk_import.py @@ -9,7 +9,8 @@ from ipam.models import * from netbox.forms import NetBoxModelImportForm from tenancy.models import Tenant from utilities.forms.fields import ( - CSVChoiceField, CSVContentTypeField, CSVModelChoiceField, CSVModelMultipleChoiceField, SlugField + CSVChoiceField, CSVContentTypeField, CSVModelChoiceField, CSVModelMultipleChoiceField, SlugField, + NumericRangeArrayField, ) from virtualization.models import VirtualMachine, VMInterface @@ -411,22 +412,13 @@ class VLANGroupImportForm(NetBoxModelImportForm): required=False, label=_('Scope type (app & model)') ) - min_vid = forms.IntegerField( - min_value=VLAN_VID_MIN, - max_value=VLAN_VID_MAX, - required=False, - label=_('Minimum child VLAN VID (default: {minimum})').format(minimum=VLAN_VID_MIN) - ) - max_vid = forms.IntegerField( - min_value=VLAN_VID_MIN, - max_value=VLAN_VID_MAX, - required=False, - label=_('Maximum child VLAN VID (default: {maximum})').format(maximum=VLAN_VID_MIN) + vid_ranges = NumericRangeArrayField( + required=False ) class Meta: model = VLANGroup - fields = ('name', 'slug', 'scope_type', 'scope_id', 'min_vid', 'max_vid', 'description', 'tags') + fields = ('name', 'slug', 'scope_type', 'scope_id', 'vid_ranges', 'description', 'tags') labels = { 'scope_id': 'Scope ID', } diff --git a/netbox/ipam/forms/filtersets.py b/netbox/ipam/forms/filtersets.py index 80fb04226..a32694321 100644 --- a/netbox/ipam/forms/filtersets.py +++ b/netbox/ipam/forms/filtersets.py @@ -413,7 +413,7 @@ class VLANGroupFilterForm(NetBoxModelFilterSetForm): FieldSet('q', 'filter_id', 'tag'), FieldSet('region', 'sitegroup', 'site', 'location', 'rack', name=_('Location')), FieldSet('cluster_group', 'cluster', name=_('Cluster')), - FieldSet('min_vid', 'max_vid', name=_('VLAN ID')), + FieldSet('contains_vid', name=_('VLANs')), ) model = VLANGroup region = DynamicModelMultipleChoiceField( @@ -441,18 +441,6 @@ class VLANGroupFilterForm(NetBoxModelFilterSetForm): required=False, label=_('Rack') ) - min_vid = forms.IntegerField( - required=False, - min_value=VLAN_VID_MIN, - max_value=VLAN_VID_MAX, - label=_('Minimum VID') - ) - max_vid = forms.IntegerField( - required=False, - min_value=VLAN_VID_MIN, - max_value=VLAN_VID_MAX, - label=_('Maximum VID') - ) cluster = DynamicModelMultipleChoiceField( queryset=Cluster.objects.all(), required=False, @@ -463,6 +451,11 @@ class VLANGroupFilterForm(NetBoxModelFilterSetForm): required=False, label=_('Cluster group') ) + contains_vid = forms.IntegerField( + min_value=0, + required=False, + label=_('Contains VLAN ID') + ) tag = TagFilterField(model) diff --git a/netbox/ipam/forms/model_forms.py b/netbox/ipam/forms/model_forms.py index 4e405a035..e6060d1af 100644 --- a/netbox/ipam/forms/model_forms.py +++ b/netbox/ipam/forms/model_forms.py @@ -1,5 +1,6 @@ from django import forms from django.contrib.contenttypes.models import ContentType +from django.contrib.postgres.forms import IntegerRangeField, SimpleArrayField from django.core.exceptions import ValidationError from django.utils.translation import gettext_lazy as _ @@ -14,7 +15,7 @@ from utilities.exceptions import PermissionsViolation from utilities.forms import add_blank_choice from utilities.forms.fields import ( CommentField, ContentTypeChoiceField, DynamicModelChoiceField, DynamicModelMultipleChoiceField, NumericArrayField, - SlugField, + NumericRangeArrayField, SlugField ) from utilities.forms.rendering import FieldSet, InlineFields, ObjectAttribute, TabbedGroups from utilities.forms.widgets import DatePicker @@ -632,10 +633,13 @@ class VLANGroupForm(NetBoxModelForm): } ) slug = SlugField() + vid_ranges = NumericRangeArrayField( + label=_('VLAN IDs') + ) fieldsets = ( FieldSet('name', 'slug', 'description', 'tags', name=_('VLAN Group')), - FieldSet('min_vid', 'max_vid', name=_('Child VLANs')), + FieldSet('vid_ranges', name=_('Child VLANs')), FieldSet( 'scope_type', 'region', 'sitegroup', 'site', 'location', 'rack', 'clustergroup', 'cluster', name=_('Scope') @@ -646,7 +650,7 @@ class VLANGroupForm(NetBoxModelForm): model = VLANGroup fields = [ 'name', 'slug', 'description', 'scope_type', 'region', 'sitegroup', 'site', 'location', 'rack', - 'clustergroup', 'cluster', 'min_vid', 'max_vid', 'tags', + 'clustergroup', 'cluster', 'vid_ranges', 'tags', ] def __init__(self, *args, **kwargs): diff --git a/netbox/ipam/graphql/types.py b/netbox/ipam/graphql/types.py index 36e09eaac..46d45816e 100644 --- a/netbox/ipam/graphql/types.py +++ b/netbox/ipam/graphql/types.py @@ -251,6 +251,7 @@ class VLANType(NetBoxObjectType): class VLANGroupType(OrganizationalObjectType): vlans: List[VLANType] + vid_ranges: List[str] @strawberry_django.field def scope(self) -> Annotated[Union[ diff --git a/netbox/ipam/migrations/0070_vlangroup_vlan_id_ranges.py b/netbox/ipam/migrations/0070_vlangroup_vlan_id_ranges.py new file mode 100644 index 000000000..b01941401 --- /dev/null +++ b/netbox/ipam/migrations/0070_vlangroup_vlan_id_ranges.py @@ -0,0 +1,55 @@ +import django.contrib.postgres.fields +import django.contrib.postgres.fields.ranges +from django.db import migrations, models +from django.db.backends.postgresql.psycopg_any import NumericRange + +import ipam.models.vlans + + +def set_vid_ranges(apps, schema_editor): + """ + Convert the min_vid & max_vid fields to a range in the new vid_ranges ArrayField. + """ + VLANGroup = apps.get_model('ipam', 'VLANGroup') + for group in VLANGroup.objects.all(): + group.vid_ranges = [ + NumericRange(group.min_vid, group.max_vid, bounds='[]') + ] + group._total_vlan_ids = group.max_vid - group.min_vid + 1 + group.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('ipam', '0069_gfk_indexes'), + ] + + operations = [ + migrations.AddField( + model_name='vlangroup', + name='vid_ranges', + field=django.contrib.postgres.fields.ArrayField( + base_field=django.contrib.postgres.fields.ranges.IntegerRangeField(), + default=ipam.models.vlans.default_vid_ranges, + size=None + ), + ), + migrations.AddField( + model_name='vlangroup', + name='_total_vlan_ids', + field=models.PositiveBigIntegerField(default=4094), + ), + migrations.RunPython( + code=set_vid_ranges, + reverse_code=migrations.RunPython.noop + ), + migrations.RemoveField( + model_name='vlangroup', + name='max_vid', + ), + migrations.RemoveField( + model_name='vlangroup', + name='min_vid', + ), + ] diff --git a/netbox/ipam/models/vlans.py b/netbox/ipam/models/vlans.py index 7434bd0b4..ca6b27d07 100644 --- a/netbox/ipam/models/vlans.py +++ b/netbox/ipam/models/vlans.py @@ -1,7 +1,9 @@ from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation +from django.contrib.postgres.fields import ArrayField, IntegerRangeField from django.core.exceptions import ValidationError from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models +from django.db.backends.postgresql.psycopg_any import NumericRange from django.urls import reverse from django.utils.translation import gettext_lazy as _ @@ -10,6 +12,7 @@ from ipam.choices import * from ipam.constants import * from ipam.querysets import VLANQuerySet, VLANGroupQuerySet from netbox.models import OrganizationalModel, PrimaryModel +from utilities.data import check_ranges_overlap, ranges_to_string from virtualization.models import VMInterface __all__ = ( @@ -18,9 +21,16 @@ __all__ = ( ) +def default_vid_ranges(): + return [ + NumericRange(VLAN_VID_MIN, VLAN_VID_MAX, bounds='[]') + ] + + class VLANGroup(OrganizationalModel): """ - A VLAN group is an arbitrary collection of VLANs within which VLAN IDs and names must be unique. + A VLAN group is an arbitrary collection of VLANs within which VLAN IDs and names must be unique. Each group must + define one or more ranges of valid VLAN IDs, and may be assigned a specific scope. """ name = models.CharField( verbose_name=_('name'), @@ -45,23 +55,13 @@ class VLANGroup(OrganizationalModel): ct_field='scope_type', fk_field='scope_id' ) - min_vid = models.PositiveSmallIntegerField( - verbose_name=_('minimum VLAN ID'), - default=VLAN_VID_MIN, - validators=( - MinValueValidator(VLAN_VID_MIN), - MaxValueValidator(VLAN_VID_MAX) - ), - help_text=_('Lowest permissible ID of a child VLAN') + vid_ranges = ArrayField( + IntegerRangeField(), + verbose_name=_('VLAN ID ranges'), + default=default_vid_ranges ) - max_vid = models.PositiveSmallIntegerField( - verbose_name=_('maximum VLAN ID'), - default=VLAN_VID_MAX, - validators=( - MinValueValidator(VLAN_VID_MIN), - MaxValueValidator(VLAN_VID_MAX) - ), - help_text=_('Highest permissible ID of a child VLAN') + _total_vlan_ids = models.PositiveBigIntegerField( + default=VLAN_VID_MAX - VLAN_VID_MIN + 1 ) objects = VLANGroupQuerySet.as_manager() @@ -96,17 +96,33 @@ class VLANGroup(OrganizationalModel): if self.scope_id and not self.scope_type: raise ValidationError(_("Cannot set scope_id without scope_type.")) - # Validate min/max child VID limits - if self.max_vid < self.min_vid: - raise ValidationError({ - 'max_vid': _("Maximum child VID must be greater than or equal to minimum child VID") - }) + # Validate VID ranges + if self.vid_ranges and check_ranges_overlap(self.vid_ranges): + raise ValidationError({'vid_ranges': _("Ranges cannot overlap.")}) + for vid_range in self.vid_ranges: + if vid_range.lower >= vid_range.upper: + raise ValidationError({ + 'vid_ranges': _( + "Maximum child VID must be greater than or equal to minimum child VID ({value})" + ).format(value=vid_range) + }) + + def save(self, *args, **kwargs): + self._total_vlan_ids = 0 + for vid_range in self.vid_ranges: + self._total_vlan_ids += vid_range.upper - vid_range.lower + 1 + + super().save(*args, **kwargs) def get_available_vids(self): """ Return all available VLANs within this group. """ - available_vlans = {vid for vid in range(self.min_vid, self.max_vid + 1)} + available_vlans = set() + for vlan_range in self.vid_ranges: + available_vlans = available_vlans.union({ + vid for vid in range(vlan_range.lower, vlan_range.upper) + }) available_vlans -= set(VLAN.objects.filter(group=self).values_list('vid', flat=True)) return sorted(available_vlans) @@ -126,6 +142,10 @@ class VLANGroup(OrganizationalModel): """ return VLAN.objects.filter(group=self).order_by('vid') + @property + def vid_ranges_list(self): + return ranges_to_string(self.vid_ranges) + class VLAN(PrimaryModel): """ @@ -231,13 +251,14 @@ class VLAN(PrimaryModel): ).format(group=self.group, scope=self.group.scope, site=self.site) ) - # Validate group min/max VIDs - if self.group and not self.group.min_vid <= self.vid <= self.group.max_vid: - raise ValidationError({ - 'vid': _( - "VID must be between {minimum} and {maximum} for VLANs in group {group}" - ).format(minimum=self.group.min_vid, maximum=self.group.max_vid, group=self.group) - }) + # Check that the VLAN ID is permitted in the assigned group (if any) + if self.group: + if not any([self.vid in r for r in self.group.vid_ranges]): + raise ValidationError({ + 'vid': _( + "VID must be in ranges {ranges} for VLANs in group {group}" + ).format(ranges=ranges_to_string(self.group.vid_ranges), group=self.group) + }) def get_status_color(self): return VLANStatusChoices.colors.get(self.status) diff --git a/netbox/ipam/querysets.py b/netbox/ipam/querysets.py index a3f37fe3c..717c63a37 100644 --- a/netbox/ipam/querysets.py +++ b/netbox/ipam/querysets.py @@ -9,6 +9,7 @@ from utilities.querysets import RestrictedQuerySet __all__ = ( 'ASNRangeQuerySet', 'PrefixQuerySet', + 'VLANGroupQuerySet', 'VLANQuerySet', ) @@ -63,7 +64,7 @@ class VLANGroupQuerySet(RestrictedQuerySet): return self.annotate( vlan_count=count_related(VLAN, 'group'), - utilization=Round(F('vlan_count') / (F('max_vid') - F('min_vid') + 1.0) * 100, 2) + utilization=Round(F('vlan_count') * 100 / F('_total_vlan_ids'), 2) ) diff --git a/netbox/ipam/search.py b/netbox/ipam/search.py index a1cddbb1a..59b741b8f 100644 --- a/netbox/ipam/search.py +++ b/netbox/ipam/search.py @@ -154,9 +154,8 @@ class VLANGroupIndex(SearchIndex): ('name', 100), ('slug', 110), ('description', 500), - ('max_vid', 2000), ) - display_attrs = ('scope_type', 'min_vid', 'max_vid', 'description') + display_attrs = ('scope_type', 'description') @register_search diff --git a/netbox/ipam/tables/vlans.py b/netbox/ipam/tables/vlans.py index 11de0381c..1b428aeb6 100644 --- a/netbox/ipam/tables/vlans.py +++ b/netbox/ipam/tables/vlans.py @@ -72,6 +72,10 @@ class VLANGroupTable(NetBoxTable): linkify=True, orderable=False ) + vid_ranges_list = tables.Column( + verbose_name=_('VID Ranges'), + orderable=False + ) vlan_count = columns.LinkedCountColumn( viewname='ipam:vlan_list', url_params={'group_id': 'pk'}, @@ -91,7 +95,7 @@ class VLANGroupTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = VLANGroup fields = ( - 'pk', 'id', 'name', 'scope_type', 'scope', 'min_vid', 'max_vid', 'vlan_count', 'slug', 'description', + 'pk', 'id', 'name', 'scope_type', 'scope', 'vid_ranges_list', 'vlan_count', 'slug', 'description', 'tags', 'created', 'last_updated', 'actions', 'utilization', ) default_columns = ('pk', 'name', 'scope_type', 'scope', 'vlan_count', 'utilization', 'description') diff --git a/netbox/ipam/tests/test_api.py b/netbox/ipam/tests/test_api.py index 2cf7a2f1c..00c240769 100644 --- a/netbox/ipam/tests/test_api.py +++ b/netbox/ipam/tests/test_api.py @@ -8,6 +8,7 @@ from dcim.models import Device, DeviceRole, DeviceType, Interface, Manufacturer, from ipam.choices import * from ipam.models import * from tenancy.models import Tenant +from utilities.data import string_to_ranges from utilities.testing import APITestCase, APIViewTestCases, create_test_device, disable_warnings @@ -882,8 +883,7 @@ class VLANGroupTest(APIViewTestCases.APIViewTestCase): vlangroup = VLANGroup.objects.create( name='VLAN Group X', slug='vlan-group-x', - min_vid=MIN_VID, - max_vid=MAX_VID + vid_ranges=string_to_ranges(f"{MIN_VID}-{MAX_VID}") ) # Create a set of VLANs within the group diff --git a/netbox/ipam/tests/test_filtersets.py b/netbox/ipam/tests/test_filtersets.py index 8f07a241a..e149c0a8d 100644 --- a/netbox/ipam/tests/test_filtersets.py +++ b/netbox/ipam/tests/test_filtersets.py @@ -1,4 +1,5 @@ from django.contrib.contenttypes.models import ContentType +from django.db.backends.postgresql.psycopg_any import NumericRange from django.test import TestCase from netaddr import IPNetwork @@ -1465,6 +1466,7 @@ class FHRPGroupAssignmentTestCase(TestCase, ChangeLoggedFilterSetTests): class VLANGroupTestCase(TestCase, ChangeLoggedFilterSetTests): queryset = VLANGroup.objects.all() filterset = VLANGroupFilterSet + ignore_fields = ('vid_ranges',) @classmethod def setUpTestData(cls): @@ -1494,14 +1496,55 @@ class VLANGroupTestCase(TestCase, ChangeLoggedFilterSetTests): cluster.save() vlan_groups = ( - VLANGroup(name='VLAN Group 1', slug='vlan-group-1', scope=region, description='foobar1'), - VLANGroup(name='VLAN Group 2', slug='vlan-group-2', scope=sitegroup, description='foobar2'), - VLANGroup(name='VLAN Group 3', slug='vlan-group-3', scope=site, description='foobar3'), - VLANGroup(name='VLAN Group 4', slug='vlan-group-4', scope=location), - VLANGroup(name='VLAN Group 5', slug='vlan-group-5', scope=rack), - VLANGroup(name='VLAN Group 6', slug='vlan-group-6', scope=clustergroup), - VLANGroup(name='VLAN Group 7', slug='vlan-group-7', scope=cluster), - VLANGroup(name='VLAN Group 8', slug='vlan-group-8'), + VLANGroup( + name='VLAN Group 1', + slug='vlan-group-1', + vid_ranges=[NumericRange(1, 11), NumericRange(100, 200)], + scope=region, + description='foobar1' + ), + VLANGroup( + name='VLAN Group 2', + slug='vlan-group-2', + vid_ranges=[NumericRange(1, 11), NumericRange(200, 300)], + scope=sitegroup, + description='foobar2' + ), + VLANGroup( + name='VLAN Group 3', + slug='vlan-group-3', + vid_ranges=[NumericRange(1, 11), NumericRange(300, 400)], + scope=site, + description='foobar3' + ), + VLANGroup( + name='VLAN Group 4', + slug='vlan-group-4', + vid_ranges=[NumericRange(1, 11), NumericRange(400, 500)], + scope=location + ), + VLANGroup( + name='VLAN Group 5', + slug='vlan-group-5', + vid_ranges=[NumericRange(1, 11), NumericRange(500, 600)], + scope=rack + ), + VLANGroup( + name='VLAN Group 6', + slug='vlan-group-6', + vid_ranges=[NumericRange(1, 11), NumericRange(600, 700)], + scope=clustergroup + ), + VLANGroup( + name='VLAN Group 7', + slug='vlan-group-7', + vid_ranges=[NumericRange(1, 11), NumericRange(700, 800)], + scope=cluster + ), + VLANGroup( + name='VLAN Group 8', + slug='vlan-group-8' + ), ) VLANGroup.objects.bulk_create(vlan_groups) @@ -1521,6 +1564,12 @@ class VLANGroupTestCase(TestCase, ChangeLoggedFilterSetTests): params = {'description': ['foobar1', 'foobar2']} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + def test_contains_vid(self): + params = {'contains_vid': 123} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + params = {'contains_vid': 1} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 8) + def test_region(self): params = {'region': Region.objects.first().pk} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) diff --git a/netbox/ipam/tests/test_models.py b/netbox/ipam/tests/test_models.py index d0f42e8a6..39eb33a4f 100644 --- a/netbox/ipam/tests/test_models.py +++ b/netbox/ipam/tests/test_models.py @@ -1,6 +1,7 @@ from django.core.exceptions import ValidationError from django.test import TestCase, override_settings from netaddr import IPNetwork, IPSet +from utilities.data import string_to_ranges from ipam.choices import * from ipam.models import * @@ -509,8 +510,7 @@ class TestVLANGroup(TestCase): vlangroup = VLANGroup.objects.create( name='VLAN Group 1', slug='vlan-group-1', - min_vid=100, - max_vid=199 + vid_ranges=string_to_ranges('100-199'), ) VLAN.objects.bulk_create(( VLAN(name='VLAN 100', vid=100, group=vlangroup), @@ -533,3 +533,13 @@ class TestVLANGroup(TestCase): VLAN.objects.create(name='VLAN 104', vid=104, group=vlangroup) self.assertEqual(vlangroup.get_next_available_vid(), 105) + + def test_vid_validation(self): + vlangroup = VLANGroup.objects.first() + + vlan = VLAN(vid=1, name='VLAN 1', group=vlangroup) + with self.assertRaises(ValidationError): + vlan.full_clean() + + vlan = VLAN(vid=109, name='VLAN 109', group=vlangroup) + vlan.full_clean() diff --git a/netbox/ipam/tests/test_views.py b/netbox/ipam/tests/test_views.py index bc42341ba..2acb80ac1 100644 --- a/netbox/ipam/tests/test_views.py +++ b/netbox/ipam/tests/test_views.py @@ -764,9 +764,8 @@ class VLANGroupTestCase(ViewTestCases.OrganizationalObjectViewTestCase): cls.form_data = { 'name': 'VLAN Group X', 'slug': 'vlan-group-x', - 'min_vid': 1, - 'max_vid': 4094, 'description': 'A new VLAN group', + 'vid_ranges': '100-199,300-399', 'tags': [t.pk for t in tags], } diff --git a/netbox/ipam/utils.py b/netbox/ipam/utils.py index 21b90fbcd..ccf6cb632 100644 --- a/netbox/ipam/utils.py +++ b/netbox/ipam/utils.py @@ -90,12 +90,12 @@ def add_available_ipaddresses(prefix, ipaddress_list, is_pool=False): return output -def add_available_vlans(vlans, vlan_group=None): +def available_vlans_from_range(vlans, vlan_group, vlan_range): """ Create fake records for all gaps between used VLANs """ - min_vid = vlan_group.min_vid if vlan_group else VLAN_VID_MIN - max_vid = vlan_group.max_vid if vlan_group else VLAN_VID_MAX + min_vid = int(vlan_range.lower) if vlan_range else VLAN_VID_MIN + max_vid = int(vlan_range.upper) if vlan_range else VLAN_VID_MAX if not vlans: return [{ @@ -128,6 +128,17 @@ def add_available_vlans(vlans, vlan_group=None): 'available': max_vid - prev_vid, }) + return new_vlans + + +def add_available_vlans(vlans, vlan_group): + """ + Create fake records for all gaps between used VLANs + """ + new_vlans = [] + for vlan_range in vlan_group.vid_ranges: + new_vlans.extend(available_vlans_from_range(vlans, vlan_group, vlan_range)) + vlans = list(vlans) + new_vlans vlans.sort(key=lambda v: v.vid if type(v) is VLAN else v['vid']) diff --git a/netbox/ipam/views.py b/netbox/ipam/views.py index e0087f5d1..67d56f15e 100644 --- a/netbox/ipam/views.py +++ b/netbox/ipam/views.py @@ -915,7 +915,7 @@ class IPAddressContactsView(ObjectContactsView): # class VLANGroupListView(generic.ObjectListView): - queryset = VLANGroup.objects.annotate_utilization().prefetch_related('tags') + queryset = VLANGroup.objects.annotate_utilization() filterset = filtersets.VLANGroupFilterSet filterset_form = forms.VLANGroupFilterForm table = tables.VLANGroupTable @@ -923,7 +923,7 @@ class VLANGroupListView(generic.ObjectListView): @register_model_view(VLANGroup) class VLANGroupView(GetRelatedModelsMixin, generic.ObjectView): - queryset = VLANGroup.objects.annotate_utilization().prefetch_related('tags') + queryset = VLANGroup.objects.annotate_utilization() def get_extra_context(self, request, instance): return { diff --git a/netbox/netbox/api/fields.py b/netbox/netbox/api/fields.py index 08ffd0bc4..e7d1ef574 100644 --- a/netbox/netbox/api/fields.py +++ b/netbox/netbox/api/fields.py @@ -1,4 +1,5 @@ from django.core.exceptions import ObjectDoesNotExist +from django.db.backends.postgresql.psycopg_any import NumericRange from django.utils.translation import gettext as _ from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field @@ -11,6 +12,7 @@ __all__ = ( 'ChoiceField', 'ContentTypeField', 'IPNetworkSerializer', + 'IntegerRangeSerializer', 'RelatedObjectCountField', 'SerializedPKRelatedField', ) @@ -154,3 +156,19 @@ class RelatedObjectCountField(serializers.ReadOnlyField): self.relation = relation super().__init__(**kwargs) + + +class IntegerRangeSerializer(serializers.Serializer): + """ + Represents a range of integers. + """ + def to_internal_value(self, data): + if not isinstance(data, (list, tuple)) or len(data) != 2: + raise ValidationError(_("Ranges must be specified in the form (lower, upper).")) + if type(data[0]) is not int or type(data[1]) is not int: + raise ValidationError(_("Range boundaries must be defined as integers.")) + + return NumericRange(data[0], data[1], bounds='[]') + + def to_representation(self, instance): + return instance.lower, instance.upper - 1 diff --git a/netbox/templates/ipam/vlangroup.html b/netbox/templates/ipam/vlangroup.html index cc8c790a9..efd2edcfc 100644 --- a/netbox/templates/ipam/vlangroup.html +++ b/netbox/templates/ipam/vlangroup.html @@ -39,8 +39,8 @@ {{ object.scope|linkify|placeholder }} - {% trans "Permitted VIDs" %} - {{ object.min_vid }} - {{ object.max_vid }} + {% trans "VLAN IDs" %} + {{ object.vid_ranges_list }} Utilization diff --git a/netbox/utilities/data.py b/netbox/utilities/data.py index 62eb68854..73c8476ec 100644 --- a/netbox/utilities/data.py +++ b/netbox/utilities/data.py @@ -1,13 +1,17 @@ import decimal +from django.db.backends.postgresql.psycopg_any import NumericRange from itertools import count, groupby __all__ = ( 'array_to_ranges', 'array_to_string', + 'check_ranges_overlap', 'deepmerge', 'drange', 'flatten_dict', + 'ranges_to_string', 'shallow_compare_dict', + 'string_to_ranges', ) @@ -113,3 +117,52 @@ def drange(start, end, step=decimal.Decimal(1)): while start > end: yield start start += step + + +def check_ranges_overlap(ranges): + """ + Check for overlap in an iterable of NumericRanges. + """ + ranges.sort(key=lambda x: x.lower) + + for i in range(1, len(ranges)): + prev_range = ranges[i - 1] + prev_upper = prev_range.upper if prev_range.upper_inc else prev_range.upper - 1 + lower = ranges[i].lower if ranges[i].lower_inc else ranges[i].lower + 1 + if prev_upper >= lower: + return True + + return False + + +def ranges_to_string(ranges): + """ + Generate a human-friendly string from a set of ranges. Intended for use with ArrayField. For example: + [[1, 100)], [200, 300)] => "1-99,200-299" + """ + if not ranges: + return '' + output = [] + for r in ranges: + lower = r.lower if r.lower_inc else r.lower + 1 + upper = r.upper if r.upper_inc else r.upper - 1 + output.append(f'{lower}-{upper}') + return ','.join(output) + + +def string_to_ranges(value): + """ + Given a string in the format "1-100, 200-300" return an list of NumericRanges. Intended for use with ArrayField. + For example: + "1-99,200-299" => [NumericRange(1, 100), NumericRange(200, 300)] + """ + if not value: + return None + value.replace(' ', '') # Remove whitespace + values = [] + for dash_range in value.split(','): + if '-' not in dash_range: + return None + lower, upper = dash_range.split('-') + values.append(NumericRange(int(lower), int(upper), bounds='[]')) + return values diff --git a/netbox/utilities/forms/fields/array.py b/netbox/utilities/forms/fields/array.py index 14f122453..e6de2d89f 100644 --- a/netbox/utilities/forms/fields/array.py +++ b/netbox/utilities/forms/fields/array.py @@ -1,11 +1,14 @@ from django import forms from django.contrib.postgres.forms import SimpleArrayField +from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ +from utilities.data import ranges_to_string, string_to_ranges from ..utils import parse_numeric_range __all__ = ( 'NumericArrayField', + 'NumericRangeArrayField', ) @@ -24,3 +27,31 @@ class NumericArrayField(SimpleArrayField): if isinstance(value, str): value = ','.join([str(n) for n in parse_numeric_range(value)]) return super().to_python(value) + + +class NumericRangeArrayField(forms.CharField): + """ + A field which allows for array of numeric ranges: + Example: 1-5,7-20,30-50 + """ + def __init__(self, *args, help_text='', **kwargs): + if not help_text: + help_text = mark_safe( + _("Specify one or more numeric ranges separated by commas. Example: " + "1-5,20-30") + ) + super().__init__(*args, help_text=help_text, **kwargs) + + def clean(self, value): + if value and not self.to_python(value): + raise forms.ValidationError( + _("Invalid ranges ({value}). Must be a range of integers in ascending order.").format(value=value) + ) + return super().clean(value) + + def prepare_value(self, value): + if isinstance(value, str): + return value + return ranges_to_string(value) + + def to_python(self, value): + return string_to_ranges(value) diff --git a/netbox/utilities/testing/base.py b/netbox/utilities/testing/base.py index 397d830db..eb45aa784 100644 --- a/netbox/utilities/testing/base.py +++ b/netbox/utilities/testing/base.py @@ -2,7 +2,7 @@ import json from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType -from django.contrib.postgres.fields import ArrayField +from django.contrib.postgres.fields import ArrayField, RangeField from django.core.exceptions import FieldDoesNotExist from django.db.models import ManyToManyField, ManyToManyRel, JSONField from django.forms.models import model_to_dict @@ -12,6 +12,7 @@ from taggit.managers import TaggableManager from core.models import ObjectType from users.models import ObjectPermission +from utilities.data import ranges_to_string from utilities.object_types import object_type_identifier from utilities.permissions import resolve_permission_type from .utils import DUMMY_CF_DATA, extract_form_failures @@ -139,6 +140,9 @@ class ModelTestCase(TestCase): if type(field.base_field) is ArrayField: # Handle nested arrays (e.g. choice sets) model_dict[key] = '\n'.join([f'{k},{v}' for k, v in value]) + elif issubclass(type(field.base_field), RangeField): + # Handle arrays of numeric ranges (e.g. VLANGroup VLAN ID ranges) + model_dict[key] = ranges_to_string(value) else: model_dict[key] = ','.join([str(v) for v in value]) diff --git a/netbox/utilities/tests/test_data.py b/netbox/utilities/tests/test_data.py new file mode 100644 index 000000000..c83885233 --- /dev/null +++ b/netbox/utilities/tests/test_data.py @@ -0,0 +1,68 @@ +from django.db.backends.postgresql.psycopg_any import NumericRange +from django.test import TestCase + +from utilities.data import check_ranges_overlap, ranges_to_string, string_to_ranges + + +class RangeFunctionsTestCase(TestCase): + + def test_check_ranges_overlap(self): + # Non-overlapping ranges + self.assertFalse( + check_ranges_overlap([ + NumericRange(9, 19, bounds='(]'), # 10-19 + NumericRange(19, 30, bounds='(]'), # 20-29 + ]) + ) + self.assertFalse( + check_ranges_overlap([ + NumericRange(10, 19, bounds='[]'), # 10-19 + NumericRange(20, 29, bounds='[]'), # 20-29 + ]) + ) + self.assertFalse( + check_ranges_overlap([ + NumericRange(10, 20, bounds='[)'), # 10-19 + NumericRange(20, 30, bounds='[)'), # 20-29 + ]) + ) + + # Overlapping ranges + self.assertTrue( + check_ranges_overlap([ + NumericRange(9, 20, bounds='(]'), # 10-20 + NumericRange(19, 30, bounds='(]'), # 20-30 + ]) + ) + self.assertTrue( + check_ranges_overlap([ + NumericRange(10, 20, bounds='[]'), # 10-20 + NumericRange(20, 30, bounds='[]'), # 20-30 + ]) + ) + self.assertTrue( + check_ranges_overlap([ + NumericRange(10, 21, bounds='[)'), # 10-20 + NumericRange(20, 31, bounds='[)'), # 10-30 + ]) + ) + + def test_ranges_to_string(self): + self.assertEqual( + ranges_to_string([ + NumericRange(10, 20), # 10-19 + NumericRange(30, 40), # 30-39 + NumericRange(100, 200), # 100-199 + ]), + '10-19,30-39,100-199' + ) + + def test_string_to_ranges(self): + self.assertEqual( + string_to_ranges('10-19, 30-39, 100-199'), + [ + NumericRange(10, 19, bounds='[]'), # 10-19 + NumericRange(30, 39, bounds='[]'), # 30-39 + NumericRange(100, 199, bounds='[]'), # 100-199 + ] + )