From b271fd32bdadd6f9cbd5e2261554f70837cd06ac Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 7 Feb 2020 11:36:22 -0500 Subject: [PATCH 01/11] Introduce NaturalOrderingField --- netbox/utilities/fields.py | 33 +++++++++++++++++++++++++++++++++ netbox/utilities/ordering.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 netbox/utilities/ordering.py diff --git a/netbox/utilities/fields.py b/netbox/utilities/fields.py index d2165aea6..6181a7ca1 100644 --- a/netbox/utilities/fields.py +++ b/netbox/utilities/fields.py @@ -1,6 +1,7 @@ from django.core.validators import RegexValidator from django.db import models +from utilities.ordering import naturalize from .forms import ColorSelect ColorValidator = RegexValidator( @@ -35,3 +36,35 @@ class ColorField(models.CharField): def formfield(self, **kwargs): kwargs['widget'] = ColorSelect return super().formfield(**kwargs) + + +class NaturalOrderingField(models.CharField): + """ + A field which stores a naturalized representation of its target field, to be used for ordering its parent model. + + :param target_field: Name of the field of the parent model to be naturalized + :param naturalize_function: The function used to generate a naturalized value (optional) + """ + description = "Stores a representation of its target field suitable for natural ordering" + + def __init__(self, target_field, naturalize_function=naturalize, *args, **kwargs): + self.target_field = target_field + self.naturalize_function = naturalize_function + super().__init__(*args, **kwargs) + + def pre_save(self, model_instance, add): + """ + Generate a naturalized value from the target field + """ + value = getattr(model_instance, self.target_field) + return self.naturalize_function(value, max_length=self.max_length) + + def deconstruct(self): + kwargs = super().deconstruct()[3] # Pass kwargs from CharField + kwargs['naturalize_function'] = self.naturalize_function + return ( + self.name, + 'utilities.fields.NaturalOrderingField', + ['target_field'], + kwargs, + ) diff --git a/netbox/utilities/ordering.py b/netbox/utilities/ordering.py new file mode 100644 index 000000000..2d6c06f63 --- /dev/null +++ b/netbox/utilities/ordering.py @@ -0,0 +1,31 @@ +import re + + +def naturalize(value, max_length=None, integer_places=8): + """ + Take an alphanumeric string and prepend all integers to `integer_places` places to ensure the strings + are ordered naturally. For example: + + site9router21 + site10router4 + site10router19 + + becomes: + + site00000009router00000021 + site00000010router00000004 + site00000010router00000019 + + :param value: The value to be naturalized + :param max_length: The maximum length of the returned string. Characters beyond this length will be stripped. + :param integer_places: The number of places to which each integer will be expanded. (Default: 8) + """ + output = [] + for segment in re.split(r'(\d+)', value): + if segment.isdigit(): + output.append(segment.rjust(integer_places, '0')) + elif segment: + output.append(segment) + ret = ''.join(output) + + return ret[:max_length] if max_length else ret From 12d09e2274d46e242dcc50128202d117a3415ee8 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 7 Feb 2020 11:36:58 -0500 Subject: [PATCH 02/11] Convert device components to use NaturalOrderingField --- .../0093_device_component_ordering.py | 140 ++++++++++++++++++ netbox/dcim/models/device_components.py | 98 +++++++----- 2 files changed, 202 insertions(+), 36 deletions(-) create mode 100644 netbox/dcim/migrations/0093_device_component_ordering.py diff --git a/netbox/dcim/migrations/0093_device_component_ordering.py b/netbox/dcim/migrations/0093_device_component_ordering.py new file mode 100644 index 000000000..371436b64 --- /dev/null +++ b/netbox/dcim/migrations/0093_device_component_ordering.py @@ -0,0 +1,140 @@ +from django.db import migrations +import utilities.fields +import utilities.ordering + + +def _update_model_names(model): + # Update each unique field value in bulk + for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + + +def naturalize_consoleports(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'ConsolePort')) + + +def naturalize_consoleserverports(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'ConsoleServerPort')) + + +def naturalize_powerports(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'PowerPort')) + + +def naturalize_poweroutlets(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'PowerPort')) + + +def naturalize_frontports(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'FrontPort')) + + +def naturalize_rearports(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'RearPort')) + + +def naturalize_devicebays(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'DeviceBay')) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dcim', '0092_fix_rack_outer_unit'), + ] + + operations = [ + migrations.AlterModelOptions( + name='consoleport', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='consoleserverport', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='devicebay', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='frontport', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='inventoryitem', + options={'ordering': ('device__id', 'parent__id', '_name')}, + ), + migrations.AlterModelOptions( + name='poweroutlet', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='powerport', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='rearport', + options={'ordering': ('device', '_name')}, + ), + migrations.AddField( + model_name='consoleport', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='consoleserverport', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='devicebay', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='frontport', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='inventoryitem', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='poweroutlet', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='powerport', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='rearport', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.RunPython( + code=naturalize_consoleports + ), + migrations.RunPython( + code=naturalize_consoleserverports + ), + migrations.RunPython( + code=naturalize_powerports + ), + migrations.RunPython( + code=naturalize_poweroutlets + ), + migrations.RunPython( + code=naturalize_frontports + ), + migrations.RunPython( + code=naturalize_rearports + ), + migrations.RunPython( + code=naturalize_devicebays + ), + ] diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 431419a7a..3eb9ac74e 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -12,7 +12,7 @@ from dcim.exceptions import LoopDetected from dcim.fields import MACAddressField from dcim.managers import InterfaceManager from extras.models import ObjectChange, TaggedItem -from utilities.managers import NaturalOrderingManager +from utilities.fields import NaturalOrderingField from utilities.utils import serialize_object from virtualization.choices import VMInterfaceTypeChoices @@ -181,6 +181,11 @@ class ConsolePort(CableTermination, ComponentModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=ConsolePortTypeChoices, @@ -197,15 +202,13 @@ class ConsolePort(CableTermination, ComponentModel): choices=CONNECTION_STATUS_CHOICES, blank=True ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'type', 'description'] class Meta: - ordering = ['device', 'name'] - unique_together = ['device', 'name'] + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return self.name @@ -238,6 +241,11 @@ class ConsoleServerPort(CableTermination, ComponentModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=ConsolePortTypeChoices, @@ -247,14 +255,13 @@ class ConsoleServerPort(CableTermination, ComponentModel): choices=CONNECTION_STATUS_CHOICES, blank=True ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'type', 'description'] class Meta: - unique_together = ['device', 'name'] + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return self.name @@ -287,6 +294,11 @@ class PowerPort(CableTermination, ComponentModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PowerPortTypeChoices, @@ -322,15 +334,13 @@ class PowerPort(CableTermination, ComponentModel): choices=CONNECTION_STATUS_CHOICES, blank=True ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'type', 'maximum_draw', 'allocated_draw', 'description'] class Meta: - ordering = ['device', 'name'] - unique_together = ['device', 'name'] + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return self.name @@ -433,6 +443,11 @@ class PowerOutlet(CableTermination, ComponentModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PowerOutletTypeChoices, @@ -455,14 +470,13 @@ class PowerOutlet(CableTermination, ComponentModel): choices=CONNECTION_STATUS_CHOICES, blank=True ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'type', 'power_port', 'feed_leg', 'description'] class Meta: - unique_together = ['device', 'name'] + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return self.name @@ -761,6 +775,11 @@ class FrontPort(CableTermination, ComponentModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PortTypeChoices @@ -774,20 +793,17 @@ class FrontPort(CableTermination, ComponentModel): default=1, validators=[MinValueValidator(1), MaxValueValidator(64)] ) - - is_path_endpoint = False - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'type', 'rear_port', 'rear_port_position', 'description'] + is_path_endpoint = False class Meta: - ordering = ['device', 'name'] - unique_together = [ - ['device', 'name'], - ['rear_port', 'rear_port_position'], - ] + ordering = ('device', '_name') + unique_together = ( + ('device', 'name'), + ('rear_port', 'rear_port_position'), + ) def __str__(self): return self.name @@ -831,6 +847,11 @@ class RearPort(CableTermination, ComponentModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PortTypeChoices @@ -839,17 +860,14 @@ class RearPort(CableTermination, ComponentModel): default=1, validators=[MinValueValidator(1), MaxValueValidator(64)] ) - - is_path_endpoint = False - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'type', 'positions', 'description'] + is_path_endpoint = False class Meta: - ordering = ['device', 'name'] - unique_together = ['device', 'name'] + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return self.name @@ -881,6 +899,11 @@ class DeviceBay(ComponentModel): max_length=50, verbose_name='Name' ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) installed_device = models.OneToOneField( to='dcim.Device', on_delete=models.SET_NULL, @@ -888,15 +911,13 @@ class DeviceBay(ComponentModel): blank=True, null=True ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'installed_device', 'description'] class Meta: - ordering = ['device', 'name'] - unique_together = ['device', 'name'] + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return '{} - {}'.format(self.device.name, self.name) @@ -960,6 +981,11 @@ class InventoryItem(ComponentModel): max_length=50, verbose_name='Name' ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) manufacturer = models.ForeignKey( to='dcim.Manufacturer', on_delete=models.PROTECT, @@ -997,8 +1023,8 @@ class InventoryItem(ComponentModel): ] class Meta: - ordering = ['device__id', 'parent__id', 'name'] - unique_together = ['device', 'parent', 'name'] + ordering = ('device__id', 'parent__id', '_name') + unique_together = ('device', 'parent', 'name') def __str__(self): return self.name From 705c35288542263113095ec7bd896882f1a8ae4e Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 7 Feb 2020 11:42:12 -0500 Subject: [PATCH 03/11] Convert device component templates to use NaturalOrderingField --- ...0094_device_component_template_ordering.py | 131 ++++++++++++++++++ .../dcim/models/device_component_templates.py | 85 +++++++----- 2 files changed, 184 insertions(+), 32 deletions(-) create mode 100644 netbox/dcim/migrations/0094_device_component_template_ordering.py diff --git a/netbox/dcim/migrations/0094_device_component_template_ordering.py b/netbox/dcim/migrations/0094_device_component_template_ordering.py new file mode 100644 index 000000000..42d49c83e --- /dev/null +++ b/netbox/dcim/migrations/0094_device_component_template_ordering.py @@ -0,0 +1,131 @@ +from django.db import migrations +import utilities.fields +import utilities.ordering + + +def _update_model_names(model): + # Update each unique field value in bulk + for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + + +def naturalize_consoleporttemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'ConsolePortTemplate')) + + +def naturalize_consoleserverporttemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'ConsoleServerPortTemplate')) + + +def naturalize_powerporttemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'PowerPortTemplate')) + + +def naturalize_poweroutlettemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'PowerPortTemplate')) + + +def naturalize_frontporttemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'FrontPortTemplate')) + + +def naturalize_rearporttemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'RearPortTemplate')) + + +def naturalize_devicebaytemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'DeviceBayTemplate')) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dcim', '0093_device_component_ordering'), + ] + + operations = [ + migrations.AlterModelOptions( + name='consoleporttemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AlterModelOptions( + name='consoleserverporttemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AlterModelOptions( + name='devicebaytemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AlterModelOptions( + name='frontporttemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AlterModelOptions( + name='poweroutlettemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AlterModelOptions( + name='powerporttemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AlterModelOptions( + name='rearporttemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AddField( + model_name='consoleporttemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='consoleserverporttemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='devicebaytemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='frontporttemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='poweroutlettemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='powerporttemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='rearporttemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.RunPython( + code=naturalize_consoleporttemplates + ), + migrations.RunPython( + code=naturalize_consoleserverporttemplates + ), + migrations.RunPython( + code=naturalize_powerporttemplates + ), + migrations.RunPython( + code=naturalize_poweroutlettemplates + ), + migrations.RunPython( + code=naturalize_frontporttemplates + ), + migrations.RunPython( + code=naturalize_rearporttemplates + ), + migrations.RunPython( + code=naturalize_devicebaytemplates + ), + ] diff --git a/netbox/dcim/models/device_component_templates.py b/netbox/dcim/models/device_component_templates.py index 2aa46d0ea..0b5c312ba 100644 --- a/netbox/dcim/models/device_component_templates.py +++ b/netbox/dcim/models/device_component_templates.py @@ -6,7 +6,7 @@ from dcim.choices import * from dcim.constants import * from dcim.managers import InterfaceManager from extras.models import ObjectChange -from utilities.managers import NaturalOrderingManager +from utilities.fields import NaturalOrderingField from utilities.utils import serialize_object from .device_components import ( ConsolePort, ConsoleServerPort, DeviceBay, FrontPort, Interface, PowerOutlet, PowerPort, RearPort, @@ -58,17 +58,20 @@ class ConsolePortTemplate(ComponentTemplateModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=ConsolePortTypeChoices, blank=True ) - objects = NaturalOrderingManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name @@ -93,17 +96,20 @@ class ConsoleServerPortTemplate(ComponentTemplateModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=ConsolePortTypeChoices, blank=True ) - objects = NaturalOrderingManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name @@ -128,6 +134,11 @@ class PowerPortTemplate(ComponentTemplateModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PowerPortTypeChoices, @@ -146,11 +157,9 @@ class PowerPortTemplate(ComponentTemplateModel): help_text="Allocated power draw (watts)" ) - objects = NaturalOrderingManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name @@ -176,6 +185,11 @@ class PowerOutletTemplate(ComponentTemplateModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PowerOutletTypeChoices, @@ -195,11 +209,9 @@ class PowerOutletTemplate(ComponentTemplateModel): help_text="Phase (for three-phase feeds)" ) - objects = NaturalOrderingManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name @@ -276,6 +288,11 @@ class FrontPortTemplate(ComponentTemplateModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PortTypeChoices @@ -290,14 +307,12 @@ class FrontPortTemplate(ComponentTemplateModel): validators=[MinValueValidator(1), MaxValueValidator(64)] ) - objects = NaturalOrderingManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = [ - ['device_type', 'name'], - ['rear_port', 'rear_port_position'], - ] + ordering = ('device_type', '_name') + unique_together = ( + ('device_type', 'name'), + ('rear_port', 'rear_port_position'), + ) def __str__(self): return self.name @@ -344,6 +359,11 @@ class RearPortTemplate(ComponentTemplateModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PortTypeChoices @@ -353,11 +373,9 @@ class RearPortTemplate(ComponentTemplateModel): validators=[MinValueValidator(1), MaxValueValidator(64)] ) - objects = NaturalOrderingManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name @@ -383,12 +401,15 @@ class DeviceBayTemplate(ComponentTemplateModel): name = models.CharField( max_length=50 ) - - objects = NaturalOrderingManager() + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name From 099c446f389344faec5f60e2d2352dd2740361fd Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 7 Feb 2020 11:54:34 -0500 Subject: [PATCH 04/11] Convert remaining DCIM models to use NaturalOrderingField --- .../migrations/0095_primary_model_ordering.py | 67 +++++++++++++++++++ netbox/dcim/models/__init__.py | 48 +++++++------ netbox/dcim/tables.py | 9 +-- netbox/utilities/ordering.py | 2 + 4 files changed, 100 insertions(+), 26 deletions(-) create mode 100644 netbox/dcim/migrations/0095_primary_model_ordering.py diff --git a/netbox/dcim/migrations/0095_primary_model_ordering.py b/netbox/dcim/migrations/0095_primary_model_ordering.py new file mode 100644 index 000000000..7a04aa31d --- /dev/null +++ b/netbox/dcim/migrations/0095_primary_model_ordering.py @@ -0,0 +1,67 @@ +from django.db import migrations +import utilities.fields +import utilities.ordering + + +def _update_model_names(model): + # Update each unique field value in bulk + for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + + +def naturalize_sites(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'Site')) + + +def naturalize_racks(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'Rack')) + + +def naturalize_devices(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'Device')) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dcim', '0094_device_component_template_ordering'), + ] + + operations = [ + migrations.AlterModelOptions( + name='device', + options={'ordering': ('_name', 'pk'), 'permissions': (('napalm_read', 'Read-only access to devices via NAPALM'), ('napalm_write', 'Read/write access to devices via NAPALM'))}, + ), + migrations.AlterModelOptions( + name='rack', + options={'ordering': ('site', 'group', '_name', 'pk')}, + ), + migrations.AlterModelOptions( + name='site', + options={'ordering': ('_name',)}, + ), + migrations.AddField( + model_name='device', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='rack', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='site', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.RunPython( + code=naturalize_sites + ), + migrations.RunPython( + code=naturalize_racks + ), + migrations.RunPython( + code=naturalize_devices + ), + ] diff --git a/netbox/dcim/models/__init__.py b/netbox/dcim/models/__init__.py index 2a4988ba4..9b9252bef 100644 --- a/netbox/dcim/models/__init__.py +++ b/netbox/dcim/models/__init__.py @@ -22,8 +22,7 @@ from dcim.choices import * from dcim.constants import * from dcim.fields import ASNField from extras.models import ConfigContextModel, CustomFieldModel, TaggedItem -from utilities.fields import ColorField -from utilities.managers import NaturalOrderingManager +from utilities.fields import ColorField, NaturalOrderingField from utilities.models import ChangeLoggedModel from utilities.utils import foreground_color, to_meters from .device_component_templates import ( @@ -134,6 +133,11 @@ class Site(ChangeLoggedModel, CustomFieldModel): max_length=50, unique=True ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) slug = models.SlugField( unique=True ) @@ -215,8 +219,6 @@ class Site(ChangeLoggedModel, CustomFieldModel): images = GenericRelation( to='extras.ImageAttachment' ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = [ @@ -235,7 +237,7 @@ class Site(ChangeLoggedModel, CustomFieldModel): } class Meta: - ordering = ['name'] + ordering = ('_name',) def __str__(self): return self.name @@ -516,6 +518,11 @@ class Rack(ChangeLoggedModel, CustomFieldModel, RackElevationHelperMixin): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) facility_id = models.CharField( max_length=50, blank=True, @@ -612,8 +619,6 @@ class Rack(ChangeLoggedModel, CustomFieldModel, RackElevationHelperMixin): images = GenericRelation( to='extras.ImageAttachment' ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = [ @@ -634,12 +639,12 @@ class Rack(ChangeLoggedModel, CustomFieldModel, RackElevationHelperMixin): } class Meta: - ordering = ('site', 'group', 'name', 'pk') # (site, group, name) may be non-unique - unique_together = [ + ordering = ('site', 'group', '_name', 'pk') # (site, group, name) may be non-unique + unique_together = ( # Name and facility_id must be unique *only* within a RackGroup - ['group', 'name'], - ['group', 'facility_id'], - ] + ('group', 'name'), + ('group', 'facility_id'), + ) def __str__(self): return self.display_name or super().__str__() @@ -1313,6 +1318,11 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel): blank=True, null=True ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) serial = models.CharField( max_length=50, blank=True, @@ -1407,8 +1417,6 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel): images = GenericRelation( to='extras.ImageAttachment' ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = [ @@ -1430,12 +1438,12 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel): } class Meta: - ordering = ('name', 'pk') # Name may be NULL - unique_together = [ - ['site', 'tenant', 'name'], # See validate_unique below - ['rack', 'position', 'face'], - ['virtual_chassis', 'vc_position'], - ] + ordering = ('_name', 'pk') # Name may be blank + unique_together = ( + ('site', 'tenant', 'name'), # See validate_unique below + ('rack', 'position', 'face'), + ('virtual_chassis', 'vc_position'), + ) permissions = ( ('napalm_read', 'Read-only access to devices via NAPALM'), ('napalm_write', 'Read/write access to devices via NAPALM'), diff --git a/netbox/dcim/tables.py b/netbox/dcim/tables.py index 1b3076c6c..4c1846c54 100644 --- a/netbox/dcim/tables.py +++ b/netbox/dcim/tables.py @@ -229,7 +229,7 @@ class RegionTable(BaseTable): class SiteTable(BaseTable): pk = ToggleColumn() - name = tables.LinkColumn(order_by=('_nat1', '_nat2', '_nat3')) + name = tables.LinkColumn() status = tables.TemplateColumn(template_code=STATUS_LABEL, verbose_name='Status') region = tables.TemplateColumn(template_code=SITE_REGION_LINK) tenant = tables.TemplateColumn(template_code=COL_TENANT) @@ -291,7 +291,7 @@ class RackRoleTable(BaseTable): class RackTable(BaseTable): pk = ToggleColumn() - name = tables.LinkColumn(order_by=('_nat1', '_nat2', '_nat3')) + name = tables.LinkColumn() site = tables.LinkColumn('dcim:site', args=[Accessor('site.slug')]) group = tables.Column(accessor=Accessor('group.name'), verbose_name='Group') tenant = tables.TemplateColumn(template_code=COL_TENANT) @@ -653,10 +653,7 @@ class PlatformTable(BaseTable): class DeviceTable(BaseTable): pk = ToggleColumn() - name = tables.TemplateColumn( - order_by=('_nat1', '_nat2', '_nat3'), - template_code=DEVICE_LINK - ) + name = tables.TemplateColumn(template_code=DEVICE_LINK) status = tables.TemplateColumn(template_code=STATUS_LABEL, verbose_name='Status') tenant = tables.TemplateColumn(template_code=COL_TENANT) site = tables.LinkColumn('dcim:site', args=[Accessor('site.slug')]) diff --git a/netbox/utilities/ordering.py b/netbox/utilities/ordering.py index 2d6c06f63..fd3010e90 100644 --- a/netbox/utilities/ordering.py +++ b/netbox/utilities/ordering.py @@ -20,6 +20,8 @@ def naturalize(value, max_length=None, integer_places=8): :param max_length: The maximum length of the returned string. Characters beyond this length will be stripped. :param integer_places: The number of places to which each integer will be expanded. (Default: 8) """ + if not value: + return '' output = [] for segment in re.split(r'(\d+)', value): if segment.isdigit(): From 35511cfdc1a2b70c3ac98f951b933e9fe5c076bf Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 7 Feb 2020 11:59:32 -0500 Subject: [PATCH 05/11] Remove NaturalOrderingManager --- netbox/utilities/managers.py | 45 ------------------------------------ 1 file changed, 45 deletions(-) delete mode 100644 netbox/utilities/managers.py diff --git a/netbox/utilities/managers.py b/netbox/utilities/managers.py deleted file mode 100644 index ad646a78e..000000000 --- a/netbox/utilities/managers.py +++ /dev/null @@ -1,45 +0,0 @@ -from django.db.models import Manager -from django.db.models.expressions import RawSQL - -NAT1 = r"CAST(SUBSTRING({}.{} FROM '^(\d{{1,9}})') AS integer)" -NAT2 = r"SUBSTRING({}.{} FROM '^\d*(.*?)\d*$')" -NAT3 = r"CAST(SUBSTRING({}.{} FROM '(\d{{1,9}})$') AS integer)" - - -class NaturalOrderingManager(Manager): - """ - Order objects naturally by a designated field (defaults to 'name'). Leading and/or trailing digits of values within - this field will be cast as independent integers and sorted accordingly. For example, "Foo2" will be ordered before - "Foo10", even though the digit 1 is normally ordered before the digit 2. - """ - natural_order_field = 'name' - - def get_queryset(self): - - queryset = super().get_queryset() - - db_table = self.model._meta.db_table - db_field = self.natural_order_field - - # Append the three subfields derived from the designated natural ordering field - queryset = ( - queryset.annotate(_nat1=RawSQL(NAT1.format(db_table, db_field), ())) - .annotate(_nat2=RawSQL(NAT2.format(db_table, db_field), ())) - .annotate(_nat3=RawSQL(NAT3.format(db_table, db_field), ())) - ) - - # Replace any instance of the designated natural ordering field with its three subfields - ordering = [] - for field in self.model._meta.ordering: - if field == self.natural_order_field: - ordering.append('_nat1') - ordering.append('_nat2') - ordering.append('_nat3') - else: - ordering.append(field) - - # Default to using the _nat indexes if Meta.ordering is empty - if not ordering: - ordering = ('_nat1', '_nat2', '_nat3') - - return queryset.order_by(*ordering) From c72a353733420fa40edabf69d4548668b0b8c32b Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 7 Feb 2020 12:23:52 -0500 Subject: [PATCH 06/11] Enable reverse migration --- .../0093_device_component_ordering.py | 21 ++++++++++++------- ...0094_device_component_template_ordering.py | 21 ++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/netbox/dcim/migrations/0093_device_component_ordering.py b/netbox/dcim/migrations/0093_device_component_ordering.py index 371436b64..a7d289892 100644 --- a/netbox/dcim/migrations/0093_device_component_ordering.py +++ b/netbox/dcim/migrations/0093_device_component_ordering.py @@ -117,24 +117,31 @@ class Migration(migrations.Migration): field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), ), migrations.RunPython( - code=naturalize_consoleports + code=naturalize_consoleports, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_consoleserverports + code=naturalize_consoleserverports, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_powerports + code=naturalize_powerports, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_poweroutlets + code=naturalize_poweroutlets, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_frontports + code=naturalize_frontports, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_rearports + code=naturalize_rearports, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_devicebays + code=naturalize_devicebays, + reverse_code=migrations.RunPython.noop ), ] diff --git a/netbox/dcim/migrations/0094_device_component_template_ordering.py b/netbox/dcim/migrations/0094_device_component_template_ordering.py index 42d49c83e..32ae6e383 100644 --- a/netbox/dcim/migrations/0094_device_component_template_ordering.py +++ b/netbox/dcim/migrations/0094_device_component_template_ordering.py @@ -108,24 +108,31 @@ class Migration(migrations.Migration): field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), ), migrations.RunPython( - code=naturalize_consoleporttemplates + code=naturalize_consoleporttemplates, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_consoleserverporttemplates + code=naturalize_consoleserverporttemplates, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_powerporttemplates + code=naturalize_powerporttemplates, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_poweroutlettemplates + code=naturalize_poweroutlettemplates, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_frontporttemplates + code=naturalize_frontporttemplates, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_rearporttemplates + code=naturalize_rearporttemplates, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_devicebaytemplates + code=naturalize_devicebaytemplates, + reverse_code=migrations.RunPython.noop ), ] From dc1b7874ff4b7a7f6a8b2bb96fdb0fc4f58754a2 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 7 Feb 2020 12:24:38 -0500 Subject: [PATCH 07/11] Store empty names as null --- netbox/dcim/migrations/0095_primary_model_ordering.py | 11 +++++++---- netbox/dcim/models/__init__.py | 5 +++-- netbox/utilities/ordering.py | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/netbox/dcim/migrations/0095_primary_model_ordering.py b/netbox/dcim/migrations/0095_primary_model_ordering.py index 7a04aa31d..9cef0a581 100644 --- a/netbox/dcim/migrations/0095_primary_model_ordering.py +++ b/netbox/dcim/migrations/0095_primary_model_ordering.py @@ -43,7 +43,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='device', name='_name', - field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize, null=True), ), migrations.AddField( model_name='rack', @@ -56,12 +56,15 @@ class Migration(migrations.Migration): field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), ), migrations.RunPython( - code=naturalize_sites + code=naturalize_sites, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_racks + code=naturalize_racks, + reverse_code=migrations.RunPython.noop ), migrations.RunPython( - code=naturalize_devices + code=naturalize_devices, + reverse_code=migrations.RunPython.noop ), ] diff --git a/netbox/dcim/models/__init__.py b/netbox/dcim/models/__init__.py index 9b9252bef..c31f4c713 100644 --- a/netbox/dcim/models/__init__.py +++ b/netbox/dcim/models/__init__.py @@ -1321,7 +1321,8 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel): _name = NaturalOrderingField( target_field='name', max_length=100, - blank=True + blank=True, + null=True ) serial = models.CharField( max_length=50, @@ -1438,7 +1439,7 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel): } class Meta: - ordering = ('_name', 'pk') # Name may be blank + ordering = ('_name', 'pk') # Name may be null unique_together = ( ('site', 'tenant', 'name'), # See validate_unique below ('rack', 'position', 'face'), diff --git a/netbox/utilities/ordering.py b/netbox/utilities/ordering.py index fd3010e90..88a46d3d3 100644 --- a/netbox/utilities/ordering.py +++ b/netbox/utilities/ordering.py @@ -21,7 +21,7 @@ def naturalize(value, max_length=None, integer_places=8): :param integer_places: The number of places to which each integer will be expanded. (Default: 8) """ if not value: - return '' + return value output = [] for segment in re.split(r'(\d+)', value): if segment.isdigit(): From 12c7d83a9178ed8bcde13bee53a9217e23f0b61b Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 7 Feb 2020 12:43:53 -0500 Subject: [PATCH 08/11] Fix PowerOutlet migrations --- netbox/dcim/migrations/0093_device_component_ordering.py | 2 +- .../dcim/migrations/0094_device_component_template_ordering.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox/dcim/migrations/0093_device_component_ordering.py b/netbox/dcim/migrations/0093_device_component_ordering.py index a7d289892..017241c8b 100644 --- a/netbox/dcim/migrations/0093_device_component_ordering.py +++ b/netbox/dcim/migrations/0093_device_component_ordering.py @@ -22,7 +22,7 @@ def naturalize_powerports(apps, schema_editor): def naturalize_poweroutlets(apps, schema_editor): - _update_model_names(apps.get_model('dcim', 'PowerPort')) + _update_model_names(apps.get_model('dcim', 'PowerOutlet')) def naturalize_frontports(apps, schema_editor): diff --git a/netbox/dcim/migrations/0094_device_component_template_ordering.py b/netbox/dcim/migrations/0094_device_component_template_ordering.py index 32ae6e383..fc39f76b2 100644 --- a/netbox/dcim/migrations/0094_device_component_template_ordering.py +++ b/netbox/dcim/migrations/0094_device_component_template_ordering.py @@ -22,7 +22,7 @@ def naturalize_powerporttemplates(apps, schema_editor): def naturalize_poweroutlettemplates(apps, schema_editor): - _update_model_names(apps.get_model('dcim', 'PowerPortTemplate')) + _update_model_names(apps.get_model('dcim', 'PowerOutletTemplate')) def naturalize_frontporttemplates(apps, schema_editor): From 9adeed55fb14f49ffaeab18056e2fafebc518f4e Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 7 Feb 2020 12:44:51 -0500 Subject: [PATCH 09/11] Update table field ordering --- netbox/dcim/tables.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/netbox/dcim/tables.py b/netbox/dcim/tables.py index 4c1846c54..473d465bd 100644 --- a/netbox/dcim/tables.py +++ b/netbox/dcim/tables.py @@ -229,7 +229,7 @@ class RegionTable(BaseTable): class SiteTable(BaseTable): pk = ToggleColumn() - name = tables.LinkColumn() + name = tables.LinkColumn(order_by=('_name',)) status = tables.TemplateColumn(template_code=STATUS_LABEL, verbose_name='Status') region = tables.TemplateColumn(template_code=SITE_REGION_LINK) tenant = tables.TemplateColumn(template_code=COL_TENANT) @@ -291,7 +291,7 @@ class RackRoleTable(BaseTable): class RackTable(BaseTable): pk = ToggleColumn() - name = tables.LinkColumn() + name = tables.LinkColumn(order_by=('_name',)) site = tables.LinkColumn('dcim:site', args=[Accessor('site.slug')]) group = tables.Column(accessor=Accessor('group.name'), verbose_name='Group') tenant = tables.TemplateColumn(template_code=COL_TENANT) @@ -409,6 +409,7 @@ class DeviceTypeTable(BaseTable): class ConsolePortTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) actions = tables.TemplateColumn( template_code=get_component_template_actions('consoleporttemplate'), attrs={'td': {'class': 'text-right noprint'}}, @@ -432,6 +433,7 @@ class ConsolePortImportTable(BaseTable): class ConsoleServerPortTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) actions = tables.TemplateColumn( template_code=get_component_template_actions('consoleserverporttemplate'), attrs={'td': {'class': 'text-right noprint'}}, @@ -455,6 +457,7 @@ class ConsoleServerPortImportTable(BaseTable): class PowerPortTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) actions = tables.TemplateColumn( template_code=get_component_template_actions('powerporttemplate'), attrs={'td': {'class': 'text-right noprint'}}, @@ -478,6 +481,7 @@ class PowerPortImportTable(BaseTable): class PowerOutletTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) actions = tables.TemplateColumn( template_code=get_component_template_actions('poweroutlettemplate'), attrs={'td': {'class': 'text-right noprint'}}, @@ -526,6 +530,7 @@ class InterfaceImportTable(BaseTable): class FrontPortTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) rear_port_position = tables.Column( verbose_name='Position' ) @@ -552,6 +557,7 @@ class FrontPortImportTable(BaseTable): class RearPortTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) actions = tables.TemplateColumn( template_code=get_component_template_actions('rearporttemplate'), attrs={'td': {'class': 'text-right noprint'}}, @@ -575,6 +581,7 @@ class RearPortImportTable(BaseTable): class DeviceBayTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) actions = tables.TemplateColumn( template_code=get_component_template_actions('devicebaytemplate'), attrs={'td': {'class': 'text-right noprint'}}, @@ -653,7 +660,10 @@ class PlatformTable(BaseTable): class DeviceTable(BaseTable): pk = ToggleColumn() - name = tables.TemplateColumn(template_code=DEVICE_LINK) + name = tables.TemplateColumn( + order_by=('_name',), + template_code=DEVICE_LINK + ) status = tables.TemplateColumn(template_code=STATUS_LABEL, verbose_name='Status') tenant = tables.TemplateColumn(template_code=COL_TENANT) site = tables.LinkColumn('dcim:site', args=[Accessor('site.slug')]) @@ -701,6 +711,7 @@ class DeviceImportTable(BaseTable): class DeviceComponentDetailTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) cable = tables.LinkColumn() class Meta(BaseTable.Meta): @@ -710,6 +721,7 @@ class DeviceComponentDetailTable(BaseTable): class ConsolePortTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = ConsolePort @@ -724,6 +736,7 @@ class ConsolePortDetailTable(DeviceComponentDetailTable): class ConsoleServerPortTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = ConsoleServerPort @@ -738,6 +751,7 @@ class ConsoleServerPortDetailTable(DeviceComponentDetailTable): class PowerPortTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = PowerPort @@ -752,6 +766,7 @@ class PowerPortDetailTable(DeviceComponentDetailTable): class PowerOutletTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = PowerOutlet @@ -783,6 +798,7 @@ class InterfaceDetailTable(DeviceComponentDetailTable): class FrontPortTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = FrontPort @@ -798,6 +814,7 @@ class FrontPortDetailTable(DeviceComponentDetailTable): class RearPortTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = RearPort @@ -813,6 +830,7 @@ class RearPortDetailTable(DeviceComponentDetailTable): class DeviceBayTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = DeviceBay From 7c74d2ca657e3ed019abfee0230c59e3b425fd3c Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 7 Feb 2020 15:47:53 -0500 Subject: [PATCH 10/11] Convert interface models to use NaturalOrderingField --- netbox/dcim/managers.py | 56 +------------------ .../migrations/0096_interface_ordering.py | 53 ++++++++++++++++++ .../dcim/models/device_component_templates.py | 14 +++-- netbox/dcim/models/device_components.py | 15 +++-- netbox/utilities/ordering.py | 47 ++++++++++++++++ 5 files changed, 120 insertions(+), 65 deletions(-) create mode 100644 netbox/dcim/migrations/0096_interface_ordering.py diff --git a/netbox/dcim/managers.py b/netbox/dcim/managers.py index e1124b84e..502719646 100644 --- a/netbox/dcim/managers.py +++ b/netbox/dcim/managers.py @@ -1,18 +1,7 @@ from django.db.models import Manager, QuerySet -from django.db.models.expressions import RawSQL from .constants import NONCONNECTABLE_IFACE_TYPES -# Regular expressions for parsing Interface names -TYPE_RE = r"SUBSTRING({} FROM '^([^0-9\.:]+)')" -SLOT_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)?(\d{{1,9}})/') AS integer), NULL)" -SUBSLOT_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9\.:]+)?\d{{1,9}}/(\d{{1,9}})') AS integer), NULL)" -POSITION_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)?(?:\d{{1,9}}/){{2}}(\d{{1,9}})') AS integer), NULL)" -SUBPOSITION_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)?(?:\d{{1,9}}/){{3}}(\d{{1,9}})') AS integer), NULL)" -ID_RE = r"CAST(SUBSTRING({} FROM '^(?:[^0-9\.:]+)?(\d{{1,9}})([^/]|$)') AS integer)" -CHANNEL_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^.*:(\d{{1,9}})(\.\d{{1,9}})?$') AS integer), 0)" -VC_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^.*\.(\d{{1,9}})$') AS integer), 0)" - class InterfaceQuerySet(QuerySet): @@ -27,47 +16,4 @@ class InterfaceQuerySet(QuerySet): class InterfaceManager(Manager): def get_queryset(self): - """ - Naturally order interfaces by their type and numeric position. To order interfaces naturally, the `name` field - is split into eight distinct components: leading text (type), slot, subslot, position, subposition, ID, channel, - and virtual circuit: - - {type}{slot or ID}/{subslot}/{position}/{subposition}:{channel}.{vc} - - Components absent from the interface name are coalesced to zero or null. For example, an interface named - GigabitEthernet1/2/3 would be parsed as follows: - - type = 'GigabitEthernet' - slot = 1 - subslot = 2 - position = 3 - subposition = None - id = None - channel = 0 - vc = 0 - - The original `name` field is considered in its entirety to serve as a fallback in the event interfaces do not - match any of the prescribed fields. - - The `id` field is included to enforce deterministic ordering of interfaces in similar vein of other device - components. - """ - - sql_col = '{}.name'.format(self.model._meta.db_table) - ordering = [ - '_slot', '_subslot', '_position', '_subposition', '_type', '_id', '_channel', '_vc', 'name', 'pk' - - ] - - fields = { - '_type': RawSQL(TYPE_RE.format(sql_col), []), - '_id': RawSQL(ID_RE.format(sql_col), []), - '_slot': RawSQL(SLOT_RE.format(sql_col), []), - '_subslot': RawSQL(SUBSLOT_RE.format(sql_col), []), - '_position': RawSQL(POSITION_RE.format(sql_col), []), - '_subposition': RawSQL(SUBPOSITION_RE.format(sql_col), []), - '_channel': RawSQL(CHANNEL_RE.format(sql_col), []), - '_vc': RawSQL(VC_RE.format(sql_col), []), - } - - return InterfaceQuerySet(self.model, using=self._db).annotate(**fields).order_by(*ordering) + return InterfaceQuerySet(self.model, using=self._db) diff --git a/netbox/dcim/migrations/0096_interface_ordering.py b/netbox/dcim/migrations/0096_interface_ordering.py new file mode 100644 index 000000000..284066462 --- /dev/null +++ b/netbox/dcim/migrations/0096_interface_ordering.py @@ -0,0 +1,53 @@ +from django.db import migrations +import utilities.fields +import utilities.ordering + + +def _update_model_names(model): + # Update each unique field value in bulk + for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize_interface(name)) + + +def naturalize_interfacetemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'InterfaceTemplate')) + + +def naturalize_interfaces(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'Interface')) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dcim', '0095_primary_model_ordering'), + ] + + operations = [ + migrations.AlterModelOptions( + name='interface', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='interfacetemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AddField( + model_name='interface', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize_interface), + ), + migrations.AddField( + model_name='interfacetemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize_interface), + ), + migrations.RunPython( + code=naturalize_interfacetemplates, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_interfaces, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/netbox/dcim/models/device_component_templates.py b/netbox/dcim/models/device_component_templates.py index 0b5c312ba..ab4a078cf 100644 --- a/netbox/dcim/models/device_component_templates.py +++ b/netbox/dcim/models/device_component_templates.py @@ -4,9 +4,9 @@ from django.db import models from dcim.choices import * from dcim.constants import * -from dcim.managers import InterfaceManager from extras.models import ObjectChange from utilities.fields import NaturalOrderingField +from utilities.ordering import naturalize_interface from utilities.utils import serialize_object from .device_components import ( ConsolePort, ConsoleServerPort, DeviceBay, FrontPort, Interface, PowerOutlet, PowerPort, RearPort, @@ -249,6 +249,12 @@ class InterfaceTemplate(ComponentTemplateModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + naturalize_function=naturalize_interface, + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=InterfaceTypeChoices @@ -258,11 +264,9 @@ class InterfaceTemplate(ComponentTemplateModel): verbose_name='Management only' ) - objects = InterfaceManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 3eb9ac74e..a41eda576 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -10,9 +10,9 @@ from dcim.choices import * from dcim.constants import * from dcim.exceptions import LoopDetected from dcim.fields import MACAddressField -from dcim.managers import InterfaceManager from extras.models import ObjectChange, TaggedItem from utilities.fields import NaturalOrderingField +from utilities.ordering import naturalize_interface from utilities.utils import serialize_object from virtualization.choices import VMInterfaceTypeChoices @@ -529,6 +529,12 @@ class Interface(CableTermination, ComponentModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + naturalize_function=naturalize_interface, + max_length=100, + blank=True + ) _connected_interface = models.OneToOneField( to='self', on_delete=models.SET_NULL, @@ -597,8 +603,6 @@ class Interface(CableTermination, ComponentModel): blank=True, verbose_name='Tagged VLANs' ) - - objects = InterfaceManager() tags = TaggableManager(through=TaggedItem) csv_headers = [ @@ -607,8 +611,9 @@ class Interface(CableTermination, ComponentModel): ] class Meta: - ordering = ['device', 'name'] - unique_together = ['device', 'name'] + # TODO: ordering and unique_together should include virtual_machine + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return self.name diff --git a/netbox/utilities/ordering.py b/netbox/utilities/ordering.py index 88a46d3d3..d459e6f6c 100644 --- a/netbox/utilities/ordering.py +++ b/netbox/utilities/ordering.py @@ -1,5 +1,14 @@ import re +INTERFACE_NAME_REGEX = r'(^(?P[^\d\.:]+)?)' \ + r'((?P\d+)/)?' \ + r'((?P\d+)/)?' \ + r'((?P\d+)/)?' \ + r'((?P\d+)/)?' \ + r'((?P\d+))?' \ + r'(:(?P\d+))?' \ + r'(.(?P\d+)$)?' + def naturalize(value, max_length=None, integer_places=8): """ @@ -31,3 +40,41 @@ def naturalize(value, max_length=None, integer_places=8): ret = ''.join(output) return ret[:max_length] if max_length else ret + + +def naturalize_interface(value, max_length=None): + """ + Similar in nature to naturalize(), but takes into account a particular naming format adapted from the old + InterfaceManager. + + :param value: The value to be naturalized + :param max_length: The maximum length of the returned string. Characters beyond this length will be stripped. + """ + output = [] + match = re.search(INTERFACE_NAME_REGEX, value) + if match is None: + return value + + # First, we order by slot/position, padding each to four digits. If a field is not present, + # set it to 9999 to ensure it is ordered last. + for part_name in ('slot', 'subslot', 'position', 'subposition'): + part = match.group(part_name) + if part is not None: + output.append(part.rjust(4, '0')) + else: + output.append('9999') + + # Append the type, if any. + if match.group('type') is not None: + output.append(match.group('type')) + + # Finally, append any remaining fields, left-padding to eight digits each. + for part_name in ('id', 'channel', 'vc'): + part = match.group(part_name) + if part is not None: + output.append(part.rjust(6, '0')) + else: + output.append('000000') + + ret = ''.join(output) + return ret[:max_length] if max_length else ret From 5bfd65b5fe278880c331c6b8168377e3b52051fb Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 7 Feb 2020 16:18:15 -0500 Subject: [PATCH 11/11] Changelog for #3799 --- docs/release-notes/version-2.7.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 8f341d2a1..44298fec3 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -2,6 +2,7 @@ ## Enhancements +* [#3799](https://github.com/netbox-community/netbox/issues/3799) - Greatly improve performance when ordering device components * [#4100](https://github.com/netbox-community/netbox/issues/4100) - Add device filter to component list views * [#4113](https://github.com/netbox-community/netbox/issues/4113) - Add bulk edit functionality for device type components * [#4116](https://github.com/netbox-community/netbox/issues/4116) - Enable bulk edit and delete functions for device component list views