From 3fb967b482a8239da8b8932f7795bd7f49adc47b Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Sat, 30 Apr 2022 02:19:11 +0200 Subject: [PATCH 1/9] Add ability to adopt components when adding a module --- netbox/dcim/forms/models.py | 15 +++++++++-- netbox/dcim/models/devices.py | 51 ++++++++++++++++++++--------------- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/netbox/dcim/forms/models.py b/netbox/dcim/forms/models.py index 31c5b957d..c8ca1daf1 100644 --- a/netbox/dcim/forms/models.py +++ b/netbox/dcim/forms/models.py @@ -633,12 +633,18 @@ class ModuleForm(NetBoxModelForm): help_text="Automatically populate components associated with this module type" ) + adopt_components = forms.BooleanField( + required=False, + initial=False, + help_text="Adopt already existing components" + ) + fieldsets = ( ('Module', ( 'device', 'module_bay', 'manufacturer', 'module_type', 'tags', )), ('Hardware', ( - 'serial', 'asset_tag', 'replicate_components', + 'serial', 'asset_tag', 'replicate_components', 'adopt_components', )), ) @@ -646,7 +652,7 @@ class ModuleForm(NetBoxModelForm): model = Module fields = [ 'device', 'module_bay', 'manufacturer', 'module_type', 'serial', 'asset_tag', 'tags', - 'replicate_components', 'comments', + 'replicate_components', 'adopt_components', 'comments', ] def __init__(self, *args, **kwargs): @@ -655,6 +661,8 @@ class ModuleForm(NetBoxModelForm): if self.instance.pk: self.fields['replicate_components'].initial = False self.fields['replicate_components'].disabled = True + self.fields['adopt_components'].initial = False + self.fields['adopt_components'].disabled = True def save(self, *args, **kwargs): @@ -662,6 +670,9 @@ class ModuleForm(NetBoxModelForm): if self.instance.pk or not self.cleaned_data['replicate_components']: self.instance._disable_replication = True + if self.cleaned_data['adopt_components']: + self.instance._adopt_components = True + return super().save(*args, **kwargs) diff --git a/netbox/dcim/models/devices.py b/netbox/dcim/models/devices.py index 6ed7b349f..f0c7f31cb 100644 --- a/netbox/dcim/models/devices.py +++ b/netbox/dcim/models/devices.py @@ -1065,31 +1065,38 @@ class Module(NetBoxModel, ConfigContextModel): super().save(*args, **kwargs) + adopt_components = getattr(self, '_adopt_components', False) + disable_replication = getattr(self, '_disable_replication', False) + # If this is a new Module and component replication has not been disabled, instantiate all its # related components per the ModuleType definition - if is_new and not getattr(self, '_disable_replication', False): - ConsolePort.objects.bulk_create( - [x.instantiate(device=self.device, module=self) for x in self.module_type.consoleporttemplates.all()] - ) - ConsoleServerPort.objects.bulk_create( - [x.instantiate(device=self.device, module=self) for x in self.module_type.consoleserverporttemplates.all()] - ) - PowerPort.objects.bulk_create( - [x.instantiate(device=self.device, module=self) for x in self.module_type.powerporttemplates.all()] - ) - PowerOutlet.objects.bulk_create( - [x.instantiate(device=self.device, module=self) for x in self.module_type.poweroutlettemplates.all()] - ) - Interface.objects.bulk_create( - [x.instantiate(device=self.device, module=self) for x in self.module_type.interfacetemplates.all()] - ) - RearPort.objects.bulk_create( - [x.instantiate(device=self.device, module=self) for x in self.module_type.rearporttemplates.all()] - ) - FrontPort.objects.bulk_create( - [x.instantiate(device=self.device, module=self) for x in self.module_type.frontporttemplates.all()] - ) + if is_new and not disable_replication: + # Iterate all component templates + for templates, component_attribute in [ + ("consoleporttemplates", "consoleports"), + ("consoleserverporttemplates", "consoleserverports"), + ("interfacetemplates", "interfaces"), + ("powerporttemplates", "powerports"), + ("poweroutlettemplates", "poweroutlets"), + ("rearporttemplates", "rearports"), + ("frontporttemplates", "frontports") + ]: + # Get the template for the module type. + for template in getattr(self.module_type, templates).all(): + template_instance = template.instantiate(device=self.device, module=self) + if adopt_components: + existing_item = getattr(self.device, component_attribute).filter(name=template_instance.name).first() + + # Check if there's a component with the same name already + if existing_item: + # Assign it to the module + existing_item.module = self + existing_item.save() + continue + + # If we are not adopting components or the component doesn't already exist + template_instance.save() # # Virtual chassis From c2a6a1c125fd4c2a286552c08529ebddf0bfc57c Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Mon, 2 May 2022 21:37:37 +0200 Subject: [PATCH 2/9] Create module components in bulk --- netbox/dcim/models/devices.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/netbox/dcim/models/devices.py b/netbox/dcim/models/devices.py index f0c7f31cb..25f07c3bd 100644 --- a/netbox/dcim/models/devices.py +++ b/netbox/dcim/models/devices.py @@ -1072,15 +1072,18 @@ class Module(NetBoxModel, ConfigContextModel): # related components per the ModuleType definition if is_new and not disable_replication: # Iterate all component templates - for templates, component_attribute in [ - ("consoleporttemplates", "consoleports"), - ("consoleserverporttemplates", "consoleserverports"), - ("interfacetemplates", "interfaces"), - ("powerporttemplates", "powerports"), - ("poweroutlettemplates", "poweroutlets"), - ("rearporttemplates", "rearports"), - ("frontporttemplates", "frontports") + for templates, component_attribute, component_model in [ + ("consoleporttemplates", "consoleports", ConsolePort), + ("consoleserverporttemplates", "consoleserverports", ConsoleServerPort), + ("interfacetemplates", "interfaces", Interface), + ("powerporttemplates", "powerports", PowerPort), + ("poweroutlettemplates", "poweroutlets", PowerOutlet), + ("rearporttemplates", "rearports", RearPort), + ("frontporttemplates", "frontports", FrontPort) ]: + create_instances = [] + update_instances = [] + # Get the template for the module type. for template in getattr(self.module_type, templates).all(): template_instance = template.instantiate(device=self.device, module=self) @@ -1092,11 +1095,15 @@ class Module(NetBoxModel, ConfigContextModel): if existing_item: # Assign it to the module existing_item.module = self - existing_item.save() + update_instances.append(existing_item) continue # If we are not adopting components or the component doesn't already exist - template_instance.save() + create_instances.append(template_instance) + + component_model.objects.bulk_create(create_instances) + component_model.objects.bulk_update(update_instances, ['module']) + # # Virtual chassis From 977ccb01f2f5d6407f0edfd29a4b64f7bd70b086 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Mon, 2 May 2022 21:55:34 +0200 Subject: [PATCH 3/9] Formatting: Remove whitespace on blank line --- netbox/dcim/models/devices.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox/dcim/models/devices.py b/netbox/dcim/models/devices.py index 25f07c3bd..980a4ea75 100644 --- a/netbox/dcim/models/devices.py +++ b/netbox/dcim/models/devices.py @@ -1097,10 +1097,10 @@ class Module(NetBoxModel, ConfigContextModel): existing_item.module = self update_instances.append(existing_item) continue - + # If we are not adopting components or the component doesn't already exist create_instances.append(template_instance) - + component_model.objects.bulk_create(create_instances) component_model.objects.bulk_update(update_instances, ['module']) From 8040804c753d070b386b41b650ec53bc10d08e26 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Tue, 3 May 2022 22:03:12 +0200 Subject: [PATCH 4/9] Allow mixture of component replication and adoption --- netbox/dcim/models/devices.py | 61 ++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/netbox/dcim/models/devices.py b/netbox/dcim/models/devices.py index 980a4ea75..023d3a83f 100644 --- a/netbox/dcim/models/devices.py +++ b/netbox/dcim/models/devices.py @@ -1068,41 +1068,44 @@ class Module(NetBoxModel, ConfigContextModel): adopt_components = getattr(self, '_adopt_components', False) disable_replication = getattr(self, '_disable_replication', False) - # If this is a new Module and component replication has not been disabled, instantiate all its - # related components per the ModuleType definition - if is_new and not disable_replication: - # Iterate all component templates - for templates, component_attribute, component_model in [ - ("consoleporttemplates", "consoleports", ConsolePort), - ("consoleserverporttemplates", "consoleserverports", ConsoleServerPort), - ("interfacetemplates", "interfaces", Interface), - ("powerporttemplates", "powerports", PowerPort), - ("poweroutlettemplates", "poweroutlets", PowerOutlet), - ("rearporttemplates", "rearports", RearPort), - ("frontporttemplates", "frontports", FrontPort) - ]: - create_instances = [] - update_instances = [] + # We skip adding components if the module is being edited or + # both replication and component adoption is disabled + if not is_new or (disable_replication and not adopt_components): + return - # Get the template for the module type. - for template in getattr(self.module_type, templates).all(): - template_instance = template.instantiate(device=self.device, module=self) + # Iterate all component types + for templates, component_attribute, component_model in [ + ("consoleporttemplates", "consoleports", ConsolePort), + ("consoleserverporttemplates", "consoleserverports", ConsoleServerPort), + ("interfacetemplates", "interfaces", Interface), + ("powerporttemplates", "powerports", PowerPort), + ("poweroutlettemplates", "poweroutlets", PowerOutlet), + ("rearporttemplates", "rearports", RearPort), + ("frontporttemplates", "frontports", FrontPort) + ]: + create_instances = [] + update_instances = [] - if adopt_components: - existing_item = getattr(self.device, component_attribute).filter(name=template_instance.name).first() + # Get the template for the module type. + for template in getattr(self.module_type, templates).all(): + template_instance = template.instantiate(device=self.device, module=self) - # Check if there's a component with the same name already - if existing_item: - # Assign it to the module - existing_item.module = self - update_instances.append(existing_item) - continue + if adopt_components: + existing_item = getattr(self.device, component_attribute).filter(name=template_instance.name).first() - # If we are not adopting components or the component doesn't already exist + # Check if there's a component with the same name already + if existing_item: + # Assign it to the module + existing_item.module = self + update_instances.append(existing_item) + continue + + # Only create new components if replication is enabled + if not disable_replication: create_instances.append(template_instance) - component_model.objects.bulk_create(create_instances) - component_model.objects.bulk_update(update_instances, ['module']) + component_model.objects.bulk_create(create_instances) + component_model.objects.bulk_update(update_instances, ['module']) # From f455f91ea3eeb38b1480ef30e6521157b783c782 Mon Sep 17 00:00:00 2001 From: kkthxbye <> Date: Wed, 4 May 2022 08:58:42 +0200 Subject: [PATCH 5/9] Add view test for module component adoption --- netbox/dcim/tests/test_views.py | 48 +++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/netbox/dcim/tests/test_views.py b/netbox/dcim/tests/test_views.py index 70eb4b659..b7020d663 100644 --- a/netbox/dcim/tests/test_views.py +++ b/netbox/dcim/tests/test_views.py @@ -1869,6 +1869,54 @@ class ModuleTestCase( self.assertHttpStatus(self.client.post(**request), 302) self.assertEqual(Interface.objects.filter(device=device).count(), 5) + @override_settings(EXEMPT_VIEW_PERMISSIONS=['*']) + def test_module_component_adoption(self): + self.add_permissions('dcim.add_module') + + interface_name = "Interface-1" + + # Add an interface to the ModuleType + module_type = ModuleType.objects.first() + InterfaceTemplate(module_type=module_type, name=interface_name).save() + + form_data = self.form_data.copy() + device = Device.objects.get(pk=form_data['device']) + + # Create a module with replicated components + form_data['module_bay'] = ModuleBay.objects.filter(device=device)[0] + form_data['replicate_components'] = True + request = { + 'path': self._get_url('add'), + 'data': post_data(form_data), + } + self.assertHttpStatus(self.client.post(**request), 302) + + # Check that the interface was created + initial_interface = Interface.objects.filter(device=device, name=interface_name).first() + self.assertIsNotNone(initial_interface) + + # Save the module id associated with the interface + initial_module_id = initial_interface.module.id + + # Create a second module (in the next bay) with adopted components + # The module id of the interface should change + form_data['module_bay'] = ModuleBay.objects.filter(device=device)[1] + form_data['replicate_components'] = False + form_data['adopt_components'] = True + request = { + 'path': self._get_url('add'), + 'data': post_data(form_data), + } + + self.assertHttpStatus(self.client.post(**request), 302) + + # Re-retrieve interface to get new module id + initial_interface.refresh_from_db() + updated_module_id = initial_interface.module.id + + # Check that the module id has changed + self.assertNotEqual(initial_module_id, updated_module_id) + class ConsolePortTestCase(ViewTestCases.DeviceComponentViewTestCase): model = ConsolePort From 7de27c69c054f382bf1baa68be9558476bab53fd Mon Sep 17 00:00:00 2001 From: kkthxbye <> Date: Wed, 4 May 2022 09:16:19 +0200 Subject: [PATCH 6/9] Fix PEP8 --- netbox/dcim/tests/test_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/dcim/tests/test_views.py b/netbox/dcim/tests/test_views.py index b7020d663..4104bd206 100644 --- a/netbox/dcim/tests/test_views.py +++ b/netbox/dcim/tests/test_views.py @@ -1894,7 +1894,7 @@ class ModuleTestCase( # Check that the interface was created initial_interface = Interface.objects.filter(device=device, name=interface_name).first() self.assertIsNotNone(initial_interface) - + # Save the module id associated with the interface initial_module_id = initial_interface.module.id From 81c7fe2084b59dcfc16c821f661119bd95adf6f0 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Wed, 4 May 2022 22:59:28 +0200 Subject: [PATCH 7/9] Don't adopt components already belonging to a module --- netbox/dcim/models/devices.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/netbox/dcim/models/devices.py b/netbox/dcim/models/devices.py index 023d3a83f..bcf0f6e79 100644 --- a/netbox/dcim/models/devices.py +++ b/netbox/dcim/models/devices.py @@ -1091,7 +1091,8 @@ class Module(NetBoxModel, ConfigContextModel): template_instance = template.instantiate(device=self.device, module=self) if adopt_components: - existing_item = getattr(self.device, component_attribute).filter(name=template_instance.name).first() + existing_item = getattr(self.device, component_attribute).filter( + module__isnull=True, name=template_instance.name).first() # Check if there's a component with the same name already if existing_item: From c52aa2196df72f30553c1610905dd3a5b0745982 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Wed, 4 May 2022 23:21:03 +0200 Subject: [PATCH 8/9] Prefetch installed components when adding modules --- netbox/dcim/models/devices.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/netbox/dcim/models/devices.py b/netbox/dcim/models/devices.py index bcf0f6e79..8d50db958 100644 --- a/netbox/dcim/models/devices.py +++ b/netbox/dcim/models/devices.py @@ -1086,13 +1086,17 @@ class Module(NetBoxModel, ConfigContextModel): create_instances = [] update_instances = [] + # Prefetch installed components + installed_components = { + component.name: component for component in getattr(self.device, component_attribute).filter(module__isnull=True) + } + # Get the template for the module type. for template in getattr(self.module_type, templates).all(): template_instance = template.instantiate(device=self.device, module=self) if adopt_components: - existing_item = getattr(self.device, component_attribute).filter( - module__isnull=True, name=template_instance.name).first() + existing_item = installed_components.get(template_instance.name) # Check if there's a component with the same name already if existing_item: From 9c3dfdfd14fe4321bbcdc1b642ea79fd2e176a60 Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Thu, 5 May 2022 09:30:13 +0200 Subject: [PATCH 9/9] Fix test_module_component_adoption --- netbox/dcim/tests/test_views.py | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/netbox/dcim/tests/test_views.py b/netbox/dcim/tests/test_views.py index 4104bd206..e17f94682 100644 --- a/netbox/dcim/tests/test_views.py +++ b/netbox/dcim/tests/test_views.py @@ -1882,25 +1882,16 @@ class ModuleTestCase( form_data = self.form_data.copy() device = Device.objects.get(pk=form_data['device']) - # Create a module with replicated components - form_data['module_bay'] = ModuleBay.objects.filter(device=device)[0] - form_data['replicate_components'] = True - request = { - 'path': self._get_url('add'), - 'data': post_data(form_data), - } - self.assertHttpStatus(self.client.post(**request), 302) + # Create an interface to be adopted + interface = Interface(device=device, name=interface_name, type=InterfaceTypeChoices.TYPE_10GE_FIXED) + interface.save() - # Check that the interface was created - initial_interface = Interface.objects.filter(device=device, name=interface_name).first() - self.assertIsNotNone(initial_interface) + # Ensure that interface is created with no module + self.assertIsNone(interface.module) - # Save the module id associated with the interface - initial_module_id = initial_interface.module.id - - # Create a second module (in the next bay) with adopted components - # The module id of the interface should change - form_data['module_bay'] = ModuleBay.objects.filter(device=device)[1] + # Create a module with adopted components + form_data['module_bay'] = ModuleBay.objects.filter(device=device).first() + form_data['module_type'] = module_type form_data['replicate_components'] = False form_data['adopt_components'] = True request = { @@ -1911,11 +1902,10 @@ class ModuleTestCase( self.assertHttpStatus(self.client.post(**request), 302) # Re-retrieve interface to get new module id - initial_interface.refresh_from_db() - updated_module_id = initial_interface.module.id + interface.refresh_from_db() - # Check that the module id has changed - self.assertNotEqual(initial_module_id, updated_module_id) + # Check that the Interface now has a module + self.assertIsNotNone(interface.module) class ConsolePortTestCase(ViewTestCases.DeviceComponentViewTestCase):