From 84c9ef814496315c30faf5b25e2befa0cdeb2e6e Mon Sep 17 00:00:00 2001 From: Sander Steffann Date: Sat, 19 Oct 2019 22:55:07 +0200 Subject: [PATCH] Adding trace information for quick display --- .../0016_generic_connected_endpoint.py | 38 +++ ...rcuittermination_old_connected_endpoint.py | 18 ++ netbox/circuits/models.py | 37 ++- .../0076_add_generic_connected_endpoint.py | 6 + .../0077_migrate_connected_endpoint.py | 223 +++++++++++++++--- netbox/dcim/models.py | 72 ++++-- netbox/dcim/signals.py | 48 ++-- 7 files changed, 366 insertions(+), 76 deletions(-) create mode 100644 netbox/circuits/migrations/0016_generic_connected_endpoint.py create mode 100644 netbox/circuits/migrations/0017_remove_circuittermination_old_connected_endpoint.py diff --git a/netbox/circuits/migrations/0016_generic_connected_endpoint.py b/netbox/circuits/migrations/0016_generic_connected_endpoint.py new file mode 100644 index 000000000..fe2e1af56 --- /dev/null +++ b/netbox/circuits/migrations/0016_generic_connected_endpoint.py @@ -0,0 +1,38 @@ +# Generated by Django 2.2.5 on 2019-10-09 14:56 + +import django.contrib.postgres.fields.jsonb +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('circuits', '0015_custom_tag_models'), + ] + + operations = [ + migrations.RenameField( + model_name='circuittermination', + old_name='connected_endpoint', + new_name='old_connected_endpoint', + ), + migrations.AddField( + model_name='circuittermination', + name='connected_endpoint_id', + field=models.PositiveIntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='circuittermination', + name='connected_endpoint_type', + field=models.ForeignKey(blank=True, limit_choices_to={ + 'model__in': ['consoleport', 'consoleserverport', 'interface', 'poweroutlet', 'powerport', 'frontport', + 'rearport', 'circuittermination'] + }, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'), + ), + migrations.AddField( + model_name='circuittermination', + name='_trace', + field=django.contrib.postgres.fields.jsonb.JSONField(default=list), + ), + ] diff --git a/netbox/circuits/migrations/0017_remove_circuittermination_old_connected_endpoint.py b/netbox/circuits/migrations/0017_remove_circuittermination_old_connected_endpoint.py new file mode 100644 index 000000000..348e5b98b --- /dev/null +++ b/netbox/circuits/migrations/0017_remove_circuittermination_old_connected_endpoint.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.5 on 2019-10-09 19:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('circuits', '0016_generic_connected_endpoint'), + ('dcim', '0077_migrate_connected_endpoint'), + ] + + operations = [ + migrations.RemoveField( + model_name='circuittermination', + name='old_connected_endpoint', + ), + ] diff --git a/netbox/circuits/models.py b/netbox/circuits/models.py index dd5d4b4f5..f027d49a5 100644 --- a/netbox/circuits/models.py +++ b/netbox/circuits/models.py @@ -1,9 +1,11 @@ -from django.contrib.contenttypes.fields import GenericRelation +from django.contrib.contenttypes.fields import GenericRelation, GenericForeignKey +from django.contrib.contenttypes.models import ContentType +from django.contrib.postgres.fields import JSONField from django.db import models from django.urls import reverse from taggit.managers import TaggableManager -from dcim.constants import CONNECTION_STATUS_CHOICES, STATUS_CLASSES +from dcim.constants import CONNECTION_STATUS_CHOICES, STATUS_CLASSES, CABLE_TERMINATION_TYPES from dcim.fields import ASNField from dcim.models import CableTermination from extras.models import CustomFieldModel, ObjectChange, TaggedItem @@ -228,13 +230,26 @@ class CircuitTermination(CableTermination): on_delete=models.PROTECT, related_name='circuit_terminations' ) - connected_endpoint = models.OneToOneField( - to='dcim.Interface', - on_delete=models.SET_NULL, + connected_endpoint_type = models.ForeignKey( + to=ContentType, + limit_choices_to={'model__in': CABLE_TERMINATION_TYPES}, + on_delete=models.PROTECT, related_name='+', blank=True, null=True ) + connected_endpoint_id = models.PositiveIntegerField( + blank=True, + null=True + ) + connected_endpoint = GenericForeignKey( + ct_field='connected_endpoint_type', + fk_field='connected_endpoint_id' + ) + _trace = JSONField( + default=list + ) + connection_status = models.NullBooleanField( choices=CONNECTION_STATUS_CHOICES, blank=True @@ -298,7 +313,11 @@ class CircuitTermination(CableTermination): return None def get_peer_port(self): - peer_termination = self.get_peer_termination() - if peer_termination is None: - return None - return peer_termination + return self.get_peer_termination() + + def get_endpoint_attributes(self): + return { + **super().get_endpoint_attributes(), + 'cid': self.circuit.cid, + 'site': self.site.name, + } diff --git a/netbox/dcim/migrations/0076_add_generic_connected_endpoint.py b/netbox/dcim/migrations/0076_add_generic_connected_endpoint.py index 455df91ac..3f1d12e68 100644 --- a/netbox/dcim/migrations/0076_add_generic_connected_endpoint.py +++ b/netbox/dcim/migrations/0076_add_generic_connected_endpoint.py @@ -1,5 +1,6 @@ # Generated by Django 2.2.5 on 2019-10-06 19:01 +import django.contrib.postgres.fields.jsonb import django.db.models.deletion from django.db import migrations, models @@ -24,4 +25,9 @@ class Migration(migrations.Migration): 'rearport', 'circuittermination'] }, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='contenttypes.ContentType'), ), + migrations.AddField( + model_name='interface', + name='_trace', + field=django.contrib.postgres.fields.jsonb.JSONField(default=list), + ), ] diff --git a/netbox/dcim/migrations/0077_migrate_connected_endpoint.py b/netbox/dcim/migrations/0077_migrate_connected_endpoint.py index ad714359e..f06cffc54 100644 --- a/netbox/dcim/migrations/0077_migrate_connected_endpoint.py +++ b/netbox/dcim/migrations/0077_migrate_connected_endpoint.py @@ -1,61 +1,220 @@ # Generated by Django 2.2.5 on 2019-10-06 19:01 +from itertools import chain from django.db import migrations -def connected_interface_to_endpoint(apps, schema_editor): +def migration_trace(apps, endpoint, cable_history=None): + """ + Return a list representing a complete cable path, with each individual segment represented as a three-tuple: + [ + (termination A, cable, termination B), + (termination C, cable, termination D), + (termination E, cable, termination F) + ] + + This is a stand-alone version of Interface.trace() that is safe to use in migrations. + """ + + def get_generic_foreign_key(model_type, object_id): + if model_type is None or object_id is None: + return None + + object_model = apps.get_model(model_type.app_label, model_type.model) + return object_model.objects.filter(pk=object_id).first() + + circuittermination_model = apps.get_model('circuits', 'CircuitTermination') + frontport_model = apps.get_model('dcim', 'FrontPort') + rearport_model = apps.get_model('dcim', 'RearPort') + + if not endpoint.cable: + return [(endpoint, None, None)] + + # Record cable history to detect loops + if cable_history is None: + cable_history = [] + elif endpoint.cable in cable_history: + return None + cable_history.append(endpoint.cable) + + termination_a = get_generic_foreign_key(endpoint.cable.termination_a_type, endpoint.cable.termination_a_id) + termination_b = get_generic_foreign_key(endpoint.cable.termination_b_type, endpoint.cable.termination_b_id) + far_end = termination_b if termination_a == endpoint else termination_a + path = [(endpoint, endpoint.cable, far_end)] + + if isinstance(far_end, circuittermination_model): + peer_side = 'Z' if far_end.term_side == 'A' else 'A' + peer_port = circuittermination_model.objects.filter( + circuit=far_end.circuit, + term_side=peer_side + ).first() + elif isinstance(far_end, frontport_model): + peer_port = far_end.rear_port + elif isinstance(far_end, rearport_model): + peer_port = frontport_model.objects.filter( + rear_port=far_end, + rear_port_position=1, + ).first() + else: + peer_port = None + + if isinstance(far_end, rearport_model) and far_end.positions > 1: + # When we end up here we have a rear port with multiple front ports, and we don't know which front port + # to continue with. So this is the end of the line + return path + + while peer_port: + path += migration_trace(apps, peer_port, cable_history) + + if isinstance(far_end, frontport_model) and isinstance(peer_port, rearport_model) and peer_port.positions > 1: + # Trace the rear port separately, then continue with the corresponding front port + saved_rear_port_position = far_end.rear_port_position + + far_end = path[-1][2] + peer_port = None + if isinstance(far_end, rearport_model): + # The trace ends with a rear port, find the corresponding front port + peer_port = frontport_model.objects.filter( + rear_port=far_end, + rear_port_position=saved_rear_port_position, + ).first() + + else: + # Everything else has already been handled by simple recursion + peer_port = None + + return path + + +def migration_get_endpoint_attributes(endpoint): + """ + This is a stand-alone version of CableTermination.get_endpoint_attributes() that is safe to use in migrations. + """ + + if not endpoint: + return {} + + # We can't use isinstance because migrations give us fake classes + attributes = { + 'id': endpoint.pk, + 'type': endpoint.__class__.__name__, + } + + if endpoint.__class__.__name__ == 'CircuitTermination': + attributes['cid'] = endpoint.circuit.cid + attributes['site'] = endpoint.site.name + elif endpoint.__class__.__name__ in ('Interface', 'FrontPort', 'RearPort'): + attributes['name'] = endpoint.name + + parent = endpoint.device or endpoint.virtual_machine + if parent.name: + attributes['device'] = parent.name + attributes['device_id'] = parent.pk + elif parent.virtual_chassis and parent.virtual_chassis.master.name: + attributes['device'] = "{}:{}".format(parent.virtual_chassis.master, parent.vc_position) + attributes['device_id'] = parent.virtual_chassis.master.pk + elif hasattr(parent, 'device_type'): + attributes['device'] = "{}".format(parent.device_type) + attributes['device_id'] = parent.pk + else: + attributes['device'] = "" + + attributes['site'] = parent.site.name + + return attributes + + +def to_generic_connected_endpoint(apps, schema_editor): + print("\nReconstructing all endpoints...", end='') + interface_model = apps.get_model('dcim', 'Interface') + circuittermination_model = apps.get_model('circuits', 'CircuitTermination') contenttype_model = apps.get_model('contenttypes', 'ContentType') + db_alias = schema_editor.connection.alias - model_type = contenttype_model.objects.get_for_model(interface_model) - for interface in interface_model.objects.exclude(_connected_interface=None): - interface.connected_endpoint_type = model_type - interface.connected_endpoint_id = interface._connected_interface.pk - interface.save() + interface_endpoints = interface_model.objects.using(db_alias).all() + circuittermination_endpoints = circuittermination_model.objects.using(db_alias).all() + for endpoint in chain(interface_endpoints, circuittermination_endpoints): + path = migration_trace(apps, endpoint) + + # The trace returns left and right, we just want a single list + # We also want to skip the first endpoint, which is the starting point itself + endpoints = [ + item for sublist in ( + [left, right] for left, cable, right in path + ) + for item in sublist if item + ][1:] + + if endpoints: + model_type = contenttype_model.objects.get_for_model(endpoints[-1]) + endpoint.connected_endpoint_type = model_type + endpoint.connected_endpoint_id = endpoints[-1].pk + else: + endpoint.connected_endpoint_type = None + endpoint.connected_endpoint_id = None + + endpoint._trace = [migration_get_endpoint_attributes(endpoint) for endpoint in endpoints] + endpoint.save() + + print(".", end='', flush=True) -def connected_endpoint_to_interface(apps, schema_editor): - interface_model = apps.get_model('dcim', 'Interface') - contenttype_model = apps.get_model('contenttypes', 'ContentType') +def from_generic_connected_endpoint(apps, schema_editor): + db_alias = schema_editor.connection.alias - model_type = contenttype_model.objects.get_for_model(interface_model) - for interface in interface_model.objects.filter(connected_endpoint_type=model_type): - interface._connected_interface = interface.connected_endpoint - interface.save() - - -def connected_circuittermination_to_endpoint(apps, schema_editor): interface_model = apps.get_model('dcim', 'Interface') contenttype_model = apps.get_model('contenttypes', 'ContentType') circuittermination_model = apps.get_model('circuits', 'CircuitTermination') - model_type = contenttype_model.objects.get_for_model(circuittermination_model) - for interface in interface_model.objects.exclude(_connected_circuittermination=None): - interface.connected_endpoint_type = model_type - interface.connected_endpoint_id = interface._connected_circuittermination.pk - interface.save() + print("\nReverting interface endpoints in interfaces...", end='') + model_type = contenttype_model.objects.get_for_model(interface_model) + for interface in interface_model.objects.using(db_alias).filter(connected_endpoint_type=model_type): + try: + interface._connected_interface_id = interface.connected_endpoint_id + interface.save() + except interface_model.DoesNotExist: + # Dangling generic foreign key + pass -def connected_endpoint_to_circuittermination(apps, schema_editor): - interface_model = apps.get_model('dcim', 'Interface') - contenttype_model = apps.get_model('contenttypes', 'ContentType') - circuittermination_model = apps.get_model('circuits', 'CircuitTermination') + print(".", end='', flush=True) + + print("\nReverting circuit termination endpoints in interfaces...", end='') model_type = contenttype_model.objects.get_for_model(circuittermination_model) - for interface in interface_model.objects.filter(connected_endpoint_type=model_type): - interface._connected_circuittermination = interface.connected_endpoint - interface.save() + for interface in interface_model.objects.using(db_alias).filter(connected_endpoint_type=model_type): + try: + interface._connected_circuittermination_id = interface.connected_endpoint_id + interface.save() + except circuittermination_model.DoesNotExist: + # Dangling generic foreign key + pass + + print(".", end='', flush=True) + + print("\nReverting circuit termination endpoints...", end='') + + model_type = contenttype_model.objects.get_for_model(interface_model) + for interface in circuittermination_model.objects.using(db_alias).filter(connected_endpoint_type=model_type): + try: + interface.old_connected_interface_id = interface.connected_endpoint_id + interface.save() + except interface_model.DoesNotExist: + # Dangling generic foreign key + pass + + print(".", end='', flush=True) class Migration(migrations.Migration): dependencies = [ + ('circuits', '0016_generic_connected_endpoint'), ('contenttypes', '0002_remove_content_type_name'), ('dcim', '0076_add_generic_connected_endpoint'), ] operations = [ - migrations.RunPython(connected_interface_to_endpoint, - connected_endpoint_to_interface), - migrations.RunPython(connected_circuittermination_to_endpoint, - connected_endpoint_to_circuittermination), + migrations.RunPython(to_generic_connected_endpoint, + from_generic_connected_endpoint), ] diff --git a/netbox/dcim/models.py b/netbox/dcim/models.py index 070b71f6f..700110654 100644 --- a/netbox/dcim/models.py +++ b/netbox/dcim/models.py @@ -160,6 +160,11 @@ class CableTermination(models.Model): if self._cabled_as_b.exists(): return self.cable.termination_a + def get_endpoint_attributes(self): + return { + 'id': self.pk, + 'type': self.__class__.__name__, + } # # Regions @@ -2168,6 +2173,9 @@ class Interface(CableTermination, ComponentModel): ct_field='connected_endpoint_type', fk_field='connected_endpoint_id' ) + _trace = JSONField( + default=list + ) connected_interface = GenericRelation( to='self', @@ -2382,6 +2390,15 @@ class Interface(CableTermination, ComponentModel): def count_ipaddresses(self): return self.ip_addresses.count() + def get_endpoint_attributes(self): + return { + **super().get_endpoint_attributes(), + 'name': self.name, + 'device': self.parent.display_name, + 'device_id': self.parent.pk, + 'site': self.parent.site.name, + } + # # Pass-through ports @@ -2456,6 +2473,15 @@ class FrontPort(CableTermination, ComponentModel): def get_peer_port(self): return self.rear_port + def get_endpoint_attributes(self): + return { + **super().get_endpoint_attributes(), + 'name': self.name, + 'device': self.parent.display_name, + 'device_id': self.parent.pk, + 'site': self.parent.site.name, + } + class RearPort(CableTermination, ComponentModel): """ @@ -2513,6 +2539,15 @@ class RearPort(CableTermination, ComponentModel): except ObjectDoesNotExist: return None + def get_endpoint_attributes(self): + return { + **super().get_endpoint_attributes(), + 'name': self.name, + 'device': self.parent.display_name, + 'device_id': self.parent.pk, + 'site': self.parent.site.name, + } + # # Device bays @@ -2933,31 +2968,34 @@ class Cable(ChangeLoggedModel): return return COMPATIBLE_TERMINATION_TYPES[self.termination_a._meta.model_name] - def get_path_endpoints(self): + def get_related_endpoints(self): """ - Traverse both ends of a cable path and return its connected endpoints. Note that one or both endpoints may be - None. + Traverse both ends of a cable path and return a list of all related endpoints. """ # Termination points trace from themselves, through the cable and beyond. Tracing from the B termination # therefore traces in the direction of A [(termination_b, cable, termination_a), (...)] and vice versa. # Every path therefore also has at least one segment (the current cable). - a_path = self.termination_b.trace() - b_path = self.termination_a.trace() + paths = [ + self.termination_b.trace(), + self.termination_a.trace(), + ] - # Determine overall path status (connected or planned) - if self.status == CONNECTION_STATUS_PLANNED: - path_status = CONNECTION_STATUS_PLANNED - else: - path_status = CONNECTION_STATUS_CONNECTED - for segment in a_path[1:] + b_path[1:]: - if segment[1] is None or segment[1].status == CONNECTION_STATUS_PLANNED: - path_status = CONNECTION_STATUS_PLANNED - break + # Use a dict here to avoid storing duplicates. The same object retrieved twice will have different identities. + endpoints = {} + while paths: + path = paths.pop() + for left, cable, right in path: + if right is not None: + key = '{cls}-{pk}'.format(cls=right.__class__.__name__, pk=right.pk) + endpoints[key] = right - a_endpoint = a_path[-1][2] - b_endpoint = b_path[-1][2] + # If a path ends in a RearPort, then everything connected through its FrontPorts is related as well + if isinstance(path[-1][2], RearPort): + front_ports = FrontPort.objects.filter(rear_port=path[-1][2]) + for front_port in front_ports: + paths.append(front_port.trace()) - return a_endpoint, b_endpoint, path_status + return list(endpoints.values()) # diff --git a/netbox/dcim/signals.py b/netbox/dcim/signals.py index c1aabf64d..f83bb9d46 100644 --- a/netbox/dcim/signals.py +++ b/netbox/dcim/signals.py @@ -43,15 +43,9 @@ def update_connected_endpoints(instance, **kwargs): instance.termination_b.cable = instance instance.termination_b.save() - # Check if this Cable has formed a complete path. If so, update both endpoints. - endpoint_a, endpoint_b, path_status = instance.get_path_endpoints() - if endpoint_a is not None and endpoint_b is not None: - endpoint_a.connected_endpoint = endpoint_b - endpoint_a.connection_status = path_status - endpoint_a.save() - endpoint_b.connected_endpoint = endpoint_a - endpoint_b.connection_status = path_status - endpoint_b.save() + # Update all endpoints affected by this cable + endpoints = instance.get_related_endpoints() + update_endpoints(endpoints) @receiver(pre_delete, sender=Cable) @@ -59,7 +53,7 @@ def nullify_connected_endpoints(instance, **kwargs): """ When a Cable is deleted, check for and update its two connected endpoints """ - endpoint_a, endpoint_b, _ = instance.get_path_endpoints() + endpoints = instance.get_related_endpoints() # Disassociate the Cable from its termination points if instance.termination_a is not None: @@ -69,11 +63,29 @@ def nullify_connected_endpoints(instance, **kwargs): instance.termination_b.cable = None instance.termination_b.save() - # If this Cable was part of a complete path, tear it down - if hasattr(endpoint_a, 'connected_endpoint') and hasattr(endpoint_b, 'connected_endpoint'): - endpoint_a.connected_endpoint = None - endpoint_a.connection_status = None - endpoint_a.save() - endpoint_b.connected_endpoint = None - endpoint_b.connection_status = None - endpoint_b.save() + # Update all endpoints affected by this cable + update_endpoints(endpoints) + + +def update_endpoints(endpoints): + """ + Update all endpoints affected by this cable + """ + for endpoint in endpoints: + if not hasattr(endpoint, 'connected_endpoint'): + continue + + path = endpoint.trace() + + # The trace returns left and right, we just want a single list + # We also want to skip the first endpoint, which is the starting point itself + endpoints = [ + item for sublist in ( + [left, right] for left, cable, right in path + ) + for item in sublist if item + ][1:] + + endpoint.connected_endpoint = endpoints[-1] if endpoints else None + endpoint._trace = [endpoint.get_endpoint_attributes() for endpoint in endpoints] + endpoint.save()