From 149a4960116ee4a3889b06da67859e7cda8e0bc2 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Tue, 25 Jul 2023 20:39:05 +0700 Subject: [PATCH 1/2] 6347 Cache the number of each component type assigned to devices/VMs (#12632) --------- Co-authored-by: Jeremy Stretch --- docs/development/application-registry.md | 4 + netbox/dcim/api/serializers.py | 26 ++++- netbox/dcim/apps.py | 6 +- .../0175_device_component_counters.py | 100 ++++++++++++++++++ netbox/dcim/models/device_components.py | 21 ++-- netbox/dcim/models/devices.py | 44 +++++++- netbox/dcim/tables/devices.py | 36 ++++++- netbox/dcim/views.py | 18 ++-- netbox/netbox/models/__init__.py | 1 + netbox/netbox/registry.py | 1 + netbox/utilities/counters.py | 93 ++++++++++++++++ netbox/utilities/fields.py | 42 ++++++++ netbox/utilities/management/__init__.py | 0 .../utilities/management/commands/__init__.py | 0 .../commands/calculate_cached_counts.py | 52 +++++++++ netbox/utilities/tests/test_counters.py | 69 ++++++++++++ netbox/utilities/tracking.py | 78 ++++++++++++++ netbox/virtualization/api/serializers.py | 6 +- netbox/virtualization/apps.py | 5 + .../0035_virtualmachine_interface_count.py | 35 ++++++ .../virtualization/models/virtualmachines.py | 11 +- .../virtualization/tables/virtualmachines.py | 8 +- netbox/virtualization/views.py | 2 +- 23 files changed, 623 insertions(+), 35 deletions(-) create mode 100644 netbox/dcim/migrations/0175_device_component_counters.py create mode 100644 netbox/utilities/counters.py create mode 100644 netbox/utilities/management/__init__.py create mode 100644 netbox/utilities/management/commands/__init__.py create mode 100644 netbox/utilities/management/commands/calculate_cached_counts.py create mode 100644 netbox/utilities/tests/test_counters.py create mode 100644 netbox/utilities/tracking.py create mode 100644 netbox/virtualization/migrations/0035_virtualmachine_interface_count.py diff --git a/docs/development/application-registry.md b/docs/development/application-registry.md index fe2c08d56..41bf6cb31 100644 --- a/docs/development/application-registry.md +++ b/docs/development/application-registry.md @@ -8,6 +8,10 @@ The registry can be inspected by importing `registry` from `extras.registry`. ## Stores +### `counter_fields` + +A dictionary mapping of models to foreign keys with which cached counter fields are associated. + ### `data_backends` A dictionary mapping data backend types to their respective classes. These are used to interact with [remote data sources](../models/core/datasource.md). diff --git a/netbox/dcim/api/serializers.py b/netbox/dcim/api/serializers.py index 32943f468..835592161 100644 --- a/netbox/dcim/api/serializers.py +++ b/netbox/dcim/api/serializers.py @@ -669,14 +669,28 @@ class DeviceSerializer(NetBoxModelSerializer): vc_position = serializers.IntegerField(allow_null=True, max_value=255, min_value=0, default=None) config_template = NestedConfigTemplateSerializer(required=False, allow_null=True, default=None) + # Counter fields + console_port_count = serializers.IntegerField(read_only=True) + console_server_port_count = serializers.IntegerField(read_only=True) + power_port_count = serializers.IntegerField(read_only=True) + power_outlet_count = serializers.IntegerField(read_only=True) + interface_count = serializers.IntegerField(read_only=True) + front_port_count = serializers.IntegerField(read_only=True) + rear_port_count = serializers.IntegerField(read_only=True) + device_bay_count = serializers.IntegerField(read_only=True) + module_bay_count = serializers.IntegerField(read_only=True) + inventory_item_count = serializers.IntegerField(read_only=True) + class Meta: model = Device fields = [ 'id', 'url', 'display', 'name', 'device_type', 'device_role', 'tenant', 'platform', 'serial', 'asset_tag', - 'site', 'location', 'rack', 'position', 'face', 'latitude', 'longitude', 'parent_device', 'status', 'airflow', - 'primary_ip', 'primary_ip4', 'primary_ip6', 'cluster', 'virtual_chassis', 'vc_position', 'vc_priority', - 'description', 'comments', 'config_template', 'local_context_data', 'tags', 'custom_fields', 'created', - 'last_updated', + 'site', 'location', 'rack', 'position', 'face', 'latitude', 'longitude', 'parent_device', 'status', + 'airflow', 'primary_ip', 'primary_ip4', 'primary_ip6', 'cluster', 'virtual_chassis', 'vc_position', + 'vc_priority', 'description', 'comments', 'config_template', 'local_context_data', 'tags', 'custom_fields', + 'created', 'last_updated', 'console_port_count', 'console_server_port_count', 'power_port_count', + 'power_outlet_count', 'interface_count', 'front_port_count', 'rear_port_count', 'device_bay_count', + 'module_bay_count', 'inventory_item_count', ] @extend_schema_field(NestedDeviceSerializer) @@ -700,7 +714,9 @@ class DeviceWithConfigContextSerializer(DeviceSerializer): 'site', 'location', 'rack', 'position', 'face', 'parent_device', 'status', 'airflow', 'primary_ip', 'primary_ip4', 'primary_ip6', 'cluster', 'virtual_chassis', 'vc_position', 'vc_priority', 'description', 'comments', 'local_context_data', 'tags', 'custom_fields', 'config_context', 'config_template', - 'created', 'last_updated', + 'created', 'last_updated', 'console_port_count', 'console_server_port_count', 'power_port_count', + 'power_outlet_count', 'interface_count', 'front_port_count', 'rear_port_count', 'device_bay_count', + 'module_bay_count', 'inventory_item_count', ] @extend_schema_field(serializers.JSONField(allow_null=True)) diff --git a/netbox/dcim/apps.py b/netbox/dcim/apps.py index bfb09e601..cc4c65f93 100644 --- a/netbox/dcim/apps.py +++ b/netbox/dcim/apps.py @@ -9,7 +9,8 @@ class DCIMConfig(AppConfig): def ready(self): from . import signals, search - from .models import CableTermination + from .models import CableTermination, Device + from utilities.counters import connect_counters # Register denormalized fields denormalized.register(CableTermination, '_device', { @@ -24,3 +25,6 @@ class DCIMConfig(AppConfig): denormalized.register(CableTermination, '_location', { '_site': 'site', }) + + # Register counters + connect_counters(Device) diff --git a/netbox/dcim/migrations/0175_device_component_counters.py b/netbox/dcim/migrations/0175_device_component_counters.py new file mode 100644 index 000000000..9d033c103 --- /dev/null +++ b/netbox/dcim/migrations/0175_device_component_counters.py @@ -0,0 +1,100 @@ +from django.db import migrations +from django.db.models import Count + +import utilities.fields + + +def recalculate_device_counts(apps, schema_editor): + Device = apps.get_model("dcim", "Device") + devices = list(Device.objects.all().annotate( + _console_port_count=Count('consoleports', distinct=True), + _console_server_port_count=Count('consoleserverports', distinct=True), + _power_port_count=Count('powerports', distinct=True), + _power_outlet_count=Count('poweroutlets', distinct=True), + _interface_count=Count('interfaces', distinct=True), + _front_port_count=Count('frontports', distinct=True), + _rear_port_count=Count('rearports', distinct=True), + _device_bay_count=Count('devicebays', distinct=True), + _module_bay_count=Count('modulebays', distinct=True), + _inventory_item_count=Count('inventoryitems', distinct=True), + )) + + for device in devices: + device.console_port_count = device._console_port_count + device.console_server_port_count = device._console_server_port_count + device.power_port_count = device._power_port_count + device.power_outlet_count = device._power_outlet_count + device.interface_count = device._interface_count + device.front_port_count = device._front_port_count + device.rear_port_count = device._rear_port_count + device.device_bay_count = device._device_bay_count + device.module_bay_count = device._module_bay_count + device.inventory_item_count = device._inventory_item_count + + Device.objects.bulk_update(devices, [ + 'console_port_count', 'console_server_port_count', 'power_port_count', 'power_outlet_count', 'interface_count', + 'front_port_count', 'rear_port_count', 'device_bay_count', 'module_bay_count', 'inventory_item_count', + ]) + + +class Migration(migrations.Migration): + dependencies = [ + ('dcim', '0174_rack_starting_unit'), + ] + + operations = [ + migrations.AddField( + model_name='device', + name='console_port_count', + field=utilities.fields.CounterCacheField(default=0, to_field='device', to_model='dcim.ConsolePort'), + ), + migrations.AddField( + model_name='device', + name='console_server_port_count', + field=utilities.fields.CounterCacheField(default=0, to_field='device', to_model='dcim.ConsoleServerPort'), + ), + migrations.AddField( + model_name='device', + name='power_port_count', + field=utilities.fields.CounterCacheField(default=0, to_field='device', to_model='dcim.PowerPort'), + ), + migrations.AddField( + model_name='device', + name='power_outlet_count', + field=utilities.fields.CounterCacheField(default=0, to_field='device', to_model='dcim.PowerOutlet'), + ), + migrations.AddField( + model_name='device', + name='interface_count', + field=utilities.fields.CounterCacheField(default=0, to_field='device', to_model='dcim.Interface'), + ), + migrations.AddField( + model_name='device', + name='front_port_count', + field=utilities.fields.CounterCacheField(default=0, to_field='device', to_model='dcim.FrontPort'), + ), + migrations.AddField( + model_name='device', + name='rear_port_count', + field=utilities.fields.CounterCacheField(default=0, to_field='device', to_model='dcim.RearPort'), + ), + migrations.AddField( + model_name='device', + name='device_bay_count', + field=utilities.fields.CounterCacheField(default=0, to_field='device', to_model='dcim.DeviceBay'), + ), + migrations.AddField( + model_name='device', + name='module_bay_count', + field=utilities.fields.CounterCacheField(default=0, to_field='device', to_model='dcim.ModuleBay'), + ), + migrations.AddField( + model_name='device', + name='inventory_item_count', + field=utilities.fields.CounterCacheField(default=0, to_field='device', to_model='dcim.InventoryItem'), + ), + migrations.RunPython( + recalculate_device_counts, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 9f6837b92..62f26776f 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -19,6 +19,7 @@ from utilities.fields import ColorField, NaturalOrderingField from utilities.mptt import TreeManager from utilities.ordering import naturalize_interface from utilities.query_functions import CollateAsChar +from utilities.tracking import TrackingModelMixin from wireless.choices import * from wireless.utils import get_channel_attr @@ -269,7 +270,7 @@ class PathEndpoint(models.Model): # Console components # -class ConsolePort(ModularComponentModel, CabledObjectModel, PathEndpoint): +class ConsolePort(ModularComponentModel, CabledObjectModel, PathEndpoint, TrackingModelMixin): """ A physical console port within a Device. ConsolePorts connect to ConsoleServerPorts. """ @@ -292,7 +293,7 @@ class ConsolePort(ModularComponentModel, CabledObjectModel, PathEndpoint): return reverse('dcim:consoleport', kwargs={'pk': self.pk}) -class ConsoleServerPort(ModularComponentModel, CabledObjectModel, PathEndpoint): +class ConsoleServerPort(ModularComponentModel, CabledObjectModel, PathEndpoint, TrackingModelMixin): """ A physical port within a Device (typically a designated console server) which provides access to ConsolePorts. """ @@ -319,7 +320,7 @@ class ConsoleServerPort(ModularComponentModel, CabledObjectModel, PathEndpoint): # Power components # -class PowerPort(ModularComponentModel, CabledObjectModel, PathEndpoint): +class PowerPort(ModularComponentModel, CabledObjectModel, PathEndpoint, TrackingModelMixin): """ A physical power supply (intake) port within a Device. PowerPorts connect to PowerOutlets. """ @@ -428,7 +429,7 @@ class PowerPort(ModularComponentModel, CabledObjectModel, PathEndpoint): } -class PowerOutlet(ModularComponentModel, CabledObjectModel, PathEndpoint): +class PowerOutlet(ModularComponentModel, CabledObjectModel, PathEndpoint, TrackingModelMixin): """ A physical power outlet (output) within a Device which provides power to a PowerPort. """ @@ -537,7 +538,7 @@ class BaseInterface(models.Model): return self.fhrp_group_assignments.count() -class Interface(ModularComponentModel, BaseInterface, CabledObjectModel, PathEndpoint): +class Interface(ModularComponentModel, BaseInterface, CabledObjectModel, PathEndpoint, TrackingModelMixin): """ A network interface within a Device. A physical Interface can connect to exactly one other Interface. """ @@ -888,7 +889,7 @@ class Interface(ModularComponentModel, BaseInterface, CabledObjectModel, PathEnd # Pass-through ports # -class FrontPort(ModularComponentModel, CabledObjectModel): +class FrontPort(ModularComponentModel, CabledObjectModel, TrackingModelMixin): """ A pass-through port on the front of a Device. """ @@ -949,7 +950,7 @@ class FrontPort(ModularComponentModel, CabledObjectModel): }) -class RearPort(ModularComponentModel, CabledObjectModel): +class RearPort(ModularComponentModel, CabledObjectModel, TrackingModelMixin): """ A pass-through port on the rear of a Device. """ @@ -990,7 +991,7 @@ class RearPort(ModularComponentModel, CabledObjectModel): # Bays # -class ModuleBay(ComponentModel): +class ModuleBay(ComponentModel, TrackingModelMixin): """ An empty space within a Device which can house a child device """ @@ -1006,7 +1007,7 @@ class ModuleBay(ComponentModel): return reverse('dcim:modulebay', kwargs={'pk': self.pk}) -class DeviceBay(ComponentModel): +class DeviceBay(ComponentModel, TrackingModelMixin): """ An empty space within a Device which can house a child device """ @@ -1064,7 +1065,7 @@ class InventoryItemRole(OrganizationalModel): return reverse('dcim:inventoryitemrole', args=[self.pk]) -class InventoryItem(MPTTModel, ComponentModel): +class InventoryItem(MPTTModel, ComponentModel, TrackingModelMixin): """ An InventoryItem represents a serialized piece of hardware within a Device, such as a line card or power supply. InventoryItems are used only for inventory purposes. diff --git a/netbox/dcim/models/devices.py b/netbox/dcim/models/devices.py index ece02105c..48b916a31 100644 --- a/netbox/dcim/models/devices.py +++ b/netbox/dcim/models/devices.py @@ -21,7 +21,7 @@ from extras.querysets import ConfigContextModelQuerySet from netbox.config import ConfigItem from netbox.models import OrganizationalModel, PrimaryModel from utilities.choices import ColorChoices -from utilities.fields import ColorField, NaturalOrderingField +from utilities.fields import ColorField, CounterCacheField, NaturalOrderingField from .device_components import * from .mixins import WeightMixin @@ -639,6 +639,48 @@ class Device(PrimaryModel, ConfigContextModel): help_text=_("GPS coordinate in decimal format (xx.yyyyyy)") ) + # Counter fields + console_port_count = CounterCacheField( + to_model='dcim.ConsolePort', + to_field='device' + ) + console_server_port_count = CounterCacheField( + to_model='dcim.ConsoleServerPort', + to_field='device' + ) + power_port_count = CounterCacheField( + to_model='dcim.PowerPort', + to_field='device' + ) + power_outlet_count = CounterCacheField( + to_model='dcim.PowerOutlet', + to_field='device' + ) + interface_count = CounterCacheField( + to_model='dcim.Interface', + to_field='device' + ) + front_port_count = CounterCacheField( + to_model='dcim.FrontPort', + to_field='device' + ) + rear_port_count = CounterCacheField( + to_model='dcim.RearPort', + to_field='device' + ) + device_bay_count = CounterCacheField( + to_model='dcim.DeviceBay', + to_field='device' + ) + module_bay_count = CounterCacheField( + to_model='dcim.ModuleBay', + to_field='device' + ) + inventory_item_count = CounterCacheField( + to_model='dcim.InventoryItem', + to_field='device' + ) + # Generic relations contacts = GenericRelation( to='tenancy.ContactAssignment' diff --git a/netbox/dcim/tables/devices.py b/netbox/dcim/tables/devices.py index a5862da68..77d53f0ec 100644 --- a/netbox/dcim/tables/devices.py +++ b/netbox/dcim/tables/devices.py @@ -1,10 +1,10 @@ import django_tables2 as tables -from dcim import models from django_tables2.utils import Accessor -from tenancy.tables import ContactsColumnMixin, TenancyColumnsMixin +from django.utils.translation import gettext as _ +from dcim import models from netbox.tables import NetBoxTable, columns - +from tenancy.tables import ContactsColumnMixin, TenancyColumnsMixin from .template_code import * __all__ = ( @@ -230,6 +230,36 @@ class DeviceTable(TenancyColumnsMixin, ContactsColumnMixin, NetBoxTable): tags = columns.TagColumn( url_name='dcim:device_list' ) + console_port_count = tables.Column( + verbose_name=_('Console ports') + ) + console_server_port_count = tables.Column( + verbose_name=_('Console server ports') + ) + power_port_count = tables.Column( + verbose_name=_('Power ports') + ) + power_outlet_count = tables.Column( + verbose_name=_('Power outlets') + ) + interface_count = tables.Column( + verbose_name=_('Interfaces') + ) + front_port_count = tables.Column( + verbose_name=_('Front ports') + ) + rear_port_count = tables.Column( + verbose_name=_('Rear ports') + ) + device_bay_count = tables.Column( + verbose_name=_('Device bays') + ) + module_bay_count = tables.Column( + verbose_name=_('Module bays') + ) + inventory_item_count = tables.Column( + verbose_name=_('Inventory items') + ) class Meta(NetBoxTable.Meta): model = models.Device diff --git a/netbox/dcim/views.py b/netbox/dcim/views.py index 008db382a..d5b24b3b9 100644 --- a/netbox/dcim/views.py +++ b/netbox/dcim/views.py @@ -1876,7 +1876,7 @@ class DeviceConsolePortsView(DeviceComponentsView): template_name = 'dcim/device/consoleports.html', tab = ViewTab( label=_('Console Ports'), - badge=lambda obj: obj.consoleports.count(), + badge=lambda obj: obj.console_port_count, permission='dcim.view_consoleport', weight=550, hide_if_empty=True @@ -1891,7 +1891,7 @@ class DeviceConsoleServerPortsView(DeviceComponentsView): template_name = 'dcim/device/consoleserverports.html' tab = ViewTab( label=_('Console Server Ports'), - badge=lambda obj: obj.consoleserverports.count(), + badge=lambda obj: obj.console_server_port_count, permission='dcim.view_consoleserverport', weight=560, hide_if_empty=True @@ -1906,7 +1906,7 @@ class DevicePowerPortsView(DeviceComponentsView): template_name = 'dcim/device/powerports.html' tab = ViewTab( label=_('Power Ports'), - badge=lambda obj: obj.powerports.count(), + badge=lambda obj: obj.power_port_count, permission='dcim.view_powerport', weight=570, hide_if_empty=True @@ -1921,7 +1921,7 @@ class DevicePowerOutletsView(DeviceComponentsView): template_name = 'dcim/device/poweroutlets.html' tab = ViewTab( label=_('Power Outlets'), - badge=lambda obj: obj.poweroutlets.count(), + badge=lambda obj: obj.power_outlet_count, permission='dcim.view_poweroutlet', weight=580, hide_if_empty=True @@ -1957,7 +1957,7 @@ class DeviceFrontPortsView(DeviceComponentsView): template_name = 'dcim/device/frontports.html' tab = ViewTab( label=_('Front Ports'), - badge=lambda obj: obj.frontports.count(), + badge=lambda obj: obj.front_port_count, permission='dcim.view_frontport', weight=530, hide_if_empty=True @@ -1972,7 +1972,7 @@ class DeviceRearPortsView(DeviceComponentsView): template_name = 'dcim/device/rearports.html' tab = ViewTab( label=_('Rear Ports'), - badge=lambda obj: obj.rearports.count(), + badge=lambda obj: obj.rear_port_count, permission='dcim.view_rearport', weight=540, hide_if_empty=True @@ -1987,7 +1987,7 @@ class DeviceModuleBaysView(DeviceComponentsView): template_name = 'dcim/device/modulebays.html' tab = ViewTab( label=_('Module Bays'), - badge=lambda obj: obj.modulebays.count(), + badge=lambda obj: obj.module_bay_count, permission='dcim.view_modulebay', weight=510, hide_if_empty=True @@ -2002,7 +2002,7 @@ class DeviceDeviceBaysView(DeviceComponentsView): template_name = 'dcim/device/devicebays.html' tab = ViewTab( label=_('Device Bays'), - badge=lambda obj: obj.devicebays.count(), + badge=lambda obj: obj.device_bay_count, permission='dcim.view_devicebay', weight=500, hide_if_empty=True @@ -2017,7 +2017,7 @@ class DeviceInventoryView(DeviceComponentsView): template_name = 'dcim/device/inventory.html' tab = ViewTab( label=_('Inventory Items'), - badge=lambda obj: obj.inventoryitems.count(), + badge=lambda obj: obj.inventory_item_count, permission='dcim.view_inventoryitem', weight=590, hide_if_empty=True diff --git a/netbox/netbox/models/__init__.py b/netbox/netbox/models/__init__.py index 21ca0087b..23dcfb985 100644 --- a/netbox/netbox/models/__init__.py +++ b/netbox/netbox/models/__init__.py @@ -8,6 +8,7 @@ from netbox.models.features import * from utilities.mptt import TreeManager from utilities.querysets import RestrictedQuerySet + __all__ = ( 'ChangeLoggedModel', 'NestedGroupModel', diff --git a/netbox/netbox/registry.py b/netbox/netbox/registry.py index 23b9ad4cb..21a869001 100644 --- a/netbox/netbox/registry.py +++ b/netbox/netbox/registry.py @@ -21,6 +21,7 @@ class Registry(dict): # Initialize the global registry registry = Registry({ + 'counter_fields': collections.defaultdict(dict), 'data_backends': dict(), 'denormalized_fields': collections.defaultdict(list), 'model_features': dict(), diff --git a/netbox/utilities/counters.py b/netbox/utilities/counters.py new file mode 100644 index 000000000..ee6865ca2 --- /dev/null +++ b/netbox/utilities/counters.py @@ -0,0 +1,93 @@ +from django.apps import apps +from django.db.models import F +from django.db.models.signals import post_delete, post_save + +from netbox.registry import registry +from .fields import CounterCacheField + + +def get_counters_for_model(model): + """ + Return field mappings for all counters registered to the given model. + """ + return registry['counter_fields'][model].items() + + +def update_counter(model, pk, counter_name, value): + """ + Increment or decrement a counter field on an object identified by its model and primary key (PK). Positive values + will increment; negative values will decrement. + """ + model.objects.filter(pk=pk).update( + **{counter_name: F(counter_name) + value} + ) + + +# +# Signal handlers +# + +def post_save_receiver(sender, instance, **kwargs): + """ + Update counter fields on related objects when a TrackingModelMixin subclass is created or modified. + """ + for field_name, counter_name in get_counters_for_model(sender): + parent_model = sender._meta.get_field(field_name).related_model + new_pk = getattr(instance, field_name, None) + old_pk = instance.tracker.get(field_name) if field_name in instance.tracker else None + + # Update the counters on the old and/or new parents as needed + if old_pk is not None: + update_counter(parent_model, old_pk, counter_name, -1) + if new_pk is not None: + update_counter(parent_model, new_pk, counter_name, 1) + + +def post_delete_receiver(sender, instance, **kwargs): + """ + Update counter fields on related objects when a TrackingModelMixin subclass is deleted. + """ + for field_name, counter_name in get_counters_for_model(sender): + parent_model = sender._meta.get_field(field_name).related_model + parent_pk = getattr(instance, field_name, None) + + # Decrement the parent's counter by one + if parent_pk is not None: + update_counter(parent_model, parent_pk, counter_name, -1) + + +# +# Registration +# + +def connect_counters(*models): + """ + Register counter fields and connect post_save & post_delete signal handlers for the affected models. + """ + for model in models: + + # Find all CounterCacheFields on the model + counter_fields = [ + field for field in model._meta.get_fields() if type(field) is CounterCacheField + ] + + for field in counter_fields: + to_model = apps.get_model(field.to_model_name) + + # Register the counter in the registry + change_tracking_fields = registry['counter_fields'][to_model] + change_tracking_fields[f"{field.to_field_name}_id"] = field.name + + # Connect the post_save and post_delete handlers + post_save.connect( + post_save_receiver, + sender=to_model, + weak=False, + dispatch_uid=f'{model._meta.label}.{field.name}' + ) + post_delete.connect( + post_delete_receiver, + sender=to_model, + weak=False, + dispatch_uid=f'{model._meta.label}.{field.name}' + ) diff --git a/netbox/utilities/fields.py b/netbox/utilities/fields.py index 8934e4ad6..ca1342df7 100644 --- a/netbox/utilities/fields.py +++ b/netbox/utilities/fields.py @@ -2,6 +2,7 @@ from collections import defaultdict from django.contrib.contenttypes.fields import GenericForeignKey from django.db import models +from django.utils.translation import gettext_lazy as _ from utilities.ordering import naturalize from .forms.widgets import ColorSelect @@ -9,6 +10,7 @@ from .validators import ColorValidator __all__ = ( 'ColorField', + 'CounterCacheField', 'NaturalOrderingField', 'NullableCharField', 'RestrictedGenericForeignKey', @@ -143,3 +145,43 @@ class RestrictedGenericForeignKey(GenericForeignKey): self.name, False, ) + + +class CounterCacheField(models.BigIntegerField): + """ + Counter field to keep track of related model counts. + """ + def __init__(self, to_model, to_field, *args, **kwargs): + if not isinstance(to_model, str): + raise TypeError( + _("%s(%r) is invalid. to_model parameter to CounterCacheField must be " + "a string in the format 'app.model'") + % ( + self.__class__.__name__, + to_model, + ) + ) + + if not isinstance(to_field, str): + raise TypeError( + _("%s(%r) is invalid. to_field parameter to CounterCacheField must be " + "a string in the format 'field'") + % ( + self.__class__.__name__, + to_field, + ) + ) + + self.to_model_name = to_model + self.to_field_name = to_field + + kwargs['default'] = kwargs.get('default', 0) + kwargs['editable'] = False + + super().__init__(*args, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + kwargs["to_model"] = self.to_model_name + kwargs["to_field"] = self.to_field_name + return name, path, args, kwargs diff --git a/netbox/utilities/management/__init__.py b/netbox/utilities/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/netbox/utilities/management/commands/__init__.py b/netbox/utilities/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/netbox/utilities/management/commands/calculate_cached_counts.py b/netbox/utilities/management/commands/calculate_cached_counts.py new file mode 100644 index 000000000..62354797c --- /dev/null +++ b/netbox/utilities/management/commands/calculate_cached_counts.py @@ -0,0 +1,52 @@ +from collections import defaultdict + +from django.core.management.base import BaseCommand +from django.db.models import Count, OuterRef, Subquery + +from netbox.registry import registry + + +class Command(BaseCommand): + help = "Force a recalculation of all cached counter fields" + + @staticmethod + def collect_models(): + """ + Query the registry to find all models which have one or more counter fields. Return a mapping of counter fields + to related query names for each model. + """ + models = defaultdict(dict) + + for model, field_mappings in registry['counter_fields'].items(): + for field_name, counter_name in field_mappings.items(): + fk_field = model._meta.get_field(field_name) # Interface.device + parent_model = fk_field.related_model # Device + related_query_name = fk_field.related_query_name() # 'interfaces' + models[parent_model][counter_name] = related_query_name + + return models + + def update_counts(self, model, field_name, related_query): + """ + Perform a bulk update for the given model and counter field. For example, + + update_counts(Device, '_interface_count', 'interfaces') + + will effectively set + + Device.objects.update(_interface_count=Count('interfaces')) + """ + self.stdout.write(f'Updating {model.__name__} {field_name}...') + subquery = Subquery( + model.objects.filter(pk=OuterRef('pk')).annotate(_count=Count(related_query)).values('_count') + ) + return model.objects.update(**{ + field_name: subquery + }) + + def handle(self, *model_names, **options): + for model, mappings in self.collect_models().items(): + for field_name, related_query in mappings.items(): + self.update_counts(model, field_name, related_query) + + self.stdout.write(self.style.SUCCESS('Finished.')) diff --git a/netbox/utilities/tests/test_counters.py b/netbox/utilities/tests/test_counters.py new file mode 100644 index 000000000..e9561c91b --- /dev/null +++ b/netbox/utilities/tests/test_counters.py @@ -0,0 +1,69 @@ +from django.test import TestCase + +from dcim.models import * +from utilities.testing.utils import create_test_device + + +class CountersTest(TestCase): + """ + Validate the operation of dict_to_filter_params(). + """ + @classmethod + def setUpTestData(cls): + + # Create devices + device1 = create_test_device('Device 1') + device2 = create_test_device('Device 2') + + # Create interfaces + Interface.objects.create(device=device1, name='Interface 1') + Interface.objects.create(device=device1, name='Interface 2') + Interface.objects.create(device=device2, name='Interface 3') + Interface.objects.create(device=device2, name='Interface 4') + + def test_interface_count_creation(self): + """ + When a tracked object (Interface) is added the tracking counter should be updated. + """ + device1, device2 = Device.objects.all() + self.assertEqual(device1.interface_count, 2) + self.assertEqual(device2.interface_count, 2) + + Interface.objects.create(device=device1, name='Interface 5') + Interface.objects.create(device=device2, name='Interface 6') + device1.refresh_from_db() + device2.refresh_from_db() + self.assertEqual(device1.interface_count, 3) + self.assertEqual(device2.interface_count, 3) + + def test_interface_count_deletion(self): + """ + When a tracked object (Interface) is deleted the tracking counter should be updated. + """ + device1, device2 = Device.objects.all() + self.assertEqual(device1.interface_count, 2) + self.assertEqual(device2.interface_count, 2) + + Interface.objects.get(name='Interface 1').delete() + Interface.objects.get(name='Interface 3').delete() + device1.refresh_from_db() + device2.refresh_from_db() + self.assertEqual(device1.interface_count, 1) + self.assertEqual(device2.interface_count, 1) + + def test_interface_count_move(self): + """ + When a tracked object (Interface) is moved the tracking counter should be updated. + """ + device1, device2 = Device.objects.all() + self.assertEqual(device1.interface_count, 2) + self.assertEqual(device2.interface_count, 2) + + interface1 = Interface.objects.get(name='Interface 1') + interface1.device = device2 + interface1.save() + + device1.refresh_from_db() + device2.refresh_from_db() + self.assertEqual(device1.interface_count, 1) + self.assertEqual(device2.interface_count, 3) diff --git a/netbox/utilities/tracking.py b/netbox/utilities/tracking.py new file mode 100644 index 000000000..88945615b --- /dev/null +++ b/netbox/utilities/tracking.py @@ -0,0 +1,78 @@ +from django.db.models.query_utils import DeferredAttribute + +from netbox.registry import registry + + +class Tracker: + """ + An ephemeral instance employed to record which tracked fields on an instance have been modified. + """ + def __init__(self): + self._changed_fields = {} + + def __contains__(self, item): + return item in self._changed_fields + + def set(self, name, value): + """ + Mark an attribute as having been changed and record its original value. + """ + self._changed_fields[name] = value + + def get(self, name): + """ + Return the original value of a changed field. Raises KeyError if name is not found. + """ + return self._changed_fields[name] + + def clear(self, *names): + """ + Clear any fields that were recorded as having been changed. + """ + for name in names: + self._changed_fields.pop(name, None) + else: + self._changed_fields = {} + + +class TrackingModelMixin: + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Mark the instance as initialized, to enable our custom __setattr__() + self._initialized = True + + @property + def tracker(self): + """ + Return the Tracker instance for this instance, first creating it if necessary. + """ + if not hasattr(self._state, "_tracker"): + self._state._tracker = Tracker() + return self._state._tracker + + def save(self, *args, **kwargs): + super().save(*args, **kwargs) + + # Clear any tracked fields now that changes have been saved + update_fields = kwargs.get('update_fields', []) + self.tracker.clear(*update_fields) + + def __setattr__(self, name, value): + if hasattr(self, "_initialized"): + # Record any changes to a tracked field + if name in registry['counter_fields'][self.__class__]: + if name not in self.tracker: + # The attribute has been created or changed + if name in self.__dict__: + old_value = getattr(self, name) + if value != old_value: + self.tracker.set(name, old_value) + else: + self.tracker.set(name, DeferredAttribute) + elif value == self.tracker.get(name): + # A previously changed attribute has been restored + self.tracker.clear(name) + + super().__setattr__(name, value) diff --git a/netbox/virtualization/api/serializers.py b/netbox/virtualization/api/serializers.py index f72215b98..693bb362f 100644 --- a/netbox/virtualization/api/serializers.py +++ b/netbox/virtualization/api/serializers.py @@ -80,12 +80,15 @@ class VirtualMachineSerializer(NetBoxModelSerializer): primary_ip4 = NestedIPAddressSerializer(required=False, allow_null=True) primary_ip6 = NestedIPAddressSerializer(required=False, allow_null=True) + # Counter fields + interface_count = serializers.IntegerField(read_only=True) + class Meta: model = VirtualMachine fields = [ 'id', 'url', 'display', 'name', 'status', 'site', 'cluster', 'device', 'role', 'tenant', 'platform', 'primary_ip', 'primary_ip4', 'primary_ip6', 'vcpus', 'memory', 'disk', 'description', 'comments', - 'local_context_data', 'tags', 'custom_fields', 'created', 'last_updated', + 'local_context_data', 'tags', 'custom_fields', 'created', 'last_updated', 'interface_count', ] validators = [] @@ -98,6 +101,7 @@ class VirtualMachineWithConfigContextSerializer(VirtualMachineSerializer): 'id', 'url', 'display', 'name', 'status', 'site', 'cluster', 'device', 'role', 'tenant', 'platform', 'primary_ip', 'primary_ip4', 'primary_ip6', 'vcpus', 'memory', 'disk', 'description', 'comments', 'local_context_data', 'tags', 'custom_fields', 'config_context', 'created', 'last_updated', + 'interface_count', ] @extend_schema_field(serializers.JSONField(allow_null=True)) diff --git a/netbox/virtualization/apps.py b/netbox/virtualization/apps.py index 1b6b110df..8db943ea1 100644 --- a/netbox/virtualization/apps.py +++ b/netbox/virtualization/apps.py @@ -6,3 +6,8 @@ class VirtualizationConfig(AppConfig): def ready(self): from . import search + from .models import VirtualMachine + from utilities.counters import connect_counters + + # Register counters + connect_counters(VirtualMachine) diff --git a/netbox/virtualization/migrations/0035_virtualmachine_interface_count.py b/netbox/virtualization/migrations/0035_virtualmachine_interface_count.py new file mode 100644 index 000000000..5f52d32e0 --- /dev/null +++ b/netbox/virtualization/migrations/0035_virtualmachine_interface_count.py @@ -0,0 +1,35 @@ +from django.db import migrations +from django.db.models import Count + +import utilities.fields + + +def populate_virtualmachine_counts(apps, schema_editor): + VirtualMachine = apps.get_model('virtualization', 'VirtualMachine') + + vms = list(VirtualMachine.objects.annotate(_interface_count=Count('interfaces', distinct=True))) + + for vm in vms: + vm.interface_count = vm._interface_count + + VirtualMachine.objects.bulk_update(vms, ['interface_count']) + + +class Migration(migrations.Migration): + dependencies = [ + ('virtualization', '0034_standardize_description_comments'), + ] + + operations = [ + migrations.AddField( + model_name='virtualmachine', + name='interface_count', + field=utilities.fields.CounterCacheField( + default=0, to_field='virtual_machine', to_model='virtualization.VMInterface' + ), + ), + migrations.RunPython( + code=populate_virtualmachine_counts, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/netbox/virtualization/models/virtualmachines.py b/netbox/virtualization/models/virtualmachines.py index 6e9cc5664..dbbfe49ed 100644 --- a/netbox/virtualization/models/virtualmachines.py +++ b/netbox/virtualization/models/virtualmachines.py @@ -11,9 +11,10 @@ from extras.models import ConfigContextModel from extras.querysets import ConfigContextModelQuerySet from netbox.config import get_config from netbox.models import NetBoxModel, PrimaryModel -from utilities.fields import NaturalOrderingField +from utilities.fields import CounterCacheField, NaturalOrderingField from utilities.ordering import naturalize_interface from utilities.query_functions import CollateAsChar +from utilities.tracking import TrackingModelMixin from virtualization.choices import * __all__ = ( @@ -120,6 +121,12 @@ class VirtualMachine(PrimaryModel, ConfigContextModel): verbose_name='Disk (GB)' ) + # Counter fields + interface_count = CounterCacheField( + to_model='virtualization.VMInterface', + to_field='virtual_machine' + ) + # Generic relation contacts = GenericRelation( to='tenancy.ContactAssignment' @@ -222,7 +229,7 @@ class VirtualMachine(PrimaryModel, ConfigContextModel): return None -class VMInterface(NetBoxModel, BaseInterface): +class VMInterface(NetBoxModel, BaseInterface, TrackingModelMixin): virtual_machine = models.ForeignKey( to='virtualization.VirtualMachine', on_delete=models.CASCADE, diff --git a/netbox/virtualization/tables/virtualmachines.py b/netbox/virtualization/tables/virtualmachines.py index b1d44ad02..03e3a1af6 100644 --- a/netbox/virtualization/tables/virtualmachines.py +++ b/netbox/virtualization/tables/virtualmachines.py @@ -1,10 +1,11 @@ import django_tables2 as tables +from django.utils.translation import gettext as _ + from dcim.tables.devices import BaseInterfaceTable +from netbox.tables import NetBoxTable, columns from tenancy.tables import ContactsColumnMixin, TenancyColumnsMixin from virtualization.models import VirtualMachine, VMInterface -from netbox.tables import NetBoxTable, columns - __all__ = ( 'VirtualMachineTable', 'VirtualMachineVMInterfaceTable', @@ -70,6 +71,9 @@ class VirtualMachineTable(TenancyColumnsMixin, ContactsColumnMixin, NetBoxTable) tags = columns.TagColumn( url_name='virtualization:virtualmachine_list' ) + interface_count = tables.Column( + verbose_name=_('Interfaces') + ) class Meta(NetBoxTable.Meta): model = VirtualMachine diff --git a/netbox/virtualization/views.py b/netbox/virtualization/views.py index 75e83f9e1..c56a8ade2 100644 --- a/netbox/virtualization/views.py +++ b/netbox/virtualization/views.py @@ -349,7 +349,7 @@ class VirtualMachineInterfacesView(generic.ObjectChildrenView): template_name = 'virtualization/virtualmachine/interfaces.html' tab = ViewTab( label=_('Interfaces'), - badge=lambda obj: obj.interfaces.count(), + badge=lambda obj: obj.interface_count, permission='virtualization.view_vminterface', weight=500 ) From 7600d7b3449e4327713c1f550aa8a1ed5aab9ec0 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Wed, 26 Jul 2023 00:43:40 +0700 Subject: [PATCH 2/2] Closes #13228: Move token management views to primary UI --- netbox/netbox/navigation/menu.py | 3 +- netbox/netbox/settings.py | 1 + netbox/templates/inc/profile_button.html | 2 +- netbox/templates/users/account/api_token.html | 58 -------- netbox/templates/users/account/base.html | 2 +- netbox/templates/users/account/token.html | 69 ++++++++++ .../{api_tokens.html => token_list.html} | 4 +- netbox/templates/users/token.html | 56 ++++++++ netbox/users/admin/__init__.py | 21 --- netbox/users/admin/forms.py | 21 --- netbox/users/filtersets.py | 1 + netbox/users/forms/bulk_edit.py | 43 +++++- netbox/users/forms/bulk_import.py | 18 ++- netbox/users/forms/filtersets.py | 35 +++++ netbox/users/forms/model_forms.py | 28 +++- netbox/users/migrations/0005_usertoken.py | 25 ++++ netbox/users/models.py | 24 +++- netbox/users/tables.py | 45 +++++-- netbox/users/tests/test_views.py | 52 ++++++- netbox/users/urls.py | 14 +- netbox/users/views.py | 127 ++++++++++++------ 21 files changed, 482 insertions(+), 167 deletions(-) delete mode 100644 netbox/templates/users/account/api_token.html create mode 100644 netbox/templates/users/account/token.html rename netbox/templates/users/account/{api_tokens.html => token_list.html} (82%) create mode 100644 netbox/templates/users/token.html delete mode 100644 netbox/users/admin/forms.py create mode 100644 netbox/users/migrations/0005_usertoken.py diff --git a/netbox/netbox/navigation/menu.py b/netbox/netbox/navigation/menu.py index 45de28f2b..7e5d26186 100644 --- a/netbox/netbox/navigation/menu.py +++ b/netbox/netbox/navigation/menu.py @@ -353,7 +353,7 @@ ADMIN_MENU = Menu( icon_class='mdi mdi-account-multiple', groups=( MenuGroup( - label=_('Users'), + label=_('Authentication'), items=( # Proxy model for auth.User MenuItem( @@ -399,6 +399,7 @@ ADMIN_MENU = Menu( ) ) ), + get_model_item('users', 'token', _('API Tokens')), get_model_item('users', 'objectpermission', _('Permissions'), actions=['add']), ), ), diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 7d2da2996..da58b0dd6 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -469,6 +469,7 @@ EXEMPT_EXCLUDE_MODELS = ( ('auth', 'group'), ('auth', 'user'), ('users', 'objectpermission'), + ('users', 'token'), ) # All URLs starting with a string listed here are exempt from login enforcement diff --git a/netbox/templates/inc/profile_button.html b/netbox/templates/inc/profile_button.html index 932b91275..a5d8cef61 100644 --- a/netbox/templates/inc/profile_button.html +++ b/netbox/templates/inc/profile_button.html @@ -34,7 +34,7 @@
  • - + API Tokens
    • diff --git a/netbox/templates/users/account/api_token.html b/netbox/templates/users/account/api_token.html deleted file mode 100644 index 7fd6f064d..000000000 --- a/netbox/templates/users/account/api_token.html +++ /dev/null @@ -1,58 +0,0 @@ -{% extends 'generic/object.html' %} -{% load form_helpers %} -{% load helpers %} -{% load plugins %} - -{% block content %} -
    -
    - {% if not settings.ALLOW_TOKEN_RETRIEVAL %} - - {% endif %} -
    -
    Token
    -
    - - - - - - - - - - - - - - - - - - - - - -
    Key -
    - {% copy_content "token_id" %} -
    -
    {{ key }}
    -
    Description{{ object.description|placeholder }}
    User{{ object.user }}
    Created{{ object.created|annotated_date }}
    Expires - {% if object.expires %} - {{ object.expires|annotated_date }} - {% else %} - Never - {% endif %} -
    -
    -
    -
    - Add Another - Cancel -
    -
    -
    -{% endblock %} diff --git a/netbox/templates/users/account/base.html b/netbox/templates/users/account/base.html index f492f89ec..9ac61bced 100644 --- a/netbox/templates/users/account/base.html +++ b/netbox/templates/users/account/base.html @@ -18,7 +18,7 @@ {% endif %} {% endblock %} diff --git a/netbox/templates/users/account/token.html b/netbox/templates/users/account/token.html new file mode 100644 index 000000000..6df1d1367 --- /dev/null +++ b/netbox/templates/users/account/token.html @@ -0,0 +1,69 @@ +{% extends 'generic/object.html' %} +{% load form_helpers %} +{% load helpers %} +{% load i18n %} +{% load plugins %} + +{% block breadcrumbs %} +
  • {% trans "My API Tokens" %}
    • +{% endblock breadcrumbs %} + +{% block title %}{% trans "Token" %} {{ object }}{% endblock %} + +{% block subtitle %}{% endblock %} + +{% block content %} +
    +
    + {% if key and not settings.ALLOW_TOKEN_RETRIEVAL %} + + {% endif %} +
    +
    {% trans "Token" %}
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    {% trans "Key" %} + {% if key %} +
    + {% copy_content "token_id" %} +
    +
    {{ key }}
    + {% else %} + {{ object.partial }} + {% endif %} +
    {% trans "Description" %}{{ object.description|placeholder }}
    {% trans "Write enabled" %}{% checkmark object.write_enabled %}
    {% trans "Created" %}{{ object.created|annotated_date }}
    {% trans "Expires" %}{{ object.expires|placeholder }}
    {% trans "Last used" %}{{ object.last_used|placeholder }}
    {% trans "Allowed IPs" %}{{ object.allowed_ips|join:", "|placeholder }}
    +
    +
    +
    +
    +{% endblock %} diff --git a/netbox/templates/users/account/api_tokens.html b/netbox/templates/users/account/token_list.html similarity index 82% rename from netbox/templates/users/account/api_tokens.html rename to netbox/templates/users/account/token_list.html index 25f5f02e6..e30b1ae96 100644 --- a/netbox/templates/users/account/api_tokens.html +++ b/netbox/templates/users/account/token_list.html @@ -2,12 +2,12 @@ {% load helpers %} {% load render_table from django_tables2 %} -{% block title %}API Tokens{% endblock %} +{% block title %}My API Tokens{% endblock %} {% block content %}
    - + Add a Token
    diff --git a/netbox/templates/users/token.html b/netbox/templates/users/token.html new file mode 100644 index 000000000..0fa8c572e --- /dev/null +++ b/netbox/templates/users/token.html @@ -0,0 +1,56 @@ +{% extends 'generic/object.html' %} +{% load i18n %} +{% load helpers %} +{% load render_table from django_tables2 %} + +{% block title %}{% trans "Token" %} {{ object }}{% endblock %} + +{% block subtitle %}{% endblock %} + +{% block content %} +
    +
    +
    +
    {% trans "Token" %}
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    {% trans "Key" %}{% if settings.ALLOW_TOKEN_RETRIEVAL %}{{ object.key }}{% else %}{{ object.partial }}{% endif %}
    {% trans "User" %} + {{ object.user }} +
    {% trans "Description" %}{{ object.description|placeholder }}
    {% trans "Write enabled" %}{% checkmark object.write_enabled %}
    {% trans "Created" %}{{ object.created|annotated_date }}
    {% trans "Expires" %}{{ object.expires|placeholder }}
    {% trans "Last used" %}{{ object.last_used|placeholder }}
    {% trans "Allowed IPs" %}{{ object.allowed_ips|join:", "|placeholder }}
    +
    +
    +
    +
    +{% endblock %} diff --git a/netbox/users/admin/__init__.py b/netbox/users/admin/__init__.py index 316346c50..bc7bf7ab2 100644 --- a/netbox/users/admin/__init__.py +++ b/netbox/users/admin/__init__.py @@ -1,11 +1,6 @@ from django.contrib import admin -from django.contrib.auth.admin import UserAdmin as UserAdmin_ from django.contrib.auth.models import Group, User -from users.models import ObjectPermission, Token -from . import filters, forms, inlines - - # # Users & groups # @@ -13,19 +8,3 @@ from . import filters, forms, inlines # Unregister the built-in GroupAdmin and UserAdmin classes so that we can use our custom admin classes below admin.site.unregister(Group) admin.site.unregister(User) - - -# -# REST API tokens -# - -@admin.register(Token) -class TokenAdmin(admin.ModelAdmin): - form = forms.TokenAdminForm - list_display = [ - 'key', 'user', 'created', 'expires', 'last_used', 'write_enabled', 'description', 'list_allowed_ips' - ] - - def list_allowed_ips(self, obj): - return obj.allowed_ips or 'Any' - list_allowed_ips.short_description = "Allowed IPs" diff --git a/netbox/users/admin/forms.py b/netbox/users/admin/forms.py deleted file mode 100644 index 7db6a124c..000000000 --- a/netbox/users/admin/forms.py +++ /dev/null @@ -1,21 +0,0 @@ -from django import forms -from django.utils.translation import gettext as _ - -from users.models import Token - -__all__ = ( - 'TokenAdminForm', -) - - -class TokenAdminForm(forms.ModelForm): - key = forms.CharField( - required=False, - help_text=_("If no key is provided, one will be generated automatically.") - ) - - class Meta: - fields = [ - 'user', 'key', 'write_enabled', 'expires', 'description', 'allowed_ips' - ] - model = Token diff --git a/netbox/users/filtersets.py b/netbox/users/filtersets.py index a4e9a9fbc..0f590e012 100644 --- a/netbox/users/filtersets.py +++ b/netbox/users/filtersets.py @@ -10,6 +10,7 @@ from users.models import ObjectPermission, Token __all__ = ( 'GroupFilterSet', 'ObjectPermissionFilterSet', + 'TokenFilterSet', 'UserFilterSet', ) diff --git a/netbox/users/forms/bulk_edit.py b/netbox/users/forms/bulk_edit.py index db40283ba..0e29109a4 100644 --- a/netbox/users/forms/bulk_edit.py +++ b/netbox/users/forms/bulk_edit.py @@ -1,13 +1,17 @@ from django import forms +from django.contrib.postgres.forms import SimpleArrayField from django.utils.translation import gettext_lazy as _ +from ipam.formfields import IPNetworkFormField +from ipam.validators import prefix_validator from users.models import * -from utilities.forms import BootstrapMixin -from utilities.forms.widgets import BulkEditNullBooleanSelect +from utilities.forms import BootstrapMixin, BulkEditForm +from utilities.forms.widgets import BulkEditNullBooleanSelect, DateTimePicker __all__ = ( 'ObjectPermissionBulkEditForm', 'UserBulkEditForm', + 'TokenBulkEditForm', ) @@ -70,3 +74,38 @@ class ObjectPermissionBulkEditForm(BootstrapMixin, forms.Form): (None, ('enabled', 'description')), ) nullable_fields = ('description',) + + +class TokenBulkEditForm(BulkEditForm): + pk = forms.ModelMultipleChoiceField( + queryset=Token.objects.all(), + widget=forms.MultipleHiddenInput + ) + write_enabled = forms.NullBooleanField( + required=False, + widget=BulkEditNullBooleanSelect, + label=_('Write enabled') + ) + description = forms.CharField( + max_length=200, + required=False, + label=_('Description') + ) + expires = forms.DateTimeField( + required=False, + widget=DateTimePicker(), + label=_('Expires') + ) + allowed_ips = SimpleArrayField( + base_field=IPNetworkFormField(validators=[prefix_validator]), + required=False, + label=_('Allowed IPs') + ) + + model = Token + fieldsets = ( + (None, ('write_enabled', 'description', 'expires', 'allowed_ips')), + ) + nullable_fields = ( + 'expires', 'description', 'allowed_ips', + ) diff --git a/netbox/users/forms/bulk_import.py b/netbox/users/forms/bulk_import.py index 25f779044..d1f03ff3c 100644 --- a/netbox/users/forms/bulk_import.py +++ b/netbox/users/forms/bulk_import.py @@ -1,9 +1,13 @@ -from users.models import NetBoxGroup, NetBoxUser +from django import forms +from django.utils.translation import gettext as _ +from users.models import * from utilities.forms import CSVModelForm + __all__ = ( 'GroupImportForm', 'UserImportForm', + 'TokenImportForm', ) @@ -30,3 +34,15 @@ class UserImportForm(CSVModelForm): self.instance.set_password(self.cleaned_data.get('password')) return super().save(*args, **kwargs) + + +class TokenImportForm(CSVModelForm): + key = forms.CharField( + label=_('Key'), + required=False, + help_text=_("If no key is provided, one will be generated automatically.") + ) + + class Meta: + model = Token + fields = ('user', 'key', 'write_enabled', 'expires', 'description',) diff --git a/netbox/users/forms/filtersets.py b/netbox/users/forms/filtersets.py index eca76dea4..ff56cbc4c 100644 --- a/netbox/users/forms/filtersets.py +++ b/netbox/users/forms/filtersets.py @@ -1,4 +1,7 @@ from django import forms +from extras.forms.mixins import SavedFiltersMixin +from utilities.forms import FilterForm +from users.models import Token from django.contrib.auth import get_user_model from django.contrib.auth.models import Group from django.utils.translation import gettext_lazy as _ @@ -7,11 +10,13 @@ from netbox.forms import NetBoxModelFilterSetForm from users.models import NetBoxGroup, NetBoxUser, ObjectPermission from utilities.forms import BOOLEAN_WITH_BLANK_CHOICES from utilities.forms.fields import DynamicModelMultipleChoiceField +from utilities.forms.widgets import DateTimePicker __all__ = ( 'GroupFilterForm', 'ObjectPermissionFilterForm', 'UserFilterForm', + 'TokenFilterForm', ) @@ -109,3 +114,33 @@ class ObjectPermissionFilterForm(NetBoxModelFilterSetForm): ), label=_('Can Delete'), ) + + +class TokenFilterForm(SavedFiltersMixin, FilterForm): + model = Token + fieldsets = ( + (None, ('q', 'filter_id',)), + (_('Token'), ('user_id', 'write_enabled', 'expires', 'last_used')), + ) + user_id = DynamicModelMultipleChoiceField( + queryset=get_user_model().objects.all(), + required=False, + label=_('User') + ) + write_enabled = forms.NullBooleanField( + required=False, + widget=forms.Select( + choices=BOOLEAN_WITH_BLANK_CHOICES + ), + label=_('Write Enabled'), + ) + expires = forms.DateTimeField( + required=False, + label=_('Expires'), + widget=DateTimePicker() + ) + last_used = forms.DateTimeField( + required=False, + label=_('Last Used'), + widget=DateTimePicker() + ) diff --git a/netbox/users/forms/model_forms.py b/netbox/users/forms/model_forms.py index 43b95893a..6ca050110 100644 --- a/netbox/users/forms/model_forms.py +++ b/netbox/users/forms/model_forms.py @@ -20,11 +20,13 @@ from utilities.permissions import qs_filter_from_constraints from utilities.utils import flatten_dict __all__ = ( + 'UserTokenForm', 'GroupForm', 'ObjectPermissionForm', 'TokenForm', 'UserConfigForm', 'UserForm', + 'TokenForm', ) @@ -107,7 +109,7 @@ class UserConfigForm(BootstrapMixin, forms.ModelForm, metaclass=UserConfigFormMe ] -class TokenForm(BootstrapMixin, forms.ModelForm): +class UserTokenForm(BootstrapMixin, forms.ModelForm): key = forms.CharField( label=_('Key'), required=False, @@ -117,8 +119,10 @@ class TokenForm(BootstrapMixin, forms.ModelForm): base_field=IPNetworkFormField(validators=[prefix_validator]), required=False, label=_('Allowed IPs'), - help_text=_('Allowed IPv4/IPv6 networks from where the token can be used. Leave blank for no restrictions. ' - 'Example: 10.1.1.0/24,192.168.10.16/32,2001:db8:1::/64'), + help_text=_( + 'Allowed IPv4/IPv6 networks from where the token can be used. Leave blank for no restrictions. ' + 'Example: 10.1.1.0/24,192.168.10.16/32,2001:db8:1::/64' + ), ) class Meta: @@ -138,6 +142,24 @@ class TokenForm(BootstrapMixin, forms.ModelForm): del self.fields['key'] +class TokenForm(UserTokenForm): + user = forms.ModelChoiceField( + queryset=get_user_model().objects.order_by( + 'username' + ), + required=False + ) + + class Meta: + model = Token + fields = [ + 'user', 'key', 'write_enabled', 'expires', 'description', 'allowed_ips', + ] + widgets = { + 'expires': DateTimePicker(), + } + + class UserForm(BootstrapMixin, forms.ModelForm): password = forms.CharField( label=_('Password'), diff --git a/netbox/users/migrations/0005_usertoken.py b/netbox/users/migrations/0005_usertoken.py new file mode 100644 index 000000000..c6aef0590 --- /dev/null +++ b/netbox/users/migrations/0005_usertoken.py @@ -0,0 +1,25 @@ +# Generated by Django 4.1.10 on 2023-07-25 15:19 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('users', '0004_netboxgroup_netboxuser'), + ] + + operations = [ + migrations.CreateModel( + name='UserToken', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + 'verbose_name': 'token', + }, + bases=('users.token',), + ), + ] diff --git a/netbox/users/models.py b/netbox/users/models.py index a8060dd63..71434d5ce 100644 --- a/netbox/users/models.py +++ b/netbox/users/models.py @@ -26,6 +26,7 @@ __all__ = ( 'ObjectPermission', 'Token', 'UserConfig', + 'UserToken', ) @@ -273,13 +274,20 @@ class Token(models.Model): blank=True, null=True, verbose_name='Allowed IPs', - help_text=_('Allowed IPv4/IPv6 networks from where the token can be used. Leave blank for no restrictions. ' - 'Ex: "10.1.1.0/24, 192.168.10.16/32, 2001:DB8:1::/64"'), + help_text=_( + 'Allowed IPv4/IPv6 networks from where the token can be used. Leave blank for no restrictions. ' + 'Ex: "10.1.1.0/24, 192.168.10.16/32, 2001:DB8:1::/64"' + ), ) + objects = RestrictedQuerySet.as_manager() + def __str__(self): return self.key if settings.ALLOW_TOKEN_RETRIEVAL else self.partial + def get_absolute_url(self): + return reverse('users:token', args=[self.pk]) + @property def partial(self): return f'**********************************{self.key[-6:]}' if self.key else '' @@ -314,6 +322,18 @@ class Token(models.Model): return False +class UserToken(Token): + """ + Proxy model for users to manage their own API tokens. + """ + class Meta: + proxy = True + verbose_name = 'token' + + def get_absolute_url(self): + return reverse('users:usertoken', args=[self.pk]) + + # # Permissions # diff --git a/netbox/users/tables.py b/netbox/users/tables.py index 741a4b024..3ef885399 100644 --- a/netbox/users/tables.py +++ b/netbox/users/tables.py @@ -1,8 +1,8 @@ import django_tables2 as tables +from django.utils.translation import gettext as _ from netbox.tables import NetBoxTable, columns -from users.models import NetBoxGroup, NetBoxUser, ObjectPermission -from .models import Token +from users.models import NetBoxGroup, NetBoxUser, ObjectPermission, Token, UserToken __all__ = ( 'GroupTable', @@ -31,17 +31,28 @@ class TokenActionsColumn(columns.ActionsColumn): } -class TokenTable(NetBoxTable): +class UserTokenTable(NetBoxTable): + """ + Table for users to manager their own API tokens under account views. + """ key = columns.TemplateColumn( - template_code=TOKEN + verbose_name=_('Key'), + template_code=TOKEN, ) write_enabled = columns.BooleanColumn( - verbose_name='Write' + verbose_name=_('Write Enabled') + ) + created = columns.DateColumn( + verbose_name=_('Created'), + ) + expires = columns.DateColumn( + verbose_name=_('Expires'), + ) + last_used = columns.DateTimeColumn( + verbose_name=_('Last Used'), ) - created = columns.DateColumn() - expired = columns.DateColumn() - last_used = columns.DateTimeColumn() allowed_ips = columns.TemplateColumn( + verbose_name=_('Allowed IPs'), template_code=ALLOWED_IPS ) actions = TokenActionsColumn( @@ -49,10 +60,26 @@ class TokenTable(NetBoxTable): extra_buttons=COPY_BUTTON ) + class Meta(NetBoxTable.Meta): + model = UserToken + fields = ( + 'pk', 'id', 'key', 'description', 'write_enabled', 'created', 'expires', 'last_used', 'allowed_ips', + ) + + +class TokenTable(UserTokenTable): + """ + General-purpose table for API token management. + """ + user = tables.Column( + linkify=True, + verbose_name=_('User') + ) + class Meta(NetBoxTable.Meta): model = Token fields = ( - 'pk', 'description', 'key', 'write_enabled', 'created', 'expires', 'last_used', 'allowed_ips', + 'pk', 'id', 'key', 'user', 'description', 'write_enabled', 'created', 'expires', 'last_used', 'allowed_ips', ) diff --git a/netbox/users/tests/test_views.py b/netbox/users/tests/test_views.py index ca62f474e..2997052eb 100644 --- a/netbox/users/tests/test_views.py +++ b/netbox/users/tests/test_views.py @@ -2,7 +2,7 @@ from django.contrib.auth.models import Group from django.contrib.contenttypes.models import ContentType from users.models import * -from utilities.testing import ViewTestCases +from utilities.testing import ViewTestCases, create_test_user class UserTestCase( @@ -149,3 +149,53 @@ class ObjectPermissionTestCase( cls.bulk_edit_data = { 'description': 'New description', } + + +class TokenTestCase( + ViewTestCases.GetObjectViewTestCase, + ViewTestCases.CreateObjectViewTestCase, + ViewTestCases.EditObjectViewTestCase, + ViewTestCases.DeleteObjectViewTestCase, + ViewTestCases.ListObjectsViewTestCase, + ViewTestCases.BulkImportObjectsViewTestCase, + ViewTestCases.BulkEditObjectsViewTestCase, + ViewTestCases.BulkDeleteObjectsViewTestCase, +): + model = Token + maxDiff = None + + @classmethod + def setUpTestData(cls): + users = ( + create_test_user('User 1'), + create_test_user('User 2'), + ) + tokens = ( + Token(key='123456790123456789012345678901234567890A', user=users[0]), + Token(key='123456790123456789012345678901234567890B', user=users[0]), + Token(key='123456790123456789012345678901234567890C', user=users[1]), + ) + Token.objects.bulk_create(tokens) + + cls.form_data = { + 'user': users[0].pk, + 'description': 'testdescription', + } + + cls.csv_data = ( + "key,user,description", + f"123456790123456789012345678901234567890D,{users[0].pk},testdescriptionD", + f"123456790123456789012345678901234567890E,{users[1].pk},testdescriptionE", + f"123456790123456789012345678901234567890F,{users[1].pk},testdescriptionF", + ) + + cls.csv_update_data = ( + "id,description", + f"{tokens[0].pk},testdescriptionH", + f"{tokens[1].pk},testdescriptionI", + f"{tokens[2].pk},testdescriptionJ", + ) + + cls.bulk_edit_data = { + 'description': 'newdescription', + } diff --git a/netbox/users/urls.py b/netbox/users/urls.py index ca331d144..ed3db4661 100644 --- a/netbox/users/urls.py +++ b/netbox/users/urls.py @@ -11,9 +11,17 @@ urlpatterns = [ path('bookmarks/', views.BookmarkListView.as_view(), name='bookmarks'), path('preferences/', views.UserConfigView.as_view(), name='preferences'), path('password/', views.ChangePasswordView.as_view(), name='change_password'), - path('api-tokens/', views.TokenListView.as_view(), name='token_list'), - path('api-tokens/add/', views.TokenEditView.as_view(), name='token_add'), - path('api-tokens//', include(get_model_urls('users', 'token'))), + path('api-tokens/', views.UserTokenListView.as_view(), name='usertoken_list'), + path('api-tokens/add/', views.UserTokenEditView.as_view(), name='usertoken_add'), + path('api-tokens//', include(get_model_urls('users', 'usertoken'))), + + # Tokens + path('tokens/', views.TokenListView.as_view(), name='token_list'), + path('tokens/add/', views.TokenEditView.as_view(), name='token_add'), + path('tokens/import/', views.TokenBulkImportView.as_view(), name='token_import'), + path('tokens/edit/', views.TokenBulkEditView.as_view(), name='token_bulk_edit'), + path('tokens/delete/', views.TokenBulkDeleteView.as_view(), name='token_bulk_delete'), + path('tokens//', include(get_model_urls('users', 'token'))), # Users path('users/', views.UserListView.as_view(), name='netboxuser_list'), diff --git a/netbox/users/views.py b/netbox/users/views.py index 99635b514..bc5cf1eeb 100644 --- a/netbox/users/views.py +++ b/netbox/users/views.py @@ -24,7 +24,7 @@ from netbox.views import generic from utilities.forms import ConfirmationForm from utilities.views import register_model_view from . import filtersets, forms, tables -from .models import Token, UserConfig, NetBoxGroup, NetBoxUser, ObjectPermission +from .models import NetBoxGroup, NetBoxUser, ObjectPermission, Token, UserConfig, UserToken # @@ -249,53 +249,61 @@ class BookmarkListView(LoginRequiredMixin, generic.ObjectListView): # -# API tokens +# User views for token management # -class TokenListView(LoginRequiredMixin, View): +class UserTokenListView(LoginRequiredMixin, View): def get(self, request): - - tokens = Token.objects.filter(user=request.user) - table = tables.TokenTable(tokens) + tokens = UserToken.objects.filter(user=request.user) + table = tables.UserTokenTable(tokens) table.configure(request) - return render(request, 'users/account/api_tokens.html', { + return render(request, 'users/account/token_list.html', { 'tokens': tokens, 'active_tab': 'api-tokens', 'table': table, }) -@register_model_view(Token, 'edit') -class TokenEditView(LoginRequiredMixin, View): +@register_model_view(UserToken) +class UserTokenView(LoginRequiredMixin, View): + + def get(self, request, pk): + token = get_object_or_404(UserToken.objects.filter(user=request.user), pk=pk) + key = token.key if settings.ALLOW_TOKEN_RETRIEVAL else None + + return render(request, 'users/account/token.html', { + 'object': token, + 'key': key, + }) + + +@register_model_view(UserToken, 'edit') +class UserTokenEditView(LoginRequiredMixin, View): def get(self, request, pk=None): - if pk: - token = get_object_or_404(Token.objects.filter(user=request.user), pk=pk) + token = get_object_or_404(UserToken.objects.filter(user=request.user), pk=pk) else: - token = Token(user=request.user) - - form = forms.TokenForm(instance=token) + token = UserToken(user=request.user) + form = forms.UserTokenForm(instance=token) return render(request, 'generic/object_edit.html', { 'object': token, 'form': form, - 'return_url': reverse('users:token_list'), + 'return_url': reverse('users:usertoken_list'), }) def post(self, request, pk=None): - if pk: - token = get_object_or_404(Token.objects.filter(user=request.user), pk=pk) - form = forms.TokenForm(request.POST, instance=token) + token = get_object_or_404(UserToken.objects.filter(user=request.user), pk=pk) + form = forms.UserTokenForm(request.POST, instance=token) else: - token = Token(user=request.user) - form = forms.TokenForm(request.POST) + token = UserToken(user=request.user) + form = forms.UserTokenForm(request.POST) if form.is_valid(): - token = form.save(commit=False) token.user = request.user token.save() @@ -304,7 +312,7 @@ class TokenEditView(LoginRequiredMixin, View): messages.success(request, msg) if not pk and not settings.ALLOW_TOKEN_RETRIEVAL: - return render(request, 'users/account/api_token.html', { + return render(request, 'users/account/token.html', { 'object': token, 'key': token.key, 'return_url': reverse('users:token_list'), @@ -312,53 +320,91 @@ class TokenEditView(LoginRequiredMixin, View): elif '_addanother' in request.POST: return redirect(request.path) else: - return redirect('users:token_list') + return redirect('users:usertoken_list') return render(request, 'generic/object_edit.html', { 'object': token, 'form': form, - 'return_url': reverse('users:token_list'), + 'return_url': reverse('users:usertoken_list'), 'disable_addanother': not settings.ALLOW_TOKEN_RETRIEVAL }) -@register_model_view(Token, 'delete') -class TokenDeleteView(LoginRequiredMixin, View): +@register_model_view(UserToken, 'delete') +class UserTokenDeleteView(LoginRequiredMixin, View): def get(self, request, pk): - - token = get_object_or_404(Token.objects.filter(user=request.user), pk=pk) - initial_data = { - 'return_url': reverse('users:token_list'), - } - form = ConfirmationForm(initial=initial_data) + token = get_object_or_404(UserToken.objects.filter(user=request.user), pk=pk) return render(request, 'generic/object_delete.html', { 'object': token, - 'form': form, - 'return_url': reverse('users:token_list'), + 'form': ConfirmationForm(), + 'return_url': reverse('users:usertoken_list'), }) def post(self, request, pk): - - token = get_object_or_404(Token.objects.filter(user=request.user), pk=pk) + token = get_object_or_404(UserToken.objects.filter(user=request.user), pk=pk) form = ConfirmationForm(request.POST) + if form.is_valid(): token.delete() messages.success(request, "Token deleted") - return redirect('users:token_list') + return redirect('users:usertoken_list') return render(request, 'generic/object_delete.html', { 'object': token, 'form': form, - 'return_url': reverse('users:token_list'), + 'return_url': reverse('users:usertoken_list'), }) + +# +# Tokens +# + +class TokenListView(generic.ObjectListView): + queryset = Token.objects.all() + filterset = filtersets.TokenFilterSet + filterset_form = forms.TokenFilterForm + table = tables.TokenTable + + +@register_model_view(Token) +class TokenView(generic.ObjectView): + queryset = Token.objects.all() + + +@register_model_view(Token, 'edit') +class TokenEditView(generic.ObjectEditView): + queryset = Token.objects.all() + form = forms.TokenForm + + +@register_model_view(Token, 'delete') +class TokenDeleteView(generic.ObjectDeleteView): + queryset = Token.objects.all() + + +class TokenBulkImportView(generic.BulkImportView): + queryset = Token.objects.all() + model_form = forms.TokenImportForm + + +class TokenBulkEditView(generic.BulkEditView): + queryset = Token.objects.all() + table = tables.TokenTable + form = forms.TokenBulkEditForm + + +class TokenBulkDeleteView(generic.BulkDeleteView): + queryset = Token.objects.all() + table = tables.TokenTable + + # # Users # - class UserListView(generic.ObjectListView): queryset = NetBoxUser.objects.all() filterset = filtersets.UserFilterSet @@ -413,7 +459,6 @@ class UserBulkDeleteView(generic.BulkDeleteView): # Groups # - class GroupListView(generic.ObjectListView): queryset = NetBoxGroup.objects.annotate(users_count=Count('user')) filterset = filtersets.GroupFilterSet @@ -448,11 +493,11 @@ class GroupBulkDeleteView(generic.BulkDeleteView): filterset = filtersets.GroupFilterSet table = tables.GroupTable + # # ObjectPermissions # - class ObjectPermissionListView(generic.ObjectListView): queryset = ObjectPermission.objects.all() filterset = filtersets.ObjectPermissionFilterSet