NeoAnd/netbox
1
0
Fork 0
mirror of https://github.com/netbox-community/netbox.git synced 2025-07-25 18:08:38 -06:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

13745 device type migration (#13747)

Browse Source 
* 13745 update migrations to use batch_size

* 13745 update migrations to use subquery update

* 13745 refactor and update other counter migrations
This commit is contained in:
Arthur Hanson 2023-09-18 06:59:26 -07:00 committed by GitHub
parent 3d1f668235
commit b0541be107
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 47 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
netbox/dcim/migrations/0176_device_component_counters.py
View File

@ -2,47 +2,22 @@ from django.db import migrations
from django.db.models import Count from django.db.models import Count
import utilities.fields import utilities.fields
from utilities.counters import update_counts
def recalculate_device_counts(apps, schema_editor): def recalculate_device_counts(apps, schema_editor):
Device = apps.get_model("dcim", "Device") Device = apps.get_model("dcim", "Device")
devices = Device.objects.annotate(
_console_port_count=Count('consoleports', distinct=True),
_console_server_port_count=Count('consoleserverports', distinct=True),
_power_port_count=Count('powerports', distinct=True),
_power_outlet_count=Count('poweroutlets', distinct=True),
_interface_count=Count('interfaces', distinct=True),
_front_port_count=Count('frontports', distinct=True),
_rear_port_count=Count('rearports', distinct=True),
_device_bay_count=Count('devicebays', distinct=True),
_module_bay_count=Count('modulebays', distinct=True),
_inventory_item_count=Count('inventoryitems', distinct=True),
)
for device in devices: update_counts(Device, 'console_port_count', 'consoleports')
device.console_port_count = device._console_port_count update_counts(Device, 'console_server_port_count', 'consoleserverports')
device.console_server_port_count = device._console_server_port_count update_counts(Device, 'power_port_count', 'powerports')
device.power_port_count = device._power_port_count update_counts(Device, 'power_outlet_count', 'poweroutlets')
device.power_outlet_count = device._power_outlet_count update_counts(Device, 'interface_count', 'interfaces')
device.interface_count = device._interface_count update_counts(Device, 'front_port_count', 'frontports')
device.front_port_count = device._front_port_count update_counts(Device, 'rear_port_count', 'rearports')
device.rear_port_count = device._rear_port_count update_counts(Device, 'device_bay_count', 'devicebays')
device.device_bay_count = device._device_bay_count update_counts(Device, 'module_bay_count', 'modulebays')
device.module_bay_count = device._module_bay_count update_counts(Device, 'inventory_item_count', 'inventoryitems')
device.inventory_item_count = device._inventory_item_count
Device.objects.bulk_update(devices, [
'console_port_count',
'console_server_port_count',
'power_port_count',
'power_outlet_count',
'interface_count',
'front_port_count',
'rear_port_count',
'device_bay_count',
'module_bay_count',
'inventory_item_count',
], batch_size=100)
class Migration(migrations.Migration): class Migration(migrations.Migration):

47
netbox/dcim/migrations/0177_devicetype_component_counters.py
View File

@ -2,47 +2,22 @@ from django.db import migrations
from django.db.models import Count from django.db.models import Count
import utilities.fields import utilities.fields
from utilities.counters import update_counts
def recalculate_devicetype_template_counts(apps, schema_editor): def recalculate_devicetype_template_counts(apps, schema_editor):
DeviceType = apps.get_model("dcim", "DeviceType") DeviceType = apps.get_model("dcim", "DeviceType")
device_types = list(DeviceType.objects.all().annotate(
_console_port_template_count=Count('consoleporttemplates', distinct=True),
_console_server_port_template_count=Count('consoleserverporttemplates', distinct=True),
_power_port_template_count=Count('powerporttemplates', distinct=True),
_power_outlet_template_count=Count('poweroutlettemplates', distinct=True),
_interface_template_count=Count('interfacetemplates', distinct=True),
_front_port_template_count=Count('frontporttemplates', distinct=True),
_rear_port_template_count=Count('rearporttemplates', distinct=True),
_device_bay_template_count=Count('devicebaytemplates', distinct=True),
_module_bay_template_count=Count('modulebaytemplates', distinct=True),
_inventory_item_template_count=Count('inventoryitemtemplates', distinct=True),
))
for devicetype in device_types: update_counts(DeviceType, 'console_port_template_count', 'consoleporttemplates')
devicetype.console_port_template_count = devicetype._console_port_template_count update_counts(DeviceType, 'console_server_port_template_count', 'consoleserverporttemplates')
devicetype.console_server_port_template_count = devicetype._console_server_port_template_count update_counts(DeviceType, 'power_port_template_count', 'powerporttemplates')
devicetype.power_port_template_count = devicetype._power_port_template_count update_counts(DeviceType, 'power_outlet_template_count', 'poweroutlettemplates')
devicetype.power_outlet_template_count = devicetype._power_outlet_template_count update_counts(DeviceType, 'interface_template_count', 'interfacetemplates')
devicetype.interface_template_count = devicetype._interface_template_count update_counts(DeviceType, 'front_port_template_count', 'frontporttemplates')
devicetype.front_port_template_count = devicetype._front_port_template_count update_counts(DeviceType, 'rear_port_template_count', 'rearporttemplates')
devicetype.rear_port_template_count = devicetype._rear_port_template_count update_counts(DeviceType, 'device_bay_template_count', 'devicebaytemplates')
devicetype.device_bay_template_count = devicetype._device_bay_template_count update_counts(DeviceType, 'module_bay_template_count', 'modulebaytemplates')
devicetype.module_bay_template_count = devicetype._module_bay_template_count update_counts(DeviceType, 'inventory_item_template_count', 'inventoryitemtemplates')
devicetype.inventory_item_template_count = devicetype._inventory_item_template_count
DeviceType.objects.bulk_update(device_types, [
'console_port_template_count',
'console_server_port_template_count',
'power_port_template_count',
'power_outlet_template_count',
'interface_template_count',
'front_port_template_count',
'rear_port_template_count',
'device_bay_template_count',
'module_bay_template_count',
'inventory_item_template_count',
])
class Migration(migrations.Migration): class Migration(migrations.Migration):

8
netbox/dcim/migrations/0178_virtual_chassis_member_counter.py
View File

@ -2,17 +2,13 @@ from django.db import migrations
from django.db.models import Count from django.db.models import Count
import utilities.fields import utilities.fields
from utilities.counters import update_counts
def populate_virtualchassis_members(apps, schema_editor): def populate_virtualchassis_members(apps, schema_editor):
VirtualChassis = apps.get_model('dcim', 'VirtualChassis') VirtualChassis = apps.get_model('dcim', 'VirtualChassis')
vcs = VirtualChassis.objects.annotate(_member_count=Count('members', distinct=True)) update_counts(VirtualChassis, 'member_count', 'members')
for vc in vcs:
vc.member_count = vc._member_count
VirtualChassis.objects.bulk_update(vcs, ['member_count'], batch_size=100)
class Migration(migrations.Migration): class Migration(migrations.Migration):

20
netbox/utilities/counters.py
View File

@ -1,5 +1,5 @@
from django.apps import apps from django.apps import apps
from django.db.models import F from django.db.models import F, Count, OuterRef, Subquery
from django.db.models.signals import post_delete, post_save from django.db.models.signals import post_delete, post_save
from netbox.registry import registry from netbox.registry import registry
@ -23,6 +23,24 @@ def update_counter(model, pk, counter_name, value):
) )
def update_counts(model, field_name, related_query):
"""
Perform a bulk update for the given model and counter field. For example,
update_counts(Device, '_interface_count', 'interfaces')
will effectively set
Device.objects.update(_interface_count=Count('interfaces'))
"""
subquery = Subquery(
model.objects.filter(pk=OuterRef('pk')).annotate(_count=Count(related_query)).values('_count')
)
return model.objects.update(**{
field_name: subquery
})
# #
# Signal handlers # Signal handlers
# #

21
netbox/utilities/management/commands/calculate_cached_counts.py
View File

@ -4,6 +4,7 @@ from django.core.management.base import BaseCommand
from django.db.models import Count, OuterRef, Subquery from django.db.models import Count, OuterRef, Subquery
from netbox.registry import registry from netbox.registry import registry
from utilities.counters import update_counts
class Command(BaseCommand): class Command(BaseCommand):
@ -26,27 +27,9 @@ class Command(BaseCommand):
return models return models
def update_counts(self, model, field_name, related_query):
"""
Perform a bulk update for the given model and counter field. For example,
update_counts(Device, '_interface_count', 'interfaces')
will effectively set
Device.objects.update(_interface_count=Count('interfaces'))
"""
self.stdout.write(f'Updating {model.__name__} {field_name}...')
subquery = Subquery(
model.objects.filter(pk=OuterRef('pk')).annotate(_count=Count(related_query)).values('_count')
)
return model.objects.update(**{
field_name: subquery
})
def handle(self, *model_names, **options): def handle(self, *model_names, **options):
for model, mappings in self.collect_models().items(): for model, mappings in self.collect_models().items():
for field_name, related_query in mappings.items(): for field_name, related_query in mappings.items():
self.update_counts(model, field_name, related_query) update_counts(model, field_name, related_query)
self.stdout.write(self.style.SUCCESS('Finished.')) self.stdout.write(self.style.SUCCESS('Finished.'))

8
netbox/virtualization/migrations/0035_virtualmachine_interface_count.py
View File

@ -2,17 +2,13 @@ from django.db import migrations
from django.db.models import Count from django.db.models import Count
import utilities.fields import utilities.fields
from utilities.counters import update_counts
def populate_virtualmachine_counts(apps, schema_editor): def populate_virtualmachine_counts(apps, schema_editor):
VirtualMachine = apps.get_model('virtualization', 'VirtualMachine') VirtualMachine = apps.get_model('virtualization', 'VirtualMachine')
vms = VirtualMachine.objects.annotate(_interface_count=Count('interfaces', distinct=True)) update_counts(VirtualMachine, 'interface_count', 'interfaces')
for vm in vms:
vm.interface_count = vm._interface_count
VirtualMachine.objects.bulk_update(vms, ['interface_count'], batch_size=100)
class Migration(migrations.Migration): class Migration(migrations.Migration):