From f0a85b1dd3314c33b556021af9392607cce461f5 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 22 Aug 2016 15:16:49 -0400 Subject: [PATCH] Optimized API performance --- netbox/circuits/api/serializers.py | 6 +++--- netbox/circuits/api/views.py | 8 ++++---- netbox/dcim/api/serializers.py | 8 ++++---- netbox/dcim/api/views.py | 14 ++++++++------ netbox/extras/api/serializers.py | 18 +++++++++++++++--- netbox/extras/api/views.py | 11 +++++++++-- netbox/extras/models.py | 3 ++- netbox/ipam/api/serializers.py | 12 ++++++------ netbox/ipam/api/views.py | 22 ++++++++++++---------- netbox/tenancy/api/serializers.py | 4 ++-- netbox/tenancy/api/views.py | 4 ++-- 11 files changed, 67 insertions(+), 43 deletions(-) diff --git a/netbox/circuits/api/serializers.py b/netbox/circuits/api/serializers.py index 45f9e1b10..d7f32f958 100644 --- a/netbox/circuits/api/serializers.py +++ b/netbox/circuits/api/serializers.py @@ -2,7 +2,7 @@ from rest_framework import serializers from circuits.models import Provider, CircuitType, Circuit from dcim.api.serializers import SiteNestedSerializer, InterfaceNestedSerializer -from extras.api.serializers import CustomFieldsSerializer +from extras.api.serializers import CustomFieldSerializer from tenancy.api.serializers import TenantNestedSerializer @@ -10,7 +10,7 @@ from tenancy.api.serializers import TenantNestedSerializer # Providers # -class ProviderSerializer(CustomFieldsSerializer, serializers.ModelSerializer): +class ProviderSerializer(CustomFieldSerializer, serializers.ModelSerializer): class Meta: model = Provider @@ -45,7 +45,7 @@ class CircuitTypeNestedSerializer(CircuitTypeSerializer): # Circuits # -class CircuitSerializer(CustomFieldsSerializer, serializers.ModelSerializer): +class CircuitSerializer(CustomFieldSerializer, serializers.ModelSerializer): provider = ProviderNestedSerializer() type = CircuitTypeNestedSerializer() tenant = TenantNestedSerializer() diff --git a/netbox/circuits/api/views.py b/netbox/circuits/api/views.py index 35b13e492..866f9283b 100644 --- a/netbox/circuits/api/views.py +++ b/netbox/circuits/api/views.py @@ -11,7 +11,7 @@ class ProviderListView(CustomFieldModelAPIView, generics.ListAPIView): """ List all providers """ - queryset = Provider.objects.prefetch_related('custom_field_values') + queryset = Provider.objects.prefetch_related('custom_field_values__field') serializer_class = serializers.ProviderSerializer @@ -19,7 +19,7 @@ class ProviderDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView): """ Retrieve a single provider """ - queryset = Provider.objects.prefetch_related('custom_field_values') + queryset = Provider.objects.prefetch_related('custom_field_values__field') serializer_class = serializers.ProviderSerializer @@ -44,7 +44,7 @@ class CircuitListView(CustomFieldModelAPIView, generics.ListAPIView): List circuits (filterable) """ queryset = Circuit.objects.select_related('type', 'tenant', 'provider', 'site', 'interface__device')\ - .prefetch_related('custom_field_values') + .prefetch_related('custom_field_values__field') serializer_class = serializers.CircuitSerializer filter_class = CircuitFilter @@ -54,5 +54,5 @@ class CircuitDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView): Retrieve a single circuit """ queryset = Circuit.objects.select_related('type', 'tenant', 'provider', 'site', 'interface__device')\ - .prefetch_related('custom_field_values') + .prefetch_related('custom_field_values__field') serializer_class = serializers.CircuitSerializer diff --git a/netbox/dcim/api/serializers.py b/netbox/dcim/api/serializers.py index 786c8f717..f8329d38b 100644 --- a/netbox/dcim/api/serializers.py +++ b/netbox/dcim/api/serializers.py @@ -6,7 +6,7 @@ from dcim.models import ( DeviceRole, Interface, InterfaceConnection, InterfaceTemplate, Manufacturer, Module, Platform, PowerOutlet, PowerOutletTemplate, PowerPort, PowerPortTemplate, Rack, RackGroup, RackRole, RACK_FACE_FRONT, RACK_FACE_REAR, Site, ) -from extras.api.serializers import CustomFieldsSerializer +from extras.api.serializers import CustomFieldSerializer from tenancy.api.serializers import TenantNestedSerializer @@ -14,7 +14,7 @@ from tenancy.api.serializers import TenantNestedSerializer # Sites # -class SiteSerializer(CustomFieldsSerializer, serializers.ModelSerializer): +class SiteSerializer(CustomFieldSerializer, serializers.ModelSerializer): tenant = TenantNestedSerializer() class Meta: @@ -69,7 +69,7 @@ class RackRoleNestedSerializer(RackRoleSerializer): # -class RackSerializer(CustomFieldsSerializer, serializers.ModelSerializer): +class RackSerializer(CustomFieldSerializer, serializers.ModelSerializer): site = SiteNestedSerializer() group = RackGroupNestedSerializer() tenant = TenantNestedSerializer() @@ -238,7 +238,7 @@ class DeviceIPAddressNestedSerializer(serializers.ModelSerializer): fields = ['id', 'family', 'address'] -class DeviceSerializer(CustomFieldsSerializer, serializers.ModelSerializer): +class DeviceSerializer(CustomFieldSerializer, serializers.ModelSerializer): device_type = DeviceTypeNestedSerializer() device_role = DeviceRoleNestedSerializer() tenant = TenantNestedSerializer() diff --git a/netbox/dcim/api/views.py b/netbox/dcim/api/views.py index 193a16ad3..143887624 100644 --- a/netbox/dcim/api/views.py +++ b/netbox/dcim/api/views.py @@ -28,7 +28,7 @@ class SiteListView(CustomFieldModelAPIView, generics.ListAPIView): """ List all sites """ - queryset = Site.objects.select_related('tenant').prefetch_related('custom_field_values') + queryset = Site.objects.select_related('tenant').prefetch_related('custom_field_values__field') serializer_class = serializers.SiteSerializer @@ -36,7 +36,7 @@ class SiteDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView): """ Retrieve a single site """ - queryset = Site.objects.select_related('tenant').prefetch_related('custom_field_values') + queryset = Site.objects.select_related('tenant').prefetch_related('custom_field_values__field') serializer_class = serializers.SiteSerializer @@ -89,7 +89,8 @@ class RackListView(CustomFieldModelAPIView, generics.ListAPIView): """ List racks (filterable) """ - queryset = Rack.objects.select_related('site', 'group__site', 'tenant').prefetch_related('custom_field_values') + queryset = Rack.objects.select_related('site', 'group__site', 'tenant')\ + .prefetch_related('custom_field_values__field') serializer_class = serializers.RackSerializer filter_class = filters.RackFilter @@ -98,7 +99,8 @@ class RackDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView): """ Retrieve a single rack """ - queryset = Rack.objects.select_related('site', 'group__site', 'tenant').prefetch_related('custom_field_values') + queryset = Rack.objects.select_related('site', 'group__site', 'tenant')\ + .prefetch_related('custom_field_values__field') serializer_class = serializers.RackDetailSerializer @@ -217,7 +219,7 @@ class DeviceListView(CustomFieldModelAPIView, generics.ListAPIView): queryset = Device.objects.select_related('device_type__manufacturer', 'device_role', 'tenant', 'platform', 'rack__site', 'parent_bay').prefetch_related('primary_ip4__nat_outside', 'primary_ip6__nat_outside', - 'custom_field_values') + 'custom_field_values__field') serializer_class = serializers.DeviceSerializer filter_class = filters.DeviceFilter renderer_classes = api_settings.DEFAULT_RENDERER_CLASSES + [BINDZoneRenderer, FlatJSONRenderer] @@ -228,7 +230,7 @@ class DeviceDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView): Retrieve a single device """ queryset = Device.objects.select_related('device_type__manufacturer', 'device_role', 'tenant', 'platform', - 'rack__site', 'parent_bay').prefetch_related('custom_field_values') + 'rack__site', 'parent_bay').prefetch_related('custom_field_values__field') serializer_class = serializers.DeviceSerializer diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index a7dbeb5ed..ff1e39967 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -1,21 +1,33 @@ from rest_framework import serializers -from extras.models import CF_TYPE_SELECT, CustomFieldChoice, Graph +from extras.models import CF_TYPE_SELECT, CustomFieldChoice, CustomFieldValue, Graph -class CustomFieldsSerializer(serializers.Serializer): +class CustomFieldSerializer(serializers.Serializer): """ Extends a ModelSerializer to render any CustomFields and their values associated with an object. """ custom_fields = serializers.SerializerMethodField() def get_custom_fields(self, obj): + + # Gather all CustomFields applicable to this object fields = {cf.name: None for cf in self.context['view'].custom_fields} + + # Attach any defined CustomFieldValues to their respective CustomFields for cfv in obj.custom_field_values.all(): + + # Suppress database lookups for CustomFieldChoices. Instead, use the cached choice set from the view + # context. if cfv.field.type == CF_TYPE_SELECT: - fields[cfv.field.name] = CustomFieldChoiceSerializer(instance=cfv.value).data + cfc = { + 'id': int(cfv.serialized_value), + 'value': self.context['view'].custom_field_choices[int(cfv.serialized_value)] + } + fields[cfv.field.name] = CustomFieldChoiceSerializer(instance=cfc).data else: fields[cfv.field.name] = cfv.value + return fields diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index ee83ddf3d..0b0638fd7 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -9,7 +9,7 @@ from django.shortcuts import get_object_or_404 from circuits.models import Provider from dcim.models import Site, Device, Interface, InterfaceConnection -from extras.models import Graph, TopologyMap, GRAPH_TYPE_INTERFACE, GRAPH_TYPE_PROVIDER, GRAPH_TYPE_SITE +from extras.models import CustomFieldChoice, Graph, TopologyMap, GRAPH_TYPE_INTERFACE, GRAPH_TYPE_PROVIDER, GRAPH_TYPE_SITE from .serializers import GraphSerializer @@ -22,7 +22,14 @@ class CustomFieldModelAPIView(object): def __init__(self): super(CustomFieldModelAPIView, self).__init__() self.content_type = ContentType.objects.get_for_model(self.queryset.model) - self.custom_fields = self.content_type.custom_fields.all() + self.custom_fields = self.content_type.custom_fields.prefetch_related('choices') + + # Cache all relevant CustomFieldChoices. This saves us from having to do a lookup per select field per object. + custom_field_choices = {} + for field in self.custom_fields: + for cfc in field.choices.all(): + custom_field_choices[cfc.id] = cfc.value + self.custom_field_choices = custom_field_choices class GraphListView(generics.ListAPIView): diff --git a/netbox/extras/models.py b/netbox/extras/models.py index a3fdc2e6d..be4876fab 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -129,7 +129,8 @@ class CustomField(models.Model): # Read date as YYYY-MM-DD return date(*[int(n) for n in serialized_value.split('-')]) if self.type == CF_TYPE_SELECT: - return CustomFieldChoice.objects.get(pk=int(serialized_value)) + # return CustomFieldChoice.objects.get(pk=int(serialized_value)) + return self.choices.get(pk=int(serialized_value)) return serialized_value diff --git a/netbox/ipam/api/serializers.py b/netbox/ipam/api/serializers.py index 8d56d5516..653d9eba5 100644 --- a/netbox/ipam/api/serializers.py +++ b/netbox/ipam/api/serializers.py @@ -1,7 +1,7 @@ from rest_framework import serializers from dcim.api.serializers import SiteNestedSerializer, InterfaceNestedSerializer -from extras.api.serializers import CustomFieldsSerializer +from extras.api.serializers import CustomFieldSerializer from ipam.models import VRF, Role, RIR, Aggregate, Prefix, IPAddress, VLAN, VLANGroup from tenancy.api.serializers import TenantNestedSerializer @@ -10,7 +10,7 @@ from tenancy.api.serializers import TenantNestedSerializer # VRFs # -class VRFSerializer(CustomFieldsSerializer, serializers.ModelSerializer): +class VRFSerializer(CustomFieldSerializer, serializers.ModelSerializer): tenant = TenantNestedSerializer() class Meta: @@ -71,7 +71,7 @@ class RIRNestedSerializer(RIRSerializer): # Aggregates # -class AggregateSerializer(CustomFieldsSerializer, serializers.ModelSerializer): +class AggregateSerializer(CustomFieldSerializer, serializers.ModelSerializer): rir = RIRNestedSerializer() class Meta: @@ -107,7 +107,7 @@ class VLANGroupNestedSerializer(VLANGroupSerializer): # VLANs # -class VLANSerializer(CustomFieldsSerializer, serializers.ModelSerializer): +class VLANSerializer(CustomFieldSerializer, serializers.ModelSerializer): site = SiteNestedSerializer() group = VLANGroupNestedSerializer() tenant = TenantNestedSerializer() @@ -129,7 +129,7 @@ class VLANNestedSerializer(VLANSerializer): # Prefixes # -class PrefixSerializer(CustomFieldsSerializer, serializers.ModelSerializer): +class PrefixSerializer(CustomFieldSerializer, serializers.ModelSerializer): site = SiteNestedSerializer() vrf = VRFTenantSerializer() tenant = TenantNestedSerializer() @@ -152,7 +152,7 @@ class PrefixNestedSerializer(PrefixSerializer): # IP addresses # -class IPAddressSerializer(CustomFieldsSerializer, serializers.ModelSerializer): +class IPAddressSerializer(CustomFieldSerializer, serializers.ModelSerializer): vrf = VRFTenantSerializer() tenant = TenantNestedSerializer() interface = InterfaceNestedSerializer() diff --git a/netbox/ipam/api/views.py b/netbox/ipam/api/views.py index 5400331da..21ab9335c 100644 --- a/netbox/ipam/api/views.py +++ b/netbox/ipam/api/views.py @@ -15,7 +15,7 @@ class VRFListView(CustomFieldModelAPIView, generics.ListAPIView): """ List all VRFs """ - queryset = VRF.objects.select_related('tenant').prefetch_related('custom_field_values') + queryset = VRF.objects.select_related('tenant').prefetch_related('custom_field_values__field') serializer_class = serializers.VRFSerializer filter_class = filters.VRFFilter @@ -24,7 +24,7 @@ class VRFDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView): """ Retrieve a single VRF """ - queryset = VRF.objects.select_related('tenant').prefetch_related('custom_field_values') + queryset = VRF.objects.select_related('tenant').prefetch_related('custom_field_values__field') serializer_class = serializers.VRFSerializer @@ -76,7 +76,7 @@ class AggregateListView(CustomFieldModelAPIView, generics.ListAPIView): """ List aggregates (filterable) """ - queryset = Aggregate.objects.select_related('rir').prefetch_related('custom_field_values') + queryset = Aggregate.objects.select_related('rir').prefetch_related('custom_field_values__field') serializer_class = serializers.AggregateSerializer filter_class = filters.AggregateFilter @@ -85,7 +85,7 @@ class AggregateDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView): """ Retrieve a single aggregate """ - queryset = Aggregate.objects.select_related('rir').prefetch_related('custom_field_values') + queryset = Aggregate.objects.select_related('rir').prefetch_related('custom_field_values__field') serializer_class = serializers.AggregateSerializer @@ -98,7 +98,7 @@ class PrefixListView(CustomFieldModelAPIView, generics.ListAPIView): List prefixes (filterable) """ queryset = Prefix.objects.select_related('site', 'vrf__tenant', 'tenant', 'vlan', 'role')\ - .prefetch_related('custom_field_values') + .prefetch_related('custom_field_values__field') serializer_class = serializers.PrefixSerializer filter_class = filters.PrefixFilter @@ -108,7 +108,7 @@ class PrefixDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView): Retrieve a single prefix """ queryset = Prefix.objects.select_related('site', 'vrf__tenant', 'tenant', 'vlan', 'role')\ - .prefetch_related('custom_field_values') + .prefetch_related('custom_field_values__field') serializer_class = serializers.PrefixSerializer @@ -121,7 +121,7 @@ class IPAddressListView(CustomFieldModelAPIView, generics.ListAPIView): List IP addresses (filterable) """ queryset = IPAddress.objects.select_related('vrf__tenant', 'tenant', 'interface__device', 'nat_inside')\ - .prefetch_related('nat_outside', 'custom_field_values') + .prefetch_related('nat_outside', 'custom_field_values__field') serializer_class = serializers.IPAddressSerializer filter_class = filters.IPAddressFilter @@ -131,7 +131,7 @@ class IPAddressDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView): Retrieve a single IP address """ queryset = IPAddress.objects.select_related('vrf__tenant', 'tenant', 'interface__device', 'nat_inside')\ - .prefetch_related('nat_outside', 'custom_field_values') + .prefetch_related('nat_outside', 'custom_field_values__field') serializer_class = serializers.IPAddressSerializer @@ -164,7 +164,8 @@ class VLANListView(CustomFieldModelAPIView, generics.ListAPIView): """ List VLANs (filterable) """ - queryset = VLAN.objects.select_related('site', 'group', 'tenant', 'role').prefetch_related('custom_field_values') + queryset = VLAN.objects.select_related('site', 'group', 'tenant', 'role')\ + .prefetch_related('custom_field_values__field') serializer_class = serializers.VLANSerializer filter_class = filters.VLANFilter @@ -173,5 +174,6 @@ class VLANDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView): """ Retrieve a single VLAN """ - queryset = VLAN.objects.select_related('site', 'group', 'tenant', 'role').prefetch_related('custom_field_values') + queryset = VLAN.objects.select_related('site', 'group', 'tenant', 'role')\ + .prefetch_related('custom_field_values__field') serializer_class = serializers.VLANSerializer diff --git a/netbox/tenancy/api/serializers.py b/netbox/tenancy/api/serializers.py index 3e8d019e7..bde6f3345 100644 --- a/netbox/tenancy/api/serializers.py +++ b/netbox/tenancy/api/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from extras.api.serializers import CustomFieldsSerializer +from extras.api.serializers import CustomFieldSerializer from tenancy.models import Tenant, TenantGroup @@ -25,7 +25,7 @@ class TenantGroupNestedSerializer(TenantGroupSerializer): # Tenants # -class TenantSerializer(CustomFieldsSerializer, serializers.ModelSerializer): +class TenantSerializer(CustomFieldSerializer, serializers.ModelSerializer): group = TenantGroupNestedSerializer() class Meta: diff --git a/netbox/tenancy/api/views.py b/netbox/tenancy/api/views.py index 963af6e44..ce08eb058 100644 --- a/netbox/tenancy/api/views.py +++ b/netbox/tenancy/api/views.py @@ -27,7 +27,7 @@ class TenantListView(CustomFieldModelAPIView, generics.ListAPIView): """ List tenants (filterable) """ - queryset = Tenant.objects.select_related('group').prefetch_related('custom_field_values') + queryset = Tenant.objects.select_related('group').prefetch_related('custom_field_values__field') serializer_class = serializers.TenantSerializer filter_class = TenantFilter @@ -36,5 +36,5 @@ class TenantDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView): """ Retrieve a single tenant """ - queryset = Tenant.objects.select_related('group').prefetch_related('custom_field_values') + queryset = Tenant.objects.select_related('group').prefetch_related('custom_field_values__field') serializer_class = serializers.TenantSerializer