Optimized API performance

netbox/circuits/api/serializers.py

@@ -2,7 +2,7 @@ from rest_framework import serializers
 
 from circuits.models import Provider, CircuitType, Circuit
 from dcim.api.serializers import SiteNestedSerializer, InterfaceNestedSerializer
-from extras.api.serializers import CustomFieldsSerializer
+from extras.api.serializers import CustomFieldSerializer
 from tenancy.api.serializers import TenantNestedSerializer
 
 
@@ -10,7 +10,7 @@ from tenancy.api.serializers import TenantNestedSerializer
 # Providers
 #
 
-class ProviderSerializer(CustomFieldsSerializer, serializers.ModelSerializer):
+class ProviderSerializer(CustomFieldSerializer, serializers.ModelSerializer):
 
     class Meta:
         model = Provider
@@ -45,7 +45,7 @@ class CircuitTypeNestedSerializer(CircuitTypeSerializer):
 # Circuits
 #
 
-class CircuitSerializer(CustomFieldsSerializer, serializers.ModelSerializer):
+class CircuitSerializer(CustomFieldSerializer, serializers.ModelSerializer):
     provider = ProviderNestedSerializer()
     type = CircuitTypeNestedSerializer()
     tenant = TenantNestedSerializer()

netbox/circuits/api/views.py

@@ -11,7 +11,7 @@ class ProviderListView(CustomFieldModelAPIView, generics.ListAPIView):
     """
     List all providers
     """
-    queryset = Provider.objects.prefetch_related('custom_field_values')
+    queryset = Provider.objects.prefetch_related('custom_field_values__field')
     serializer_class = serializers.ProviderSerializer
 
 
@@ -19,7 +19,7 @@ class ProviderDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView):
     """
     Retrieve a single provider
     """
-    queryset = Provider.objects.prefetch_related('custom_field_values')
+    queryset = Provider.objects.prefetch_related('custom_field_values__field')
     serializer_class = serializers.ProviderSerializer
 
 
@@ -44,7 +44,7 @@ class CircuitListView(CustomFieldModelAPIView, generics.ListAPIView):
     List circuits (filterable)
     """
     queryset = Circuit.objects.select_related('type', 'tenant', 'provider', 'site', 'interface__device')\
-        .prefetch_related('custom_field_values')
+        .prefetch_related('custom_field_values__field')
     serializer_class = serializers.CircuitSerializer
     filter_class = CircuitFilter
 
@@ -54,5 +54,5 @@ class CircuitDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView):
     Retrieve a single circuit
     """
     queryset = Circuit.objects.select_related('type', 'tenant', 'provider', 'site', 'interface__device')\
-        .prefetch_related('custom_field_values')
+        .prefetch_related('custom_field_values__field')
     serializer_class = serializers.CircuitSerializer

netbox/dcim/api/serializers.py

@@ -6,7 +6,7 @@ from dcim.models import (
     DeviceRole, Interface, InterfaceConnection, InterfaceTemplate, Manufacturer, Module, Platform, PowerOutlet,
     PowerOutletTemplate, PowerPort, PowerPortTemplate, Rack, RackGroup, RackRole, RACK_FACE_FRONT, RACK_FACE_REAR, Site,
 )
-from extras.api.serializers import CustomFieldsSerializer
+from extras.api.serializers import CustomFieldSerializer
 from tenancy.api.serializers import TenantNestedSerializer
 
 
@@ -14,7 +14,7 @@ from tenancy.api.serializers import TenantNestedSerializer
 # Sites
 #
 
-class SiteSerializer(CustomFieldsSerializer, serializers.ModelSerializer):
+class SiteSerializer(CustomFieldSerializer, serializers.ModelSerializer):
     tenant = TenantNestedSerializer()
 
     class Meta:
@@ -69,7 +69,7 @@ class RackRoleNestedSerializer(RackRoleSerializer):
 #
 
 
-class RackSerializer(CustomFieldsSerializer, serializers.ModelSerializer):
+class RackSerializer(CustomFieldSerializer, serializers.ModelSerializer):
     site = SiteNestedSerializer()
     group = RackGroupNestedSerializer()
     tenant = TenantNestedSerializer()
@@ -238,7 +238,7 @@ class DeviceIPAddressNestedSerializer(serializers.ModelSerializer):
         fields = ['id', 'family', 'address']
 
 
-class DeviceSerializer(CustomFieldsSerializer, serializers.ModelSerializer):
+class DeviceSerializer(CustomFieldSerializer, serializers.ModelSerializer):
     device_type = DeviceTypeNestedSerializer()
     device_role = DeviceRoleNestedSerializer()
     tenant = TenantNestedSerializer()

netbox/dcim/api/views.py

@@ -28,7 +28,7 @@ class SiteListView(CustomFieldModelAPIView, generics.ListAPIView):
     """
     List all sites
     """
-    queryset = Site.objects.select_related('tenant').prefetch_related('custom_field_values')
+    queryset = Site.objects.select_related('tenant').prefetch_related('custom_field_values__field')
     serializer_class = serializers.SiteSerializer
 
 
@@ -36,7 +36,7 @@ class SiteDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView):
     """
     Retrieve a single site
     """
-    queryset = Site.objects.select_related('tenant').prefetch_related('custom_field_values')
+    queryset = Site.objects.select_related('tenant').prefetch_related('custom_field_values__field')
     serializer_class = serializers.SiteSerializer
 
 
@@ -89,7 +89,8 @@ class RackListView(CustomFieldModelAPIView, generics.ListAPIView):
     """
     List racks (filterable)
     """
-    queryset = Rack.objects.select_related('site', 'group__site', 'tenant').prefetch_related('custom_field_values')
+    queryset = Rack.objects.select_related('site', 'group__site', 'tenant')\
+        .prefetch_related('custom_field_values__field')
     serializer_class = serializers.RackSerializer
     filter_class = filters.RackFilter
 
@@ -98,7 +99,8 @@ class RackDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView):
     """
     Retrieve a single rack
     """
-    queryset = Rack.objects.select_related('site', 'group__site', 'tenant').prefetch_related('custom_field_values')
+    queryset = Rack.objects.select_related('site', 'group__site', 'tenant')\
+        .prefetch_related('custom_field_values__field')
     serializer_class = serializers.RackDetailSerializer
 
 
@@ -217,7 +219,7 @@ class DeviceListView(CustomFieldModelAPIView, generics.ListAPIView):
     queryset = Device.objects.select_related('device_type__manufacturer', 'device_role', 'tenant', 'platform',
                                              'rack__site', 'parent_bay').prefetch_related('primary_ip4__nat_outside',
                                                                                           'primary_ip6__nat_outside',
-                                                                                          'custom_field_values')
+                                                                                          'custom_field_values__field')
     serializer_class = serializers.DeviceSerializer
     filter_class = filters.DeviceFilter
     renderer_classes = api_settings.DEFAULT_RENDERER_CLASSES + [BINDZoneRenderer, FlatJSONRenderer]
@@ -228,7 +230,7 @@ class DeviceDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView):
     Retrieve a single device
     """
     queryset = Device.objects.select_related('device_type__manufacturer', 'device_role', 'tenant', 'platform',
-                                             'rack__site', 'parent_bay').prefetch_related('custom_field_values')
+                                             'rack__site', 'parent_bay').prefetch_related('custom_field_values__field')
     serializer_class = serializers.DeviceSerializer
 
 

netbox/extras/api/serializers.py

@@ -1,21 +1,33 @@
 from rest_framework import serializers
 
-from extras.models import CF_TYPE_SELECT, CustomFieldChoice, Graph
+from extras.models import CF_TYPE_SELECT, CustomFieldChoice, CustomFieldValue, Graph
 
 
-class CustomFieldsSerializer(serializers.Serializer):
+class CustomFieldSerializer(serializers.Serializer):
     """
     Extends a ModelSerializer to render any CustomFields and their values associated with an object.
     """
     custom_fields = serializers.SerializerMethodField()
 
     def get_custom_fields(self, obj):
+
+        # Gather all CustomFields applicable to this object
         fields = {cf.name: None for cf in self.context['view'].custom_fields}
+
+        # Attach any defined CustomFieldValues to their respective CustomFields
         for cfv in obj.custom_field_values.all():
+
+            # Suppress database lookups for CustomFieldChoices. Instead, use the cached choice set from the view
+            # context.
             if cfv.field.type == CF_TYPE_SELECT:
-                fields[cfv.field.name] = CustomFieldChoiceSerializer(instance=cfv.value).data
+                cfc = {
+                    'id': int(cfv.serialized_value),
+                    'value': self.context['view'].custom_field_choices[int(cfv.serialized_value)]
+                }
+                fields[cfv.field.name] = CustomFieldChoiceSerializer(instance=cfc).data
             else:
                 fields[cfv.field.name] = cfv.value
+
         return fields
 
 

netbox/extras/api/views.py

@@ -9,7 +9,7 @@ from django.shortcuts import get_object_or_404
 
 from circuits.models import Provider
 from dcim.models import Site, Device, Interface, InterfaceConnection
-from extras.models import Graph, TopologyMap, GRAPH_TYPE_INTERFACE, GRAPH_TYPE_PROVIDER, GRAPH_TYPE_SITE
+from extras.models import CustomFieldChoice, Graph, TopologyMap, GRAPH_TYPE_INTERFACE, GRAPH_TYPE_PROVIDER, GRAPH_TYPE_SITE
 
 from .serializers import GraphSerializer
 
@@ -22,7 +22,14 @@ class CustomFieldModelAPIView(object):
     def __init__(self):
         super(CustomFieldModelAPIView, self).__init__()
         self.content_type = ContentType.objects.get_for_model(self.queryset.model)
-        self.custom_fields = self.content_type.custom_fields.all()
+        self.custom_fields = self.content_type.custom_fields.prefetch_related('choices')
+
+        # Cache all relevant CustomFieldChoices. This saves us from having to do a lookup per select field per object.
+        custom_field_choices = {}
+        for field in self.custom_fields:
+            for cfc in field.choices.all():
+                custom_field_choices[cfc.id] = cfc.value
+        self.custom_field_choices = custom_field_choices
 
 
 class GraphListView(generics.ListAPIView):

netbox/extras/models.py

@@ -129,7 +129,8 @@ class CustomField(models.Model):
             # Read date as YYYY-MM-DD
             return date(*[int(n) for n in serialized_value.split('-')])
         if self.type == CF_TYPE_SELECT:
-            return CustomFieldChoice.objects.get(pk=int(serialized_value))
+            # return CustomFieldChoice.objects.get(pk=int(serialized_value))
+            return self.choices.get(pk=int(serialized_value))
         return serialized_value
 
 

netbox/ipam/api/serializers.py

@@ -1,7 +1,7 @@
 from rest_framework import serializers
 
 from dcim.api.serializers import SiteNestedSerializer, InterfaceNestedSerializer
-from extras.api.serializers import CustomFieldsSerializer
+from extras.api.serializers import CustomFieldSerializer
 from ipam.models import VRF, Role, RIR, Aggregate, Prefix, IPAddress, VLAN, VLANGroup
 from tenancy.api.serializers import TenantNestedSerializer
 
@@ -10,7 +10,7 @@ from tenancy.api.serializers import TenantNestedSerializer
 # VRFs
 #
 
-class VRFSerializer(CustomFieldsSerializer, serializers.ModelSerializer):
+class VRFSerializer(CustomFieldSerializer, serializers.ModelSerializer):
     tenant = TenantNestedSerializer()
 
     class Meta:
@@ -71,7 +71,7 @@ class RIRNestedSerializer(RIRSerializer):
 # Aggregates
 #
 
-class AggregateSerializer(CustomFieldsSerializer, serializers.ModelSerializer):
+class AggregateSerializer(CustomFieldSerializer, serializers.ModelSerializer):
     rir = RIRNestedSerializer()
 
     class Meta:
@@ -107,7 +107,7 @@ class VLANGroupNestedSerializer(VLANGroupSerializer):
 # VLANs
 #
 
-class VLANSerializer(CustomFieldsSerializer, serializers.ModelSerializer):
+class VLANSerializer(CustomFieldSerializer, serializers.ModelSerializer):
     site = SiteNestedSerializer()
     group = VLANGroupNestedSerializer()
     tenant = TenantNestedSerializer()
@@ -129,7 +129,7 @@ class VLANNestedSerializer(VLANSerializer):
 # Prefixes
 #
 
-class PrefixSerializer(CustomFieldsSerializer, serializers.ModelSerializer):
+class PrefixSerializer(CustomFieldSerializer, serializers.ModelSerializer):
     site = SiteNestedSerializer()
     vrf = VRFTenantSerializer()
     tenant = TenantNestedSerializer()
@@ -152,7 +152,7 @@ class PrefixNestedSerializer(PrefixSerializer):
 # IP addresses
 #
 
-class IPAddressSerializer(CustomFieldsSerializer, serializers.ModelSerializer):
+class IPAddressSerializer(CustomFieldSerializer, serializers.ModelSerializer):
     vrf = VRFTenantSerializer()
     tenant = TenantNestedSerializer()
     interface = InterfaceNestedSerializer()

netbox/ipam/api/views.py

@@ -15,7 +15,7 @@ class VRFListView(CustomFieldModelAPIView, generics.ListAPIView):
     """
     List all VRFs
     """
-    queryset = VRF.objects.select_related('tenant').prefetch_related('custom_field_values')
+    queryset = VRF.objects.select_related('tenant').prefetch_related('custom_field_values__field')
     serializer_class = serializers.VRFSerializer
     filter_class = filters.VRFFilter
 
@@ -24,7 +24,7 @@ class VRFDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView):
     """
     Retrieve a single VRF
     """
-    queryset = VRF.objects.select_related('tenant').prefetch_related('custom_field_values')
+    queryset = VRF.objects.select_related('tenant').prefetch_related('custom_field_values__field')
     serializer_class = serializers.VRFSerializer
 
 
@@ -76,7 +76,7 @@ class AggregateListView(CustomFieldModelAPIView, generics.ListAPIView):
     """
     List aggregates (filterable)
     """
-    queryset = Aggregate.objects.select_related('rir').prefetch_related('custom_field_values')
+    queryset = Aggregate.objects.select_related('rir').prefetch_related('custom_field_values__field')
     serializer_class = serializers.AggregateSerializer
     filter_class = filters.AggregateFilter
 
@@ -85,7 +85,7 @@ class AggregateDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView):
     """
     Retrieve a single aggregate
     """
-    queryset = Aggregate.objects.select_related('rir').prefetch_related('custom_field_values')
+    queryset = Aggregate.objects.select_related('rir').prefetch_related('custom_field_values__field')
     serializer_class = serializers.AggregateSerializer
 
 
@@ -98,7 +98,7 @@ class PrefixListView(CustomFieldModelAPIView, generics.ListAPIView):
     List prefixes (filterable)
     """
     queryset = Prefix.objects.select_related('site', 'vrf__tenant', 'tenant', 'vlan', 'role')\
-        .prefetch_related('custom_field_values')
+        .prefetch_related('custom_field_values__field')
     serializer_class = serializers.PrefixSerializer
     filter_class = filters.PrefixFilter
 
@@ -108,7 +108,7 @@ class PrefixDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView):
     Retrieve a single prefix
     """
     queryset = Prefix.objects.select_related('site', 'vrf__tenant', 'tenant', 'vlan', 'role')\
-        .prefetch_related('custom_field_values')
+        .prefetch_related('custom_field_values__field')
     serializer_class = serializers.PrefixSerializer
 
 
@@ -121,7 +121,7 @@ class IPAddressListView(CustomFieldModelAPIView, generics.ListAPIView):
     List IP addresses (filterable)
     """
     queryset = IPAddress.objects.select_related('vrf__tenant', 'tenant', 'interface__device', 'nat_inside')\
-        .prefetch_related('nat_outside', 'custom_field_values')
+        .prefetch_related('nat_outside', 'custom_field_values__field')
     serializer_class = serializers.IPAddressSerializer
     filter_class = filters.IPAddressFilter
 
@@ -131,7 +131,7 @@ class IPAddressDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView):
     Retrieve a single IP address
     """
     queryset = IPAddress.objects.select_related('vrf__tenant', 'tenant', 'interface__device', 'nat_inside')\
-        .prefetch_related('nat_outside', 'custom_field_values')
+        .prefetch_related('nat_outside', 'custom_field_values__field')
     serializer_class = serializers.IPAddressSerializer
 
 
@@ -164,7 +164,8 @@ class VLANListView(CustomFieldModelAPIView, generics.ListAPIView):
     """
     List VLANs (filterable)
     """
-    queryset = VLAN.objects.select_related('site', 'group', 'tenant', 'role').prefetch_related('custom_field_values')
+    queryset = VLAN.objects.select_related('site', 'group', 'tenant', 'role')\
+        .prefetch_related('custom_field_values__field')
     serializer_class = serializers.VLANSerializer
     filter_class = filters.VLANFilter
 
@@ -173,5 +174,6 @@ class VLANDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView):
     """
     Retrieve a single VLAN
     """
-    queryset = VLAN.objects.select_related('site', 'group', 'tenant', 'role').prefetch_related('custom_field_values')
+    queryset = VLAN.objects.select_related('site', 'group', 'tenant', 'role')\
+        .prefetch_related('custom_field_values__field')
     serializer_class = serializers.VLANSerializer

netbox/tenancy/api/serializers.py

@@ -1,6 +1,6 @@
 from rest_framework import serializers
 
-from extras.api.serializers import CustomFieldsSerializer
+from extras.api.serializers import CustomFieldSerializer
 from tenancy.models import Tenant, TenantGroup
 
 
@@ -25,7 +25,7 @@ class TenantGroupNestedSerializer(TenantGroupSerializer):
 # Tenants
 #
 
-class TenantSerializer(CustomFieldsSerializer, serializers.ModelSerializer):
+class TenantSerializer(CustomFieldSerializer, serializers.ModelSerializer):
     group = TenantGroupNestedSerializer()
 
     class Meta:

netbox/tenancy/api/views.py

@@ -27,7 +27,7 @@ class TenantListView(CustomFieldModelAPIView, generics.ListAPIView):
     """
     List tenants (filterable)
     """
-    queryset = Tenant.objects.select_related('group').prefetch_related('custom_field_values')
+    queryset = Tenant.objects.select_related('group').prefetch_related('custom_field_values__field')
     serializer_class = serializers.TenantSerializer
     filter_class = TenantFilter
 
@@ -36,5 +36,5 @@ class TenantDetailView(CustomFieldModelAPIView, generics.RetrieveAPIView):
     """
     Retrieve a single tenant
     """
-    queryset = Tenant.objects.select_related('group').prefetch_related('custom_field_values')
+    queryset = Tenant.objects.select_related('group').prefetch_related('custom_field_values__field')
     serializer_class = serializers.TenantSerializer