Closes #21300: Cache model-specific custom field lookups for the duration of a request

netbox/extras/api/customfields.py

@@ -4,7 +4,6 @@ from drf_spectacular.utils import extend_schema_field
 from rest_framework.fields import Field
 from rest_framework.serializers import ValidationError
 
-from core.models import ObjectType
 from extras.choices import CustomFieldTypeChoices
 from extras.constants import CUSTOMFIELD_EMPTY_VALUES
 from extras.models import CustomField
@@ -25,8 +24,7 @@ class CustomFieldDefaultValues:
         self.model = serializer_field.parent.Meta.model
 
         # Retrieve the CustomFields for the parent model
-        object_type = ObjectType.objects.get_for_model(self.model)
+        fields = CustomField.objects.get_for_model(self.model)
-        fields = CustomField.objects.filter(object_types=object_type)
 
         # Populate the default value for each CustomField
         value = {}
@@ -47,8 +45,7 @@ class CustomFieldsDataField(Field):
         Cache CustomFields assigned to this model to avoid redundant database queries
         """
         if not hasattr(self, '_custom_fields'):
-            object_type = ObjectType.objects.get_for_model(self.parent.Meta.model)
+            self._custom_fields = CustomField.objects.get_for_model(self.parent.Meta.model)
-            self._custom_fields = CustomField.objects.filter(object_types=object_type)
         return self._custom_fields
 
     def to_representation(self, obj):

netbox/extras/models/customfields.py

@@ -19,6 +19,7 @@ from django.utils.translation import gettext_lazy as _
 from core.models import ObjectType
 from extras.choices import *
 from extras.data import CHOICE_SETS
+from netbox.context import query_cache
 from netbox.models import ChangeLoggedModel
 from netbox.models.features import CloningMixin, ExportTemplatesMixin
 from netbox.models.mixins import OwnerMixin
@@ -58,8 +59,20 @@ class CustomFieldManager(models.Manager.from_queryset(RestrictedQuerySet)):
         """
         Return all CustomFields assigned to the given model.
         """
+        # Check the request cache before hitting the database
+        cache = query_cache.get()
+        if cache is not None:
+            if custom_fields := cache['custom_fields'].get(model._meta.model):
+                return custom_fields
+
         content_type = ObjectType.objects.get_for_model(model._meta.concrete_model)
-        return self.get_queryset().filter(object_types=content_type)
+        custom_fields = self.get_queryset().filter(object_types=content_type)
+
+        # Populate the request cache to avoid redundant lookups
+        if cache is not None:
+            cache['custom_fields'][model._meta.model] = custom_fields
+
+        return custom_fields
 
     def get_defaults_for_model(self, model):
         """

netbox/netbox/filtersets.py

@@ -306,11 +306,10 @@ class NetBoxModelFilterSet(ChangeLoggedModelFilterSet):
         super().__init__(*args, **kwargs)
 
         # Dynamically add a Filter for each CustomField applicable to the parent model
-        custom_fields = CustomField.objects.filter(
+        custom_fields = [
-            object_types=ContentType.objects.get_for_model(self._meta.model)
+            cf for cf in CustomField.objects.get_for_model(self._meta.model)
-        ).exclude(
+            if cf.filter_logic != CustomFieldFilterLogicChoices.FILTER_DISABLED
-            filter_logic=CustomFieldFilterLogicChoices.FILTER_DISABLED
+        ]
-        )
 
         custom_field_filters = {}
         for custom_field in custom_fields:

netbox/netbox/forms/bulk_import.py

@@ -31,10 +31,10 @@ class NetBoxModelImportForm(CSVModelForm, NetBoxModelForm):
     )
 
     def _get_custom_fields(self, content_type):
-        return CustomField.objects.filter(
+        return [
-            object_types=content_type,
+            cf for cf in CustomField.objects.get_for_model(content_type.model_class())
-            ui_editable=CustomFieldUIEditableChoices.YES
+            if cf.ui_editable == CustomFieldUIEditableChoices.YES
-        )
+        ]
 
     def _get_form_field(self, customfield):
         return customfield.to_form_field(for_csv_import=True)

netbox/netbox/forms/filtersets.py

@@ -1,5 +1,4 @@
 from django import forms
-from django.db.models import Q
 from django.utils.translation import gettext_lazy as _
 
 from extras.choices import *
@@ -35,10 +34,12 @@ class NetBoxModelFilterSetForm(FilterModifierMixin, CustomFieldsMixin, SavedFilt
     selector_fields = ('filter_id', 'q')
 
     def _get_custom_fields(self, content_type):
-        return super()._get_custom_fields(content_type).exclude(
+        return [
-            Q(filter_logic=CustomFieldFilterLogicChoices.FILTER_DISABLED) |
+            cf for cf in super()._get_custom_fields(content_type) if (
-            Q(type=CustomFieldTypeChoices.TYPE_JSON)
+                cf.filter_logic != CustomFieldFilterLogicChoices.FILTER_DISABLED and
-        )
+                cf.type != CustomFieldTypeChoices.TYPE_JSON
+            )
+        ]
 
     def _get_form_field(self, customfield):
         return customfield.to_form_field(

netbox/netbox/forms/mixins.py

@@ -65,9 +65,10 @@ class CustomFieldsMixin:
         return ObjectType.objects.get_for_model(self.model)
 
     def _get_custom_fields(self, content_type):
-        return CustomField.objects.filter(object_types=content_type).exclude(
+        return [
-            ui_editable=CustomFieldUIEditableChoices.HIDDEN
+            cf for cf in CustomField.objects.get_for_model(content_type.model_class())
-        )
+            if cf.ui_editable != CustomFieldUIEditableChoices.HIDDEN
+        ]
 
     def _get_form_field(self, customfield):
         return customfield.to_form_field()

netbox/netbox/models/features.py

@@ -319,9 +319,11 @@ class CustomFieldsMixin(models.Model):
                 raise ValidationError(_("Missing required custom field '{name}'.").format(name=cf.name))
 
     def save(self, *args, **kwargs):
+        from extras.models import CustomField
+
         # Populate default values if omitted
-        for cf in self.custom_fields.filter(default__isnull=False):
+        for cf in CustomField.objects.get_for_model(self):
-            if cf.name not in self.custom_field_data:
+            if cf.name not in self.custom_field_data and cf.default is not None:
                 self.custom_field_data[cf.name] = cf.default
 
         super().save(*args, **kwargs)

netbox/netbox/search/backends.py

@@ -208,9 +208,12 @@ class CachedValueSearchBackend(SearchBackend):
                     except KeyError:
                         break
 
-                # Prefetch any associated custom fields
+                # Prefetch any associated custom fields (excluding those with a zero search weight)
                 object_type = ObjectType.objects.get_for_model(indexer.model)
-                custom_fields = CustomField.objects.filter(object_types=object_type).exclude(search_weight=0)
+                custom_fields = [
+                    cf for cf in CustomField.objects.get_for_model(indexer.model)
+                    if cf.search_weight > 0
+                ]
 
             # Wipe out any previously cached values for the object
             if remove_existing:

netbox/netbox/tables/tables.py

@@ -244,9 +244,10 @@ class NetBoxTable(BaseTable):
 
         # Add custom field & custom link columns
         object_type = ObjectType.objects.get_for_model(self._meta.model)
-        custom_fields = CustomField.objects.filter(
+        custom_fields = [
-            object_types=object_type
+            cf for cf in CustomField.objects.get_for_model(self._meta.model)
-        ).exclude(ui_visible=CustomFieldUIVisibleChoices.HIDDEN)
+            if cf.ui_visible != CustomFieldUIVisibleChoices.HIDDEN
+        ]
         extra_columns.extend([
             (f'cf_{cf.name}', columns.CustomFieldColumn(cf)) for cf in custom_fields
         ])

netbox/netbox/views/generic/bulk_views.py

@@ -5,7 +5,6 @@ from copy import deepcopy
 
 from django.contrib import messages
 from django.contrib.contenttypes.fields import GenericForeignKey, GenericRel
-from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError
 from django.db import IntegrityError, router, transaction
 from django.db.models import ManyToManyField, ProtectedError, RestrictedError
@@ -485,10 +484,10 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
                 instance = self.queryset.model()
 
                 # For newly created objects, apply any default custom field values
-                custom_fields = CustomField.objects.filter(
+                custom_fields = [
-                    object_types=ContentType.objects.get_for_model(self.queryset.model),
+                    cf for cf in CustomField.objects.get_for_model(self.queryset.model)
-                    ui_editable=CustomFieldUIEditableChoices.YES
+                    if cf.ui_editable == CustomFieldUIEditableChoices.YES
-                )
+                ]
                 for cf in custom_fields:
                     field_name = f'cf_{cf.name}'
                     if field_name not in record: