Closes #21300: Cache model-specific custom field lookups for the duration of a request (#21334)

netbox/extras/api/customfields.py

@@ -4,7 +4,6 @@ from drf_spectacular.utils import extend_schema_field
 from rest_framework.fields import Field
 from rest_framework.serializers import ValidationError
 
-from core.models import ObjectType
 from extras.choices import CustomFieldTypeChoices
 from extras.constants import CUSTOMFIELD_EMPTY_VALUES
 from extras.models import CustomField
@@ -24,13 +23,9 @@ class CustomFieldDefaultValues:
     def __call__(self, serializer_field):
         self.model = serializer_field.parent.Meta.model
 
-        # Retrieve the CustomFields for the parent model
+        # Populate the default value for each CustomField on the model
-        object_type = ObjectType.objects.get_for_model(self.model)
-        fields = CustomField.objects.filter(object_types=object_type)
-
-        # Populate the default value for each CustomField
         value = {}
-        for field in fields:
+        for field in CustomField.objects.get_for_model(self.model):
             if field.default is not None:
                 value[field.name] = field.default
             else:
@@ -47,8 +42,7 @@ class CustomFieldsDataField(Field):
         Cache CustomFields assigned to this model to avoid redundant database queries
         """
         if not hasattr(self, '_custom_fields'):
-            object_type = ObjectType.objects.get_for_model(self.parent.Meta.model)
+            self._custom_fields = CustomField.objects.get_for_model(self.parent.Meta.model)
-            self._custom_fields = CustomField.objects.filter(object_types=object_type)
         return self._custom_fields
 
     def to_representation(self, obj):

netbox/extras/models/customfields.py

@@ -19,6 +19,7 @@ from django.utils.translation import gettext_lazy as _
 from core.models import ObjectType
 from extras.choices import *
 from extras.data import CHOICE_SETS
+from netbox.context import query_cache
 from netbox.models import ChangeLoggedModel
 from netbox.models.features import CloningMixin, ExportTemplatesMixin
 from netbox.models.mixins import OwnerMixin
@@ -58,8 +59,20 @@ class CustomFieldManager(models.Manager.from_queryset(RestrictedQuerySet)):
         """
         Return all CustomFields assigned to the given model.
         """
+        # Check the request cache before hitting the database
+        cache = query_cache.get()
+        if cache is not None:
+            if custom_fields := cache['custom_fields'].get(model._meta.model):
+                return custom_fields
+
         content_type = ObjectType.objects.get_for_model(model._meta.concrete_model)
-        return self.get_queryset().filter(object_types=content_type)
+        custom_fields = self.get_queryset().filter(object_types=content_type)
+
+        # Populate the request cache to avoid redundant lookups
+        if cache is not None:
+            cache['custom_fields'][model._meta.model] = custom_fields
+
+        return custom_fields
 
     def get_defaults_for_model(self, model):
         """

netbox/netbox/filtersets.py

@@ -305,18 +305,13 @@ class NetBoxModelFilterSet(ChangeLoggedModelFilterSet):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        # Dynamically add a Filter for each CustomField applicable to the parent model
-        custom_fields = CustomField.objects.filter(
-            object_types=ContentType.objects.get_for_model(self._meta.model)
-        ).exclude(
-            filter_logic=CustomFieldFilterLogicChoices.FILTER_DISABLED
-        )
-
         custom_field_filters = {}
-        for custom_field in custom_fields:
+        for custom_field in CustomField.objects.get_for_model(self._meta.model):
+            if custom_field.filter_logic == CustomFieldFilterLogicChoices.FILTER_DISABLED:
+                # Skip disabled fields
+                continue
+            if filter_instance := custom_field.to_filter():
                 filter_name = f'cf_{custom_field.name}'
-            filter_instance = custom_field.to_filter()
-            if filter_instance:
                 custom_field_filters[filter_name] = filter_instance
 
                 # Add relevant additional lookups

netbox/netbox/forms/bulk_import.py

@@ -31,10 +31,11 @@ class NetBoxModelImportForm(CSVModelForm, NetBoxModelForm):
     )
 
     def _get_custom_fields(self, content_type):
-        return CustomField.objects.filter(
+        # Return only custom fields that are editable in the UI
-            object_types=content_type,
+        return [
-            ui_editable=CustomFieldUIEditableChoices.YES
+            cf for cf in CustomField.objects.get_for_model(content_type.model_class())
-        )
+            if cf.ui_editable == CustomFieldUIEditableChoices.YES
+        ]
 
     def _get_form_field(self, customfield):
         return customfield.to_form_field(for_csv_import=True)

netbox/netbox/forms/filtersets.py

@@ -1,5 +1,4 @@
 from django import forms
-from django.db.models import Q
 from django.utils.translation import gettext_lazy as _
 
 from extras.choices import *
@@ -35,10 +34,13 @@ class NetBoxModelFilterSetForm(FilterModifierMixin, CustomFieldsMixin, SavedFilt
     selector_fields = ('filter_id', 'q')
 
     def _get_custom_fields(self, content_type):
-        return super()._get_custom_fields(content_type).exclude(
+        # Return only non-hidden custom fields for which filtering is enabled (excluding JSON fields)
-            Q(filter_logic=CustomFieldFilterLogicChoices.FILTER_DISABLED) |
+        return [
-            Q(type=CustomFieldTypeChoices.TYPE_JSON)
+            cf for cf in super()._get_custom_fields(content_type) if (
+                cf.filter_logic != CustomFieldFilterLogicChoices.FILTER_DISABLED and
+                cf.type != CustomFieldTypeChoices.TYPE_JSON
             )
+        ]
 
     def _get_form_field(self, customfield):
         return customfield.to_form_field(

netbox/netbox/forms/mixins.py

@@ -65,9 +65,11 @@ class CustomFieldsMixin:
         return ObjectType.objects.get_for_model(self.model)
 
     def _get_custom_fields(self, content_type):
-        return CustomField.objects.filter(object_types=content_type).exclude(
+        # Return only custom fields that are not hidden from the UI
-            ui_editable=CustomFieldUIEditableChoices.HIDDEN
+        return [
-        )
+            cf for cf in CustomField.objects.get_for_model(content_type.model_class())
+            if cf.ui_editable != CustomFieldUIEditableChoices.HIDDEN
+        ]
 
     def _get_form_field(self, customfield):
         return customfield.to_form_field()

netbox/netbox/models/features.py

@@ -326,9 +326,11 @@ class CustomFieldsMixin(models.Model):
                 raise ValidationError(_("Missing required custom field '{name}'.").format(name=cf.name))
 
     def save(self, *args, **kwargs):
-        # Populate default values if omitted
+        from extras.models import CustomField
-        for cf in self.custom_fields.filter(default__isnull=False):
+
-            if cf.name not in self.custom_field_data:
+        # Populate default values for custom fields not already present in the object data
+        for cf in CustomField.objects.get_for_model(self):
+            if cf.name not in self.custom_field_data and cf.default is not None:
                 self.custom_field_data[cf.name] = cf.default
 
         super().save(*args, **kwargs)

netbox/netbox/search/backends.py

@@ -187,7 +187,6 @@ class CachedValueSearchBackend(SearchBackend):
         return ret
 
     def cache(self, instances, indexer=None, remove_existing=True):
-        object_type = None
         custom_fields = None
 
         # Convert a single instance to an iterable
@@ -208,15 +207,18 @@ class CachedValueSearchBackend(SearchBackend):
                     except KeyError:
                         break
 
-                # Prefetch any associated custom fields
+                # Prefetch any associated custom fields (excluding those with a zero search weight)
-                object_type = ObjectType.objects.get_for_model(indexer.model)
+                custom_fields = [
-                custom_fields = CustomField.objects.filter(object_types=object_type).exclude(search_weight=0)
+                    cf for cf in CustomField.objects.get_for_model(indexer.model)
+                    if cf.search_weight > 0
+                ]
 
             # Wipe out any previously cached values for the object
             if remove_existing:
                 self.remove(instance)
 
             # Generate cache data
+            object_type = ObjectType.objects.get_for_model(indexer.model)
             for field in indexer.to_cache(instance, custom_fields=custom_fields):
                 buffer.append(
                     CachedValue(

netbox/netbox/tables/tables.py

@@ -242,14 +242,17 @@ class NetBoxTable(BaseTable):
                 (name, deepcopy(column)) for name, column in registered_columns.items()
             ])
 
-        # Add custom field & custom link columns
+        # Add columns for custom fields
-        object_type = ObjectType.objects.get_for_model(self._meta.model)
+        custom_fields = [
-        custom_fields = CustomField.objects.filter(
+            cf for cf in CustomField.objects.get_for_model(self._meta.model)
-            object_types=object_type
+            if cf.ui_visible != CustomFieldUIVisibleChoices.HIDDEN
-        ).exclude(ui_visible=CustomFieldUIVisibleChoices.HIDDEN)
+        ]
         extra_columns.extend([
             (f'cf_{cf.name}', columns.CustomFieldColumn(cf)) for cf in custom_fields
         ])
+
+        # Add columns for custom links
+        object_type = ObjectType.objects.get_for_model(self._meta.model)
         custom_links = CustomLink.objects.filter(object_types=object_type, enabled=True)
         extra_columns.extend([
             (f'cl_{cl.name}', columns.CustomLinkColumn(cl)) for cl in custom_links

netbox/netbox/views/generic/bulk_views.py

@@ -5,7 +5,6 @@ from copy import deepcopy
 
 from django.contrib import messages
 from django.contrib.contenttypes.fields import GenericForeignKey, GenericRel
-from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError
 from django.db import IntegrityError, router, transaction
 from django.db.models import ManyToManyField, ProtectedError, RestrictedError
@@ -484,12 +483,11 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
             else:
                 instance = self.queryset.model()
 
-                # For newly created objects, apply any default custom field values
+                # For newly created objects, apply any default values for custom fields
-                custom_fields = CustomField.objects.filter(
+                for cf in CustomField.objects.get_for_model(self.queryset.model):
-                    object_types=ContentType.objects.get_for_model(self.queryset.model),
+                    if cf.ui_editable != CustomFieldUIEditableChoices.YES:
-                    ui_editable=CustomFieldUIEditableChoices.YES
+                        # Skip custom fields which are not editable via the UI
-                )
+                        continue
-                for cf in custom_fields:
                     field_name = f'cf_{cf.name}'
                     if field_name not in record:
                         record[field_name] = cf.default