diff --git a/docs/configuration/system.md b/docs/configuration/system.md index 93f8fa902..3756b6a83 100644 --- a/docs/configuration/system.md +++ b/docs/configuration/system.md @@ -157,6 +157,14 @@ The file path to the location where [custom scripts](../customization/custom-scr --- +## SEARCH_BACKEND + +Default: `'netbox.search.backends.CachedValueSearchBackend'` + +The dotted path to the desired search backend class. `CachedValueSearchBackend` is currently the only search backend provided in NetBox, however this setting can be used to enable a custom backend. + +--- + ## STORAGE_BACKEND Default: None (local storage) diff --git a/docs/development/search.md b/docs/development/search.md new file mode 100644 index 000000000..02bcaa898 --- /dev/null +++ b/docs/development/search.md @@ -0,0 +1,37 @@ +# Search + +NetBox v3.4 introduced a new global search mechanism, which employs the `extras.CachedValue` model to store discrete field values from many models in a single table. + +## SearchIndex + +To enable search support for a model, declare and register a subclass of `netbox.search.SearchIndex` for it. Typically, this will be done within an app's `search.py` module. + +```python +from netbox.search import SearchIndex, register_search + +@register_search +class MyModelIndex(SearchIndex): + model = MyModel + fields = ( + ('name', 100), + ('description', 500), + ('comments', 5000), + ) +``` + +A SearchIndex subclass defines both its model and a list of two-tuples specifying which model fields to be indexed and the weight (precedence) associated with each. Guidance on weight assignment for fields is provided below. + +### Field Weight Guidance + +| Weight | Field Role | Examples | +|--------|--------------------------------------------------|----------------------------------------------------| +| 50 | Unique serialized attribute | Device.asset_tag | +| 60 | Unique serialized attribute (per related object) | Device.serial | +| 100 | Primary human identifier | Device.name, Circuit.cid, Cable.label | +| 110 | Slug | Site.slug | +| 200 | Secondary identifier | Provider.account, DeviceType.part_number | +| 300 | Highly unique descriptive attribute | CircuitTermination.xconnect_id, IPAddress.dns_name | +| 500 | Description | Site.description | +| 1000 | Custom field default | - | +| 2000 | Other discrete attribute | CircuitTermination.port_speed | +| 5000 | Comment field | Site.comments | diff --git a/docs/plugins/development/search.md b/docs/plugins/development/search.md index 13edd4527..e3b861f00 100644 --- a/docs/plugins/development/search.md +++ b/docs/plugins/development/search.md @@ -4,17 +4,16 @@ Plugins can define and register their own models to extend NetBox's core search ```python # search.py -from netbox.search import SearchMixin -from .filters import MyModelFilterSet -from .tables import MyModelTable +from netbox.search import SearchIndex from .models import MyModel -class MyModelIndex(SearchMixin): +class MyModelIndex(SearchIndex): model = MyModel - queryset = MyModel.objects.all() - filterset = MyModelFilterSet - table = MyModelTable - url = 'plugins:myplugin:mymodel_list' + fields = ( + ('name', 100), + ('description', 500), + ('comments', 5000), + ) ``` To register one or more indexes with NetBox, define a list named `indexes` at the end of this file: diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 8a9c61e22..80b94d6c2 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -11,6 +11,10 @@ ### New Features +#### New Global Search ([#10560](https://github.com/netbox-community/netbox/issues/10560)) + +NetBox's global search functionality has been completely overhauled and replaced by a new cache-based lookup. + #### Top-Level Plugin Navigation Menus ([#9071](https://github.com/netbox-community/netbox/issues/9071)) A new `PluginMenu` class has been introduced, which enables a plugin to inject a top-level menu in NetBox's navigation menu. This menu can have one or more groups of menu items, just like core items. Backward compatibility with the existing `menu_items` has been maintained. diff --git a/mkdocs.yml b/mkdocs.yml index fc3a40632..011d4414f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -245,6 +245,7 @@ nav: - Adding Models: 'development/adding-models.md' - Extending Models: 'development/extending-models.md' - Signals: 'development/signals.md' + - Search: 'development/search.md' - Application Registry: 'development/application-registry.md' - User Preferences: 'development/user-preferences.md' - Web UI: 'development/web-ui.md' diff --git a/netbox/circuits/search.py b/netbox/circuits/search.py index 5adfb97fb..673f6308f 100644 --- a/netbox/circuits/search.py +++ b/netbox/circuits/search.py @@ -1,34 +1,55 @@ -import circuits.filtersets -import circuits.tables -from circuits.models import Circuit, Provider, ProviderNetwork from netbox.search import SearchIndex, register_search -from utilities.utils import count_related +from . import models -@register_search() -class ProviderIndex(SearchIndex): - model = Provider - queryset = Provider.objects.annotate(count_circuits=count_related(Circuit, 'provider')) - filterset = circuits.filtersets.ProviderFilterSet - table = circuits.tables.ProviderTable - url = 'circuits:provider_list' - - -@register_search() +@register_search class CircuitIndex(SearchIndex): - model = Circuit - queryset = Circuit.objects.prefetch_related( - 'type', 'provider', 'tenant', 'tenant__group', 'terminations__site' + model = models.Circuit + fields = ( + ('cid', 100), + ('description', 500), + ('comments', 5000), ) - filterset = circuits.filtersets.CircuitFilterSet - table = circuits.tables.CircuitTable - url = 'circuits:circuit_list' -@register_search() +@register_search +class CircuitTerminationIndex(SearchIndex): + model = models.CircuitTermination + fields = ( + ('xconnect_id', 300), + ('pp_info', 300), + ('description', 500), + ('port_speed', 2000), + ('upstream_speed', 2000), + ) + + +@register_search +class CircuitTypeIndex(SearchIndex): + model = models.CircuitType + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class ProviderIndex(SearchIndex): + model = models.Provider + fields = ( + ('name', 100), + ('account', 200), + ('comments', 5000), + ) + + +@register_search class ProviderNetworkIndex(SearchIndex): - model = ProviderNetwork - queryset = ProviderNetwork.objects.prefetch_related('provider') - filterset = circuits.filtersets.ProviderNetworkFilterSet - table = circuits.tables.ProviderNetworkTable - url = 'circuits:providernetwork_list' + model = models.ProviderNetwork + fields = ( + ('name', 100), + ('service_id', 200), + ('description', 500), + ('comments', 5000), + ) diff --git a/netbox/dcim/search.py b/netbox/dcim/search.py index b179402ce..d34a78888 100644 --- a/netbox/dcim/search.py +++ b/netbox/dcim/search.py @@ -1,143 +1,293 @@ -import dcim.filtersets -import dcim.tables -from dcim.models import ( - Cable, - Device, - DeviceType, - Location, - Module, - ModuleType, - PowerFeed, - Rack, - RackReservation, - Site, - VirtualChassis, -) from netbox.search import SearchIndex, register_search -from utilities.utils import count_related +from . import models -@register_search() -class SiteIndex(SearchIndex): - model = Site - queryset = Site.objects.prefetch_related('region', 'tenant', 'tenant__group') - filterset = dcim.filtersets.SiteFilterSet - table = dcim.tables.SiteTable - url = 'dcim:site_list' - - -@register_search() -class RackIndex(SearchIndex): - model = Rack - queryset = Rack.objects.prefetch_related('site', 'location', 'tenant', 'tenant__group', 'role').annotate( - device_count=count_related(Device, 'rack') - ) - filterset = dcim.filtersets.RackFilterSet - table = dcim.tables.RackTable - url = 'dcim:rack_list' - - -@register_search() -class RackReservationIndex(SearchIndex): - model = RackReservation - queryset = RackReservation.objects.prefetch_related('rack', 'user') - filterset = dcim.filtersets.RackReservationFilterSet - table = dcim.tables.RackReservationTable - url = 'dcim:rackreservation_list' - - -@register_search() -class LocationIndex(SearchIndex): - model = Location - queryset = Location.objects.add_related_count( - Location.objects.add_related_count(Location.objects.all(), Device, 'location', 'device_count', cumulative=True), - Rack, - 'location', - 'rack_count', - cumulative=True, - ).prefetch_related('site') - filterset = dcim.filtersets.LocationFilterSet - table = dcim.tables.LocationTable - url = 'dcim:location_list' - - -@register_search() -class DeviceTypeIndex(SearchIndex): - model = DeviceType - queryset = DeviceType.objects.prefetch_related('manufacturer').annotate( - instance_count=count_related(Device, 'device_type') - ) - filterset = dcim.filtersets.DeviceTypeFilterSet - table = dcim.tables.DeviceTypeTable - url = 'dcim:devicetype_list' - - -@register_search() -class DeviceIndex(SearchIndex): - model = Device - queryset = Device.objects.prefetch_related( - 'device_type__manufacturer', - 'device_role', - 'tenant', - 'tenant__group', - 'site', - 'rack', - 'primary_ip4', - 'primary_ip6', - ) - filterset = dcim.filtersets.DeviceFilterSet - table = dcim.tables.DeviceTable - url = 'dcim:device_list' - - -@register_search() -class ModuleTypeIndex(SearchIndex): - model = ModuleType - queryset = ModuleType.objects.prefetch_related('manufacturer').annotate( - instance_count=count_related(Module, 'module_type') - ) - filterset = dcim.filtersets.ModuleTypeFilterSet - table = dcim.tables.ModuleTypeTable - url = 'dcim:moduletype_list' - - -@register_search() -class ModuleIndex(SearchIndex): - model = Module - queryset = Module.objects.prefetch_related( - 'module_type__manufacturer', - 'device', - 'module_bay', - ) - filterset = dcim.filtersets.ModuleFilterSet - table = dcim.tables.ModuleTable - url = 'dcim:module_list' - - -@register_search() -class VirtualChassisIndex(SearchIndex): - model = VirtualChassis - queryset = VirtualChassis.objects.prefetch_related('master').annotate( - member_count=count_related(Device, 'virtual_chassis') - ) - filterset = dcim.filtersets.VirtualChassisFilterSet - table = dcim.tables.VirtualChassisTable - url = 'dcim:virtualchassis_list' - - -@register_search() +@register_search class CableIndex(SearchIndex): - model = Cable - queryset = Cable.objects.all() - filterset = dcim.filtersets.CableFilterSet - table = dcim.tables.CableTable - url = 'dcim:cable_list' + model = models.Cable + fields = ( + ('label', 100), + ) -@register_search() +@register_search +class ConsolePortIndex(SearchIndex): + model = models.ConsolePort + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ('speed', 2000), + ) + + +@register_search +class ConsoleServerPortIndex(SearchIndex): + model = models.ConsoleServerPort + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ('speed', 2000), + ) + + +@register_search +class DeviceIndex(SearchIndex): + model = models.Device + fields = ( + ('asset_tag', 50), + ('serial', 60), + ('name', 100), + ('comments', 5000), + ) + + +@register_search +class DeviceBayIndex(SearchIndex): + model = models.DeviceBay + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ) + + +@register_search +class DeviceRoleIndex(SearchIndex): + model = models.DeviceRole + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class DeviceTypeIndex(SearchIndex): + model = models.DeviceType + fields = ( + ('model', 100), + ('part_number', 200), + ('comments', 5000), + ) + + +@register_search +class FrontPortIndex(SearchIndex): + model = models.FrontPort + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ) + + +@register_search +class InterfaceIndex(SearchIndex): + model = models.Interface + fields = ( + ('name', 100), + ('label', 200), + ('mac_address', 300), + ('wwn', 300), + ('description', 500), + ('mtu', 2000), + ('speed', 2000), + ) + + +@register_search +class InventoryItemIndex(SearchIndex): + model = models.InventoryItem + fields = ( + ('asset_tag', 50), + ('serial', 60), + ('name', 100), + ('label', 200), + ('description', 500), + ('part_id', 2000), + ) + + +@register_search +class LocationIndex(SearchIndex): + model = models.Location + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class ManufacturerIndex(SearchIndex): + model = models.Manufacturer + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class ModuleIndex(SearchIndex): + model = models.Module + fields = ( + ('asset_tag', 50), + ('serial', 60), + ('comments', 5000), + ) + + +@register_search +class ModuleBayIndex(SearchIndex): + model = models.ModuleBay + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ) + + +@register_search +class ModuleTypeIndex(SearchIndex): + model = models.ModuleType + fields = ( + ('model', 100), + ('part_number', 200), + ('comments', 5000), + ) + + +@register_search +class PlatformIndex(SearchIndex): + model = models.Platform + fields = ( + ('name', 100), + ('slug', 110), + ('napalm_driver', 300), + ('description', 500), + ) + + +@register_search class PowerFeedIndex(SearchIndex): - model = PowerFeed - queryset = PowerFeed.objects.all() - filterset = dcim.filtersets.PowerFeedFilterSet - table = dcim.tables.PowerFeedTable - url = 'dcim:powerfeed_list' + model = models.PowerFeed + fields = ( + ('name', 100), + ('comments', 5000), + ) + + +@register_search +class PowerOutletIndex(SearchIndex): + model = models.PowerOutlet + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ) + + +@register_search +class PowerPanelIndex(SearchIndex): + model = models.PowerPanel + fields = ( + ('name', 100), + ) + + +@register_search +class PowerPortIndex(SearchIndex): + model = models.PowerPort + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ('maximum_draw', 2000), + ('allocated_draw', 2000), + ) + + +@register_search +class RackIndex(SearchIndex): + model = models.Rack + fields = ( + ('asset_tag', 50), + ('serial', 60), + ('name', 100), + ('facility_id', 200), + ('comments', 5000), + ) + + +@register_search +class RackReservationIndex(SearchIndex): + model = models.RackReservation + fields = ( + ('description', 500), + ) + + +@register_search +class RackRoleIndex(SearchIndex): + model = models.RackRole + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class RearPortIndex(SearchIndex): + model = models.RearPort + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ) + + +@register_search +class RegionIndex(SearchIndex): + model = models.Region + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500) + ) + + +@register_search +class SiteIndex(SearchIndex): + model = models.Site + fields = ( + ('name', 100), + ('facility', 100), + ('slug', 110), + ('description', 500), + ('physical_address', 2000), + ('shipping_address', 2000), + ('comments', 5000), + ) + + +@register_search +class SiteGroupIndex(SearchIndex): + model = models.SiteGroup + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500) + ) + + +@register_search +class VirtualChassisIndex(SearchIndex): + model = models.VirtualChassis + fields = ( + ('name', 100), + ('domain', 300) + ) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index b34f5fba3..99f4dd02b 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -92,8 +92,8 @@ class CustomFieldSerializer(ValidatedModelSerializer): model = CustomField fields = [ 'id', 'url', 'display', 'content_types', 'type', 'object_type', 'data_type', 'name', 'label', 'group_name', - 'description', 'required', 'filter_logic', 'ui_visibility', 'default', 'weight', 'validation_minimum', - 'validation_maximum', 'validation_regex', 'choices', 'created', 'last_updated', + 'description', 'required', 'search_weight', 'filter_logic', 'ui_visibility', 'default', 'weight', + 'validation_minimum', 'validation_maximum', 'validation_regex', 'choices', 'created', 'last_updated', ] def get_data_type(self, obj): diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index 8c9c58a13..1b1b049c7 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -73,8 +73,8 @@ class CustomFieldFilterSet(BaseFilterSet): class Meta: model = CustomField fields = [ - 'id', 'content_types', 'name', 'group_name', 'required', 'filter_logic', 'ui_visibility', 'weight', - 'description', + 'id', 'content_types', 'name', 'group_name', 'required', 'search_weight', 'filter_logic', 'ui_visibility', + 'weight', 'description', ] def search(self, queryset, name, value): diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index e83cac3b9..0303dae30 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -46,8 +46,8 @@ class CustomFieldCSVForm(CSVModelForm): class Meta: model = CustomField fields = ( - 'name', 'label', 'group_name', 'type', 'content_types', 'object_type', 'required', 'description', 'weight', - 'filter_logic', 'default', 'choices', 'weight', 'validation_minimum', 'validation_maximum', + 'name', 'label', 'group_name', 'type', 'content_types', 'object_type', 'required', 'description', + 'search_weight', 'filter_logic', 'default', 'choices', 'weight', 'validation_minimum', 'validation_maximum', 'validation_regex', 'ui_visibility', ) diff --git a/netbox/extras/forms/models.py b/netbox/extras/forms/models.py index bea1fbcc1..eca93849b 100644 --- a/netbox/extras/forms/models.py +++ b/netbox/extras/forms/models.py @@ -41,9 +41,9 @@ class CustomFieldForm(BootstrapMixin, forms.ModelForm): fieldsets = ( ('Custom Field', ( - 'content_types', 'name', 'label', 'group_name', 'type', 'object_type', 'weight', 'required', 'description', + 'content_types', 'name', 'label', 'group_name', 'type', 'object_type', 'required', 'description', )), - ('Behavior', ('filter_logic', 'ui_visibility')), + ('Behavior', ('search_weight', 'filter_logic', 'ui_visibility', 'weight')), ('Values', ('default', 'choices')), ('Validation', ('validation_minimum', 'validation_maximum', 'validation_regex')), ) diff --git a/netbox/extras/management/commands/reindex.py b/netbox/extras/management/commands/reindex.py new file mode 100644 index 000000000..6dc9bbb2d --- /dev/null +++ b/netbox/extras/management/commands/reindex.py @@ -0,0 +1,77 @@ +from django.contrib.contenttypes.models import ContentType +from django.core.management.base import BaseCommand, CommandError + +from extras.registry import registry +from netbox.search.backends import search_backend + + +class Command(BaseCommand): + help = 'Reindex objects for search' + + def add_arguments(self, parser): + parser.add_argument( + 'args', + metavar='app_label[.ModelName]', + nargs='*', + help='One or more apps or models to reindex', + ) + + def _get_indexers(self, *model_names): + indexers = {} + + # No models specified; pull in all registered indexers + if not model_names: + for idx in registry['search'].values(): + indexers[idx.model] = idx + + # Return only indexers for the specified models + else: + for label in model_names: + try: + app_label, model_name = label.lower().split('.') + except ValueError: + raise CommandError( + f"Invalid model: {label}. Model names must be in the format .." + ) + try: + idx = registry['search'][f'{app_label}.{model_name}'] + indexers[idx.model] = idx + except KeyError: + raise CommandError(f"No indexer registered for {label}") + + return indexers + + def handle(self, *model_labels, **kwargs): + + # Determine which models to reindex + indexers = self._get_indexers(*model_labels) + if not indexers: + raise CommandError("No indexers found!") + self.stdout.write(f'Reindexing {len(indexers)} models.') + + # Clear all cached values for the specified models + self.stdout.write('Clearing cached values... ', ending='') + self.stdout.flush() + content_types = [ + ContentType.objects.get_for_model(model) for model in indexers.keys() + ] + deleted_count = search_backend.clear(content_types) + self.stdout.write(f'{deleted_count} entries deleted.') + + # Index models + self.stdout.write('Indexing models') + for model, idx in indexers.items(): + app_label = model._meta.app_label + model_name = model._meta.model_name + self.stdout.write(f' {app_label}.{model_name}... ', ending='') + self.stdout.flush() + i = search_backend.cache(model.objects.iterator(), remove_existing=False) + if i: + self.stdout.write(f'{i} entries cached.') + else: + self.stdout.write(f'None found.') + + msg = f'Completed.' + if total_count := search_backend.size: + msg += f' Total entries: {total_count}' + self.stdout.write(msg, self.style.SUCCESS) diff --git a/netbox/extras/migrations/0079_change_jobresult_order.py b/netbox/extras/migrations/0079_change_jobresult_order.py deleted file mode 100644 index 12e35bf67..000000000 --- a/netbox/extras/migrations/0079_change_jobresult_order.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 4.1.1 on 2022-10-09 18:37 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('extras', '0078_unique_constraints'), - ] - - operations = [ - migrations.AlterModelOptions( - name='jobresult', - options={'ordering': ['-created']}, - ), - ] diff --git a/netbox/extras/migrations/0080_add_jobresult_scheduled_time.py b/netbox/extras/migrations/0079_jobresult_scheduled_time.py similarity index 64% rename from netbox/extras/migrations/0080_add_jobresult_scheduled_time.py rename to netbox/extras/migrations/0079_jobresult_scheduled_time.py index fddde4bc5..c9646f13c 100644 --- a/netbox/extras/migrations/0080_add_jobresult_scheduled_time.py +++ b/netbox/extras/migrations/0079_jobresult_scheduled_time.py @@ -1,12 +1,10 @@ -# Generated by Django 4.1.1 on 2022-10-16 09:52 - from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('extras', '0079_change_jobresult_order'), + ('extras', '0078_unique_constraints'), ] operations = [ @@ -15,4 +13,8 @@ class Migration(migrations.Migration): name='scheduled_time', field=models.DateTimeField(blank=True, null=True), ), + migrations.AlterModelOptions( + name='jobresult', + options={'ordering': ['-created']}, + ), ] diff --git a/netbox/extras/migrations/0080_search.py b/netbox/extras/migrations/0080_search.py new file mode 100644 index 000000000..7a133e84b --- /dev/null +++ b/netbox/extras/migrations/0080_search.py @@ -0,0 +1,35 @@ +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('extras', '0079_jobresult_scheduled_time'), + ] + + operations = [ + migrations.AddField( + model_name='customfield', + name='search_weight', + field=models.PositiveSmallIntegerField(default=1000), + ), + migrations.CreateModel( + name='CachedValue', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('object_id', models.PositiveBigIntegerField()), + ('field', models.CharField(max_length=200)), + ('type', models.CharField(max_length=30)), + ('value', models.TextField(db_index=True)), + ('weight', models.PositiveSmallIntegerField(default=1000)), + ('object_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='contenttypes.contenttype')), + ], + options={ + 'ordering': ('weight', 'object_type', 'object_id'), + }, + ), + ] diff --git a/netbox/extras/models/__init__.py b/netbox/extras/models/__init__.py index 3cb6372be..e3a4be3fe 100644 --- a/netbox/extras/models/__init__.py +++ b/netbox/extras/models/__init__.py @@ -2,9 +2,11 @@ from .change_logging import ObjectChange from .configcontexts import ConfigContext, ConfigContextModel from .customfields import CustomField from .models import * +from .search import * from .tags import Tag, TaggedItem __all__ = ( + 'CachedValue', 'ConfigContext', 'ConfigContextModel', 'ConfigRevision', diff --git a/netbox/extras/models/customfields.py b/netbox/extras/models/customfields.py index c3c298a44..2de806ca6 100644 --- a/netbox/extras/models/customfields.py +++ b/netbox/extras/models/customfields.py @@ -16,6 +16,7 @@ from extras.choices import * from extras.utils import FeatureQuery from netbox.models import ChangeLoggedModel from netbox.models.features import CloningMixin, ExportTemplatesMixin, WebhooksMixin +from netbox.search import FieldTypes from utilities import filters from utilities.forms import ( CSVChoiceField, CSVMultipleChoiceField, DatePicker, DynamicModelChoiceField, DynamicModelMultipleChoiceField, @@ -30,6 +31,15 @@ __all__ = ( 'CustomFieldManager', ) +SEARCH_TYPES = { + CustomFieldTypeChoices.TYPE_TEXT: FieldTypes.STRING, + CustomFieldTypeChoices.TYPE_LONGTEXT: FieldTypes.STRING, + CustomFieldTypeChoices.TYPE_INTEGER: FieldTypes.INTEGER, + CustomFieldTypeChoices.TYPE_DECIMAL: FieldTypes.FLOAT, + CustomFieldTypeChoices.TYPE_DATE: FieldTypes.STRING, + CustomFieldTypeChoices.TYPE_URL: FieldTypes.STRING, +} + class CustomFieldManager(models.Manager.from_queryset(RestrictedQuerySet)): use_in_migrations = True @@ -94,6 +104,11 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge help_text='If true, this field is required when creating new objects ' 'or editing an existing object.' ) + search_weight = models.PositiveSmallIntegerField( + default=1000, + help_text='Weighting for search. Lower values are considered more important. ' + 'Fields with a search weight of zero will be ignored.' + ) filter_logic = models.CharField( max_length=50, choices=CustomFieldFilterLogicChoices, @@ -109,6 +124,7 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge ) weight = models.PositiveSmallIntegerField( default=100, + verbose_name='Display weight', help_text='Fields with higher weights appear lower in a form.' ) validation_minimum = models.IntegerField( @@ -148,8 +164,9 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge objects = CustomFieldManager() clone_fields = ( - 'content_types', 'type', 'object_type', 'group_name', 'description', 'required', 'filter_logic', 'default', - 'weight', 'validation_minimum', 'validation_maximum', 'validation_regex', 'choices', 'ui_visibility', + 'content_types', 'type', 'object_type', 'group_name', 'description', 'required', 'search_weight', + 'filter_logic', 'default', 'weight', 'validation_minimum', 'validation_maximum', 'validation_regex', 'choices', + 'ui_visibility', ) class Meta: @@ -167,6 +184,10 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge # Cache instance's original name so we can check later whether it has changed self._name = self.name + @property + def search_type(self): + return SEARCH_TYPES.get(self.type) + def populate_initial_data(self, content_types): """ Populate initial custom field data upon either a) the creation of a new CustomField, or diff --git a/netbox/extras/models/search.py b/netbox/extras/models/search.py new file mode 100644 index 000000000..b7bb104e7 --- /dev/null +++ b/netbox/extras/models/search.py @@ -0,0 +1,50 @@ +import uuid + +from django.contrib.contenttypes.models import ContentType +from django.db import models + +from utilities.fields import RestrictedGenericForeignKey + +__all__ = ( + 'CachedValue', +) + + +class CachedValue(models.Model): + id = models.UUIDField( + primary_key=True, + default=uuid.uuid4, + editable=False + ) + timestamp = models.DateTimeField( + auto_now_add=True, + editable=False + ) + object_type = models.ForeignKey( + to=ContentType, + on_delete=models.CASCADE, + related_name='+' + ) + object_id = models.PositiveBigIntegerField() + object = RestrictedGenericForeignKey( + ct_field='object_type', + fk_field='object_id' + ) + field = models.CharField( + max_length=200 + ) + type = models.CharField( + max_length=30 + ) + value = models.TextField( + db_index=True + ) + weight = models.PositiveSmallIntegerField( + default=1000 + ) + + class Meta: + ordering = ('weight', 'object_type', 'object_id') + + def __str__(self): + return f'{self.object_type} {self.object_id}: {self.field}={self.value}' diff --git a/netbox/extras/plugins/__init__.py b/netbox/extras/plugins/__init__.py index ea88c0f51..f855027e2 100644 --- a/netbox/extras/plugins/__init__.py +++ b/netbox/extras/plugins/__init__.py @@ -75,7 +75,7 @@ class PluginConfig(AppConfig): try: search_indexes = import_string(f"{self.__module__}.{self.search_indexes}") for idx in search_indexes: - register_search()(idx) + register_search(idx) except ImportError: pass diff --git a/netbox/extras/registry.py b/netbox/extras/registry.py index f89499842..76886e791 100644 --- a/netbox/extras/registry.py +++ b/netbox/extras/registry.py @@ -29,5 +29,5 @@ registry['model_features'] = { feature: collections.defaultdict(set) for feature in EXTRAS_FEATURES } registry['denormalized_fields'] = collections.defaultdict(list) -registry['search'] = collections.defaultdict(dict) +registry['search'] = dict() registry['views'] = collections.defaultdict(dict) diff --git a/netbox/extras/search.py b/netbox/extras/search.py index ae6c9e7b9..da4aa1c84 100644 --- a/netbox/extras/search.py +++ b/netbox/extras/search.py @@ -1,14 +1,11 @@ -import extras.filtersets -import extras.tables -from extras.models import JournalEntry from netbox.search import SearchIndex, register_search +from . import models -@register_search() +@register_search class JournalEntryIndex(SearchIndex): - model = JournalEntry - queryset = JournalEntry.objects.prefetch_related('assigned_object', 'created_by') - filterset = extras.filtersets.JournalEntryFilterSet - table = extras.tables.JournalEntryTable - url = 'extras:journalentry_list' + model = models.JournalEntry + fields = ( + ('comments', 5000), + ) category = 'Journal' diff --git a/netbox/extras/tables/tables.py b/netbox/extras/tables/tables.py index 8f365a58b..73d3e98b2 100644 --- a/netbox/extras/tables/tables.py +++ b/netbox/extras/tables/tables.py @@ -34,8 +34,8 @@ class CustomFieldTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = CustomField fields = ( - 'pk', 'id', 'name', 'content_types', 'label', 'type', 'group_name', 'required', 'weight', 'default', - 'description', 'filter_logic', 'ui_visibility', 'choices', 'created', 'last_updated', + 'pk', 'id', 'name', 'content_types', 'label', 'type', 'group_name', 'required', 'default', 'description', + 'search_weight', 'filter_logic', 'ui_visibility', 'weight', 'choices', 'created', 'last_updated', ) default_columns = ('pk', 'name', 'content_types', 'label', 'group_name', 'type', 'required', 'description') diff --git a/netbox/extras/tests/dummy_plugin/search.py b/netbox/extras/tests/dummy_plugin/search.py index 4a1b7e666..4b1c6f10e 100644 --- a/netbox/extras/tests/dummy_plugin/search.py +++ b/netbox/extras/tests/dummy_plugin/search.py @@ -4,8 +4,9 @@ from .models import DummyModel class DummyModelIndex(SearchIndex): model = DummyModel - queryset = DummyModel.objects.all() - url = 'plugins:dummy_plugin:dummy_models' + fields = ( + ('name', 100), + ) indexes = ( diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index 6080ce2e5..c6ba96a82 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -292,6 +292,7 @@ class CustomFieldTest(TestCase): cf = CustomField.objects.create( name='object_field', type=CustomFieldTypeChoices.TYPE_OBJECT, + object_type=ContentType.objects.get_for_model(VLAN), required=False ) cf.content_types.set([self.object_type]) @@ -323,6 +324,7 @@ class CustomFieldTest(TestCase): cf = CustomField.objects.create( name='object_field', type=CustomFieldTypeChoices.TYPE_MULTIOBJECT, + object_type=ContentType.objects.get_for_model(VLAN), required=False ) cf.content_types.set([self.object_type]) diff --git a/netbox/extras/tests/test_views.py b/netbox/extras/tests/test_views.py index 936213cbf..9634038c1 100644 --- a/netbox/extras/tests/test_views.py +++ b/netbox/extras/tests/test_views.py @@ -32,6 +32,7 @@ class CustomFieldTestCase(ViewTestCases.PrimaryObjectViewTestCase): 'label': 'Field X', 'type': 'text', 'content_types': [site_ct.pk], + 'search_weight': 2000, 'filter_logic': CustomFieldFilterLogicChoices.FILTER_EXACT, 'default': None, 'weight': 200, @@ -40,11 +41,11 @@ class CustomFieldTestCase(ViewTestCases.PrimaryObjectViewTestCase): } cls.csv_data = ( - 'name,label,type,content_types,object_type,weight,filter_logic,choices,validation_minimum,validation_maximum,validation_regex,ui_visibility', - 'field4,Field 4,text,dcim.site,,100,exact,,,,[a-z]{3},read-write', - 'field5,Field 5,integer,dcim.site,,100,exact,,1,100,,read-write', - 'field6,Field 6,select,dcim.site,,100,exact,"A,B,C",,,,read-write', - 'field7,Field 7,object,dcim.site,dcim.region,100,exact,,,,,read-write', + 'name,label,type,content_types,object_type,weight,search_weight,filter_logic,choices,validation_minimum,validation_maximum,validation_regex,ui_visibility', + 'field4,Field 4,text,dcim.site,,100,1000,exact,,,,[a-z]{3},read-write', + 'field5,Field 5,integer,dcim.site,,100,2000,exact,,1,100,,read-write', + 'field6,Field 6,select,dcim.site,,100,3000,exact,"A,B,C",,,,read-write', + 'field7,Field 7,object,dcim.site,dcim.region,100,4000,exact,,,,,read-write', ) cls.bulk_edit_data = { diff --git a/netbox/ipam/search.py b/netbox/ipam/search.py index 2f4599321..d1d25da76 100644 --- a/netbox/ipam/search.py +++ b/netbox/ipam/search.py @@ -1,69 +1,139 @@ -import ipam.filtersets -import ipam.tables -from ipam.models import ASN, VLAN, VRF, Aggregate, IPAddress, Prefix, Service +from . import models from netbox.search import SearchIndex, register_search -@register_search() -class VRFIndex(SearchIndex): - model = VRF - queryset = VRF.objects.prefetch_related('tenant', 'tenant__group') - filterset = ipam.filtersets.VRFFilterSet - table = ipam.tables.VRFTable - url = 'ipam:vrf_list' - - -@register_search() +@register_search class AggregateIndex(SearchIndex): - model = Aggregate - queryset = Aggregate.objects.prefetch_related('rir') - filterset = ipam.filtersets.AggregateFilterSet - table = ipam.tables.AggregateTable - url = 'ipam:aggregate_list' - - -@register_search() -class PrefixIndex(SearchIndex): - model = Prefix - queryset = Prefix.objects.prefetch_related( - 'site', 'vrf__tenant', 'tenant', 'tenant__group', 'vlan', 'role' + model = models.Aggregate + fields = ( + ('prefix', 100), + ('description', 500), + ('date_added', 2000), ) - filterset = ipam.filtersets.PrefixFilterSet - table = ipam.tables.PrefixTable - url = 'ipam:prefix_list' -@register_search() -class IPAddressIndex(SearchIndex): - model = IPAddress - queryset = IPAddress.objects.prefetch_related('vrf__tenant', 'tenant', 'tenant__group') - filterset = ipam.filtersets.IPAddressFilterSet - table = ipam.tables.IPAddressTable - url = 'ipam:ipaddress_list' - - -@register_search() -class VLANIndex(SearchIndex): - model = VLAN - queryset = VLAN.objects.prefetch_related('site', 'group', 'tenant', 'tenant__group', 'role') - filterset = ipam.filtersets.VLANFilterSet - table = ipam.tables.VLANTable - url = 'ipam:vlan_list' - - -@register_search() +@register_search class ASNIndex(SearchIndex): - model = ASN - queryset = ASN.objects.prefetch_related('rir', 'tenant', 'tenant__group') - filterset = ipam.filtersets.ASNFilterSet - table = ipam.tables.ASNTable - url = 'ipam:asn_list' + model = models.ASN + fields = ( + ('asn', 100), + ('description', 500), + ) -@register_search() +@register_search +class FHRPGroupIndex(SearchIndex): + model = models.FHRPGroup + fields = ( + ('name', 100), + ('group_id', 2000), + ('description', 500), + ) + + +@register_search +class IPAddressIndex(SearchIndex): + model = models.IPAddress + fields = ( + ('address', 100), + ('dns_name', 300), + ('description', 500), + ) + + +@register_search +class IPRangeIndex(SearchIndex): + model = models.IPRange + fields = ( + ('start_address', 100), + ('end_address', 300), + ('description', 500), + ) + + +@register_search +class L2VPNIndex(SearchIndex): + model = models.L2VPN + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class PrefixIndex(SearchIndex): + model = models.Prefix + fields = ( + ('prefix', 100), + ('description', 500), + ) + + +@register_search +class RIRIndex(SearchIndex): + model = models.RIR + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class RoleIndex(SearchIndex): + model = models.Role + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class RouteTargetIndex(SearchIndex): + model = models.RouteTarget + fields = ( + ('name', 100), + ('description', 500), + ) + + +@register_search class ServiceIndex(SearchIndex): - model = Service - queryset = Service.objects.prefetch_related('device', 'virtual_machine') - filterset = ipam.filtersets.ServiceFilterSet - table = ipam.tables.ServiceTable - url = 'ipam:service_list' + model = models.Service + fields = ( + ('name', 100), + ('description', 500), + ) + + +@register_search +class VLANIndex(SearchIndex): + model = models.VLAN + fields = ( + ('name', 100), + ('vid', 100), + ('description', 500), + ) + + +@register_search +class VLANGroupIndex(SearchIndex): + model = models.VLANGroup + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ('max_vid', 2000), + ) + + +@register_search +class VRFIndex(SearchIndex): + model = models.VRF + fields = ( + ('name', 100), + ('rd', 200), + ('description', 500), + ) diff --git a/netbox/netbox/constants.py b/netbox/netbox/constants.py index 776938a97..c8054b3b0 100644 --- a/netbox/netbox/constants.py +++ b/netbox/netbox/constants.py @@ -1,5 +1,2 @@ # Prefix for nested serializers NESTED_SERIALIZER_PREFIX = 'Nested' - -# Max results per object type -SEARCH_MAX_RESULTS = 15 diff --git a/netbox/netbox/forms/__init__.py b/netbox/netbox/forms/__init__.py index eb1311d98..dd1fb7726 100644 --- a/netbox/netbox/forms/__init__.py +++ b/netbox/netbox/forms/__init__.py @@ -1,38 +1,45 @@ from django import forms +from django.utils.translation import gettext as _ -from netbox.search.backends import default_search_engine -from utilities.forms import BootstrapMixin +from netbox.search import LookupTypes +from netbox.search.backends import search_backend +from utilities.forms import BootstrapMixin, StaticSelect, StaticSelectMultiple from .base import * - -def build_options(choices): - options = [{"label": choices[0][1], "items": []}] - - for label, choices in choices[1:]: - items = [] - - for value, choice_label in choices: - items.append({"label": choice_label, "value": value}) - - options.append({"label": label, "items": items}) - return options +LOOKUP_CHOICES = ( + ('', _('Partial match')), + (LookupTypes.EXACT, _('Exact match')), + (LookupTypes.STARTSWITH, _('Starts with')), + (LookupTypes.ENDSWITH, _('Ends with')), +) class SearchForm(BootstrapMixin, forms.Form): - q = forms.CharField(label='Search') - options = None + q = forms.CharField( + label='Search', + widget=forms.TextInput( + attrs={ + 'hx-get': '', + 'hx-target': '#object_list', + 'hx-trigger': 'keyup[target.value.length >= 3] changed delay:500ms', + } + ) + ) + obj_types = forms.MultipleChoiceField( + choices=[], + required=False, + label='Object type(s)', + widget=StaticSelectMultiple() + ) + lookup = forms.ChoiceField( + choices=LOOKUP_CHOICES, + initial=LookupTypes.PARTIAL, + required=False, + widget=StaticSelect() + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields["obj_type"] = forms.ChoiceField( - choices=default_search_engine.get_search_choices(), - required=False, - label='Type' - ) - def get_options(self): - if not self.options: - self.options = build_options(default_search_engine.get_search_choices()) - - return self.options + self.fields['obj_types'].choices = search_backend.get_object_types() diff --git a/netbox/netbox/search/__init__.py b/netbox/netbox/search/__init__.py index 0664dc6ca..568bf8652 100644 --- a/netbox/netbox/search/__init__.py +++ b/netbox/netbox/search/__init__.py @@ -1,5 +1,24 @@ +from collections import namedtuple + +from django.db import models + from extras.registry import registry +ObjectFieldValue = namedtuple('ObjectFieldValue', ('name', 'type', 'weight', 'value')) + + +class FieldTypes: + FLOAT = 'float' + INTEGER = 'int' + STRING = 'str' + + +class LookupTypes: + PARTIAL = 'icontains' + EXACT = 'iexact' + STARTSWITH = 'istartswith' + ENDSWITH = 'iendswith' + class SearchIndex: """ @@ -7,27 +26,90 @@ class SearchIndex: Attrs: model: The model class for which this index is used. + category: The label of the group under which this indexer is categorized (for form field display). If none, + the name of the model's app will be used. + fields: An iterable of two-tuples defining the model fields to be indexed and the weight associated with each. """ model = None + category = None + fields = () + + @staticmethod + def get_field_type(instance, field_name): + """ + Return the data type of the specified model field. + """ + field_cls = instance._meta.get_field(field_name).__class__ + if issubclass(field_cls, (models.FloatField, models.DecimalField)): + return FieldTypes.FLOAT + if issubclass(field_cls, models.IntegerField): + return FieldTypes.INTEGER + return FieldTypes.STRING + + @staticmethod + def get_field_value(instance, field_name): + """ + Return the value of the specified model field as a string. + """ + return str(getattr(instance, field_name)) @classmethod def get_category(cls): + return cls.category or cls.model._meta.app_config.verbose_name + + @classmethod + def to_cache(cls, instance, custom_fields=None): """ - Return the title of the search category under which this model is registered. + Return a list of ObjectFieldValue representing the instance fields to be cached. + + Args: + instance: The instance being cached. + custom_fields: An iterable of CustomFields to include when caching the instance. If None, all custom fields + defined for the model will be included. (This can also be provided during bulk caching to avoid looking + up the available custom fields for each instance.) """ - if hasattr(cls, 'category'): - return cls.category - return cls.model._meta.app_config.verbose_name + values = [] + + # Capture built-in fields + for name, weight in cls.fields: + type_ = cls.get_field_type(instance, name) + value = cls.get_field_value(instance, name) + if type_ and value: + values.append( + ObjectFieldValue(name, type_, weight, value) + ) + + # Capture custom fields + if getattr(instance, 'custom_field_data', None): + if custom_fields is None: + custom_fields = instance.get_custom_fields().keys() + for cf in custom_fields: + type_ = cf.search_type + value = instance.custom_field_data.get(cf.name) + weight = cf.search_weight + if type_ and value and weight: + values.append( + ObjectFieldValue(f'cf_{cf.name}', type_, weight, value) + ) + + return values -def register_search(): - def _wrapper(cls): - model = cls.model - app_label = model._meta.app_label - model_name = model._meta.model_name +def get_indexer(model): + """ + Get the SearchIndex class for the given model. + """ + label = f'{model._meta.app_label}.{model._meta.model_name}' - registry['search'][app_label][model_name] = cls + return registry['search'][label] - return cls - return _wrapper +def register_search(cls): + """ + Decorator for registering a SearchIndex class. + """ + model = cls.model + label = f'{model._meta.app_label}.{model._meta.model_name}' + registry['search'][label] = cls + + return cls diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index b6cead5bd..f1e00b86b 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -1,125 +1,236 @@ from collections import defaultdict -from importlib import import_module from django.conf import settings +from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ImproperlyConfigured -from django.urls import reverse +from django.db.models import F, Window +from django.db.models.functions import window +from django.db.models.signals import post_delete, post_save +from django.utils.module_loading import import_string +from extras.models import CachedValue, CustomField from extras.registry import registry -from netbox.constants import SEARCH_MAX_RESULTS +from utilities.querysets import RestrictedPrefetch +from utilities.templatetags.builtins.filters import bettertitle +from . import FieldTypes, LookupTypes, get_indexer -# The cache for the initialized backend. -_backends_cache = {} - - -class SearchEngineError(Exception): - """Something went wrong with a search engine.""" - pass +DEFAULT_LOOKUP_TYPE = LookupTypes.PARTIAL +MAX_RESULTS = 1000 class SearchBackend: - """A search engine capable of performing multi-table searches.""" - _search_choice_options = tuple() + """ + Base class for search backends. Subclasses must extend the `cache()`, `remove()`, and `clear()` methods below. + """ + _object_types = None - def get_registry(self): - r = {} - for app_label, models in registry['search'].items(): - r.update(**models) - - return r - - def get_search_choices(self): - """Return the set of choices for individual object types, organized by category.""" - if not self._search_choice_options: + def get_object_types(self): + """ + Return a list of all registered object types, organized by category, suitable for populating a form's + ChoiceField. + """ + if not self._object_types: # Organize choices by category categories = defaultdict(dict) - for app_label, models in registry['search'].items(): - for name, cls in models.items(): - title = cls.model._meta.verbose_name.title() - categories[cls.get_category()][name] = title + for label, idx in registry['search'].items(): + title = bettertitle(idx.model._meta.verbose_name) + categories[idx.get_category()][label] = title # Compile a nested tuple of choices for form rendering results = ( ('', 'All Objects'), - *[(category, choices.items()) for category, choices in categories.items()] + *[(category, list(choices.items())) for category, choices in categories.items()] ) - self._search_choice_options = results + self._object_types = results - return self._search_choice_options + return self._object_types - def search(self, request, value, **kwargs): - """Execute a search query for the given value.""" + def search(self, value, user=None, object_types=None, lookup=DEFAULT_LOOKUP_TYPE): + """ + Search cached object representations for the given value. + """ raise NotImplementedError - def cache(self, instance): - """Create or update the cached copy of an instance.""" + def caching_handler(self, sender, instance, **kwargs): + """ + Receiver for the post_save signal, responsible for caching object creation/changes. + """ + self.cache(instance) + + def removal_handler(self, sender, instance, **kwargs): + """ + Receiver for the post_delete signal, responsible for caching object deletion. + """ + self.remove(instance) + + def cache(self, instances, indexer=None, remove_existing=True): + """ + Create or update the cached representation of an instance. + """ raise NotImplementedError + def remove(self, instance): + """ + Delete any cached representation of an instance. + """ + raise NotImplementedError -class FilterSetSearchBackend(SearchBackend): - """ - Legacy search backend. Performs a discrete database query for each registered object type, using the FilterSet - class specified by the index for each. - """ - def search(self, request, value, **kwargs): - results = [] + def clear(self, object_types=None): + """ + Delete *all* cached data. + """ + raise NotImplementedError - search_registry = self.get_registry() - for obj_type in search_registry.keys(): + @property + def size(self): + """ + Return a total number of cached entries. The meaning of this value will be + backend-dependent. + """ + return None - queryset = search_registry[obj_type].queryset - url = search_registry[obj_type].url - # Restrict the queryset for the current user - if hasattr(queryset, 'restrict'): - queryset = queryset.restrict(request.user, 'view') +class CachedValueSearchBackend(SearchBackend): - filterset = getattr(search_registry[obj_type], 'filterset', None) - if not filterset: - # This backend requires a FilterSet class for the model - continue + def search(self, value, user=None, object_types=None, lookup=DEFAULT_LOOKUP_TYPE): - table = getattr(search_registry[obj_type], 'table', None) - if not table: - # This backend requires a Table class for the model - continue + # Define the search parameters + params = { + f'value__{lookup}': value + } + if lookup != LookupTypes.EXACT: + # Partial matches are valid only on string values + params['type'] = FieldTypes.STRING + if object_types: + params['object_type__in'] = object_types - # Construct the results table for this object type - filtered_queryset = filterset({'q': value}, queryset=queryset).qs - table = table(filtered_queryset, orderable=False) - table.paginate(per_page=SEARCH_MAX_RESULTS) + # Construct the base queryset to retrieve matching results + queryset = CachedValue.objects.filter(**params).annotate( + # Annotate the rank of each result for its object according to its weight + row_number=Window( + expression=window.RowNumber(), + partition_by=[F('object_type'), F('object_id')], + order_by=[F('weight').asc()], + ) + )[:MAX_RESULTS] - if table.page: - results.append({ - 'name': queryset.model._meta.verbose_name_plural, - 'table': table, - 'url': f"{reverse(url)}?q={value}" - }) + # Construct a Prefetch to pre-fetch only those related objects for which the + # user has permission to view. + if user: + prefetch = (RestrictedPrefetch('object', user, 'view'), 'object_type') + else: + prefetch = ('object', 'object_type') - return results + # Wrap the base query to return only the lowest-weight result for each object + # Hat-tip to https://blog.oyam.dev/django-filter-by-window-function/ for the solution + sql, params = queryset.query.sql_with_params() + results = CachedValue.objects.prefetch_related(*prefetch).raw( + f"SELECT * FROM ({sql}) t WHERE row_number = 1", + params + ) - def cache(self, instance): - # This backend does not utilize a cache - pass + # Omit any results pertaining to an object the user does not have permission to view + return [ + r for r in results if r.object is not None + ] + + def cache(self, instances, indexer=None, remove_existing=True): + content_type = None + custom_fields = None + + # Convert a single instance to an iterable + if not hasattr(instances, '__iter__'): + instances = [instances] + + buffer = [] + counter = 0 + for instance in instances: + + # First item + if not counter: + + # Determine the indexer + if indexer is None: + try: + indexer = get_indexer(instance) + except KeyError: + break + + # Prefetch any associated custom fields + content_type = ContentType.objects.get_for_model(indexer.model) + custom_fields = CustomField.objects.filter(content_types=content_type).exclude(search_weight=0) + + # Wipe out any previously cached values for the object + if remove_existing: + self.remove(instance) + + # Generate cache data + for field in indexer.to_cache(instance, custom_fields=custom_fields): + buffer.append( + CachedValue( + object_type=content_type, + object_id=instance.pk, + field=field.name, + type=field.type, + weight=field.weight, + value=field.value + ) + ) + + # Check whether the buffer needs to be flushed + if len(buffer) >= 2000: + counter += len(CachedValue.objects.bulk_create(buffer)) + buffer = [] + + # Final buffer flush + if buffer: + counter += len(CachedValue.objects.bulk_create(buffer)) + + return counter + + def remove(self, instance): + # Avoid attempting to query for non-cacheable objects + try: + get_indexer(instance) + except KeyError: + return + + ct = ContentType.objects.get_for_model(instance) + qs = CachedValue.objects.filter(object_type=ct, object_id=instance.pk) + + # Call _raw_delete() on the queryset to avoid first loading instances into memory + return qs._raw_delete(using=qs.db) + + def clear(self, object_types=None): + qs = CachedValue.objects.all() + if object_types: + qs = qs.filter(object_type__in=object_types) + + # Call _raw_delete() on the queryset to avoid first loading instances into memory + return qs._raw_delete(using=qs.db) + + @property + def size(self): + return CachedValue.objects.count() def get_backend(): - """Initializes and returns the configured search backend.""" - backend_name = settings.SEARCH_BACKEND - - # Load the backend class - backend_module_name, backend_cls_name = backend_name.rsplit('.', 1) - backend_module = import_module(backend_module_name) + """ + Initializes and returns the configured search backend. + """ try: - backend_cls = getattr(backend_module, backend_cls_name) + backend_cls = import_string(settings.SEARCH_BACKEND) except AttributeError: - raise ImproperlyConfigured(f"Could not find a class named {backend_module_name} in {backend_cls_name}") + raise ImproperlyConfigured(f"Failed to import configured SEARCH_BACKEND: {settings.SEARCH_BACKEND}") # Initialize and return the backend instance return backend_cls() -default_search_engine = get_backend() -search = default_search_engine.search +search_backend = get_backend() + +# Connect handlers to the appropriate model signals +post_save.connect(search_backend.caching_handler) +post_delete.connect(search_backend.removal_handler) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 980d3dc7e..b1b285283 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -116,7 +116,7 @@ REMOTE_AUTH_GROUP_SEPARATOR = getattr(configuration, 'REMOTE_AUTH_GROUP_SEPARATO REPORTS_ROOT = getattr(configuration, 'REPORTS_ROOT', os.path.join(BASE_DIR, 'reports')).rstrip('/') RQ_DEFAULT_TIMEOUT = getattr(configuration, 'RQ_DEFAULT_TIMEOUT', 300) SCRIPTS_ROOT = getattr(configuration, 'SCRIPTS_ROOT', os.path.join(BASE_DIR, 'scripts')).rstrip('/') -SEARCH_BACKEND = getattr(configuration, 'SEARCH_BACKEND', 'netbox.search.backends.FilterSetSearchBackend') +SEARCH_BACKEND = getattr(configuration, 'SEARCH_BACKEND', 'netbox.search.backends.CachedValueSearchBackend') SENTRY_DSN = getattr(configuration, 'SENTRY_DSN', DEFAULT_SENTRY_DSN) SENTRY_ENABLED = getattr(configuration, 'SENTRY_ENABLED', False) SENTRY_SAMPLE_RATE = getattr(configuration, 'SENTRY_SAMPLE_RATE', 1.0) diff --git a/netbox/netbox/tables/tables.py b/netbox/netbox/tables/tables.py index 38399b5fe..9b86b2ed3 100644 --- a/netbox/netbox/tables/tables.py +++ b/netbox/netbox/tables/tables.py @@ -4,16 +4,21 @@ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.core.exceptions import FieldDoesNotExist from django.db.models.fields.related import RelatedField +from django.utils.safestring import mark_safe +from django.utils.translation import gettext as _ from django_tables2.data import TableQuerysetData from extras.models import CustomField, CustomLink from extras.choices import CustomFieldVisibilityChoices from netbox.tables import columns from utilities.paginator import EnhancedPaginator, get_paginate_count +from utilities.templatetags.builtins.filters import bettertitle +from utilities.utils import highlight_string __all__ = ( 'BaseTable', 'NetBoxTable', + 'SearchTable', ) @@ -192,3 +197,39 @@ class NetBoxTable(BaseTable): ]) super().__init__(*args, extra_columns=extra_columns, **kwargs) + + +class SearchTable(tables.Table): + object_type = columns.ContentTypeColumn( + verbose_name=_('Type') + ) + object = tables.Column( + linkify=True + ) + field = tables.Column() + value = tables.Column() + + trim_length = 30 + + class Meta: + attrs = { + 'class': 'table table-hover object-list', + } + empty_text = _('No results found') + + def __init__(self, data, highlight=None, **kwargs): + self.highlight = highlight + super().__init__(data, **kwargs) + + def render_field(self, value, record): + if hasattr(record.object, value): + return bettertitle(record.object._meta.get_field(value).verbose_name) + return value + + def render_value(self, value): + if not self.highlight: + return value + + value = highlight_string(value, self.highlight, trim_pre=self.trim_length, trim_post=self.trim_length) + + return mark_safe(value) diff --git a/netbox/netbox/tests/test_search.py b/netbox/netbox/tests/test_search.py new file mode 100644 index 000000000..1b6fe9eac --- /dev/null +++ b/netbox/netbox/tests/test_search.py @@ -0,0 +1,153 @@ +from django.contrib.contenttypes.models import ContentType +from django.test import TestCase + +from dcim.models import Site +from dcim.search import SiteIndex +from extras.models import CachedValue +from netbox.search.backends import search_backend + + +class SearchBackendTestCase(TestCase): + + @classmethod + def setUpTestData(cls): + # Create sites with a value for each cacheable field defined on SiteIndex + sites = ( + Site( + name='Site 1', + slug='site-1', + facility='Alpha', + description='First test site', + physical_address='123 Fake St Lincoln NE 68588', + shipping_address='123 Fake St Lincoln NE 68588', + comments='Lorem ipsum etcetera' + ), + Site( + name='Site 2', + slug='site-2', + facility='Bravo', + description='Second test site', + physical_address='725 Cyrus Valleys Suite 761 Douglasfort NE 57761', + shipping_address='725 Cyrus Valleys Suite 761 Douglasfort NE 57761', + comments='Lorem ipsum etcetera' + ), + Site( + name='Site 3', + slug='site-3', + facility='Charlie', + description='Third test site', + physical_address='2321 Dovie Dale East Cristobal AK 71959', + shipping_address='2321 Dovie Dale East Cristobal AK 71959', + comments='Lorem ipsum etcetera' + ), + ) + Site.objects.bulk_create(sites) + + def test_cache_single_object(self): + """ + Test that a single object is cached appropriately + """ + site = Site.objects.first() + search_backend.cache(site) + + content_type = ContentType.objects.get_for_model(Site) + self.assertEqual( + CachedValue.objects.filter(object_type=content_type, object_id=site.pk).count(), + len(SiteIndex.fields) + ) + for field_name, weight in SiteIndex.fields: + self.assertTrue( + CachedValue.objects.filter( + object_type=content_type, + object_id=site.pk, + field=field_name, + value=getattr(site, field_name), + weight=weight + ), + ) + + def test_cache_multiple_objects(self): + """ + Test that multiples objects are cached appropriately + """ + sites = Site.objects.all() + search_backend.cache(sites) + + content_type = ContentType.objects.get_for_model(Site) + self.assertEqual( + CachedValue.objects.filter(object_type=content_type).count(), + len(SiteIndex.fields) * sites.count() + ) + for site in sites: + for field_name, weight in SiteIndex.fields: + self.assertTrue( + CachedValue.objects.filter( + object_type=content_type, + object_id=site.pk, + field=field_name, + value=getattr(site, field_name), + weight=weight + ), + ) + + def test_cache_on_save(self): + """ + Test that an object is automatically cached on calling save(). + """ + site = Site( + name='Site 4', + slug='site-4', + facility='Delta', + description='Fourth test site', + physical_address='7915 Lilla Plains West Ladariusport TX 19429', + shipping_address='7915 Lilla Plains West Ladariusport TX 19429', + comments='Lorem ipsum etcetera' + ) + site.save() + + content_type = ContentType.objects.get_for_model(Site) + self.assertEqual( + CachedValue.objects.filter(object_type=content_type, object_id=site.pk).count(), + len(SiteIndex.fields) + ) + + def test_remove_on_delete(self): + """ + Test that any cached value for an object are automatically removed on delete(). + """ + site = Site.objects.first() + site.delete() + + content_type = ContentType.objects.get_for_model(Site) + self.assertFalse( + CachedValue.objects.filter(object_type=content_type, object_id=site.pk).exists() + ) + + def test_clear_all(self): + """ + Test that calling clear() on the backend removes all cached entries. + """ + sites = Site.objects.all() + search_backend.cache(sites) + self.assertTrue( + CachedValue.objects.exists() + ) + + search_backend.clear() + self.assertFalse( + CachedValue.objects.exists() + ) + + def test_search(self): + """ + Test various searches. + """ + sites = Site.objects.all() + search_backend.cache(sites) + + results = search_backend.search('site') + self.assertEqual(len(results), 3) + results = search_backend.search('first') + self.assertEqual(len(results), 1) + results = search_backend.search('xxxxx') + self.assertEqual(len(results), 0) diff --git a/netbox/netbox/views/__init__.py b/netbox/netbox/views/__init__.py index d880ba64c..0f35dab49 100644 --- a/netbox/netbox/views/__init__.py +++ b/netbox/netbox/views/__init__.py @@ -2,15 +2,16 @@ import platform import sys from django.conf import settings +from django.contrib.contenttypes.models import ContentType from django.core.cache import cache from django.http import HttpResponseServerError from django.shortcuts import redirect, render from django.template import loader from django.template.exceptions import TemplateDoesNotExist -from django.urls import reverse from django.views.decorators.csrf import requires_csrf_token from django.views.defaults import ERROR_500_TEMPLATE_NAME, page_not_found from django.views.generic import View +from django_tables2 import RequestConfig from packaging import version from sentry_sdk import capture_message @@ -21,10 +22,13 @@ from dcim.models import ( from extras.models import ObjectChange from extras.tables import ObjectChangeTable from ipam.models import Aggregate, IPAddress, IPRange, Prefix, VLAN, VRF -from netbox.constants import SEARCH_MAX_RESULTS from netbox.forms import SearchForm -from netbox.search.backends import default_search_engine +from netbox.search import LookupTypes +from netbox.search.backends import search_backend +from netbox.tables import SearchTable from tenancy.models import Tenant +from utilities.htmx import is_htmx +from utilities.paginator import EnhancedPaginator, get_paginate_count from virtualization.models import Cluster, VirtualMachine from wireless.models import WirelessLAN, WirelessLink @@ -149,22 +153,48 @@ class HomeView(View): class SearchView(View): def get(self, request): - form = SearchForm(request.GET) results = [] + highlight = None + + # Initialize search form + form = SearchForm(request.GET) if 'q' in request.GET else SearchForm() if form.is_valid(): - search_registry = default_search_engine.get_registry() - # If an object type has been specified, redirect to the dedicated view for it - if form.cleaned_data['obj_type']: - object_type = form.cleaned_data['obj_type'] - url = reverse(search_registry[object_type].url) - return redirect(f"{url}?q={form.cleaned_data['q']}") - results = default_search_engine.search(request, form.cleaned_data['q']) + # Restrict results by object type + object_types = [] + for obj_type in form.cleaned_data['obj_types']: + app_label, model_name = obj_type.split('.') + object_types.append(ContentType.objects.get_by_natural_key(app_label, model_name)) + + lookup = form.cleaned_data['lookup'] or LookupTypes.PARTIAL + results = search_backend.search( + form.cleaned_data['q'], + user=request.user, + object_types=object_types, + lookup=lookup + ) + + if form.cleaned_data['lookup'] != LookupTypes.EXACT: + highlight = form.cleaned_data['q'] + + table = SearchTable(results, highlight=highlight) + + # Paginate the table results + RequestConfig(request, { + 'paginator_class': EnhancedPaginator, + 'per_page': get_paginate_count(request) + }).configure(table) + + # If this is an HTMX request, return only the rendered table HTML + if is_htmx(request): + return render(request, 'htmx/table.html', { + 'table': table, + }) return render(request, 'search.html', { 'form': form, - 'results': results, + 'table': table, }) diff --git a/netbox/project-static/dist/netbox.js b/netbox/project-static/dist/netbox.js index 1213d719f..19cdae0bd 100644 Binary files a/netbox/project-static/dist/netbox.js and b/netbox/project-static/dist/netbox.js differ diff --git a/netbox/project-static/dist/netbox.js.map b/netbox/project-static/dist/netbox.js.map index 90b87a262..d0563b9fc 100644 Binary files a/netbox/project-static/dist/netbox.js.map and b/netbox/project-static/dist/netbox.js.map differ diff --git a/netbox/project-static/src/netbox.ts b/netbox/project-static/src/netbox.ts index d711150ed..f19b879fe 100644 --- a/netbox/project-static/src/netbox.ts +++ b/netbox/project-static/src/netbox.ts @@ -1,6 +1,6 @@ import { initForms } from './forms'; import { initBootstrap } from './bs'; -import { initSearch } from './search'; +import { initQuickSearch } from './search'; import { initSelect } from './select'; import { initButtons } from './buttons'; import { initColorMode } from './colorMode'; @@ -20,7 +20,7 @@ function initDocument(): void { initColorMode, initMessages, initForms, - initSearch, + initQuickSearch, initSelect, initDateSelector, initButtons, diff --git a/netbox/project-static/src/search.ts b/netbox/project-static/src/search.ts index 97fe1826a..e3bdc18dc 100644 --- a/netbox/project-static/src/search.ts +++ b/netbox/project-static/src/search.ts @@ -1,31 +1,4 @@ -import { getElements, findFirstAdjacent, isTruthy } from './util'; - -/** - * Change the display value and hidden input values of the search filter based on dropdown - * selection. - * - * @param event "click" event for each dropdown item. - * @param button Each dropdown item element. - */ -function handleSearchDropdownClick(event: Event, button: HTMLButtonElement): void { - const dropdown = event.currentTarget as HTMLButtonElement; - const selectedValue = findFirstAdjacent(dropdown, 'span.search-obj-selected'); - const selectedType = findFirstAdjacent(dropdown, 'input.search-obj-type'); - const searchValue = dropdown.getAttribute('data-search-value'); - let selected = '' as string; - - if (selectedValue !== null && selectedType !== null) { - if (isTruthy(searchValue) && selected !== searchValue) { - selected = searchValue; - selectedValue.innerHTML = button.textContent ?? 'Error'; - selectedType.value = searchValue; - } else { - selected = ''; - selectedValue.innerHTML = 'All Objects'; - selectedType.value = ''; - } - } -} +import { isTruthy } from './util'; /** * Show/hide quicksearch clear button. @@ -44,23 +17,10 @@ function quickSearchEventHandler(event: Event): void { } } -/** - * Initialize Search Bar Elements. - */ -function initSearchBar(): void { - for (const dropdown of getElements('.search-obj-selector')) { - for (const button of dropdown.querySelectorAll( - 'li > button.dropdown-item', - )) { - button.addEventListener('click', event => handleSearchDropdownClick(event, button)); - } - } -} - /** * Initialize Quicksearch Event listener/handlers. */ -function initQuickSearch(): void { +export function initQuickSearch(): void { const quicksearch = document.getElementById("quicksearch") as HTMLInputElement; const clearbtn = document.getElementById("quicksearch_clear") as HTMLButtonElement; if (isTruthy(quicksearch)) { @@ -82,10 +42,3 @@ function initQuickSearch(): void { } } } - -export function initSearch(): void { - for (const func of [initSearchBar]) { - func(); - } - initQuickSearch(); -} diff --git a/netbox/templates/base/layout.html b/netbox/templates/base/layout.html index dd0412eac..2835e1ef2 100644 --- a/netbox/templates/base/layout.html +++ b/netbox/templates/base/layout.html @@ -1,7 +1,6 @@ {# Base layout for the core NetBox UI w/navbar and page content #} {% extends 'base/base.html' %} {% load helpers %} -{% load search %} {% load static %} {% comment %} @@ -41,7 +40,7 @@ Blocks:
- {% search_options request %} + {% include 'inc/searchbar.html' %}
@@ -53,7 +52,7 @@ Blocks: {# Search bar #}
- {% search_options request %} + {% include 'inc/searchbar.html' %}
{# Proflie/login button #} diff --git a/netbox/templates/extras/customfield.html b/netbox/templates/extras/customfield.html index ff4e6e08c..4350bb738 100644 --- a/netbox/templates/extras/customfield.html +++ b/netbox/templates/extras/customfield.html @@ -39,13 +39,23 @@ {% checkmark object.required %} - Weight - {{ object.weight }} + Search Weight + + {% if object.search_weight %} + {{ object.search_weight }} + {% else %} + Disabled + {% endif %} + Filter Logic {{ object.get_filter_logic_display }} + + Display Weight + {{ object.weight }} + UI Visibility {{ object.get_ui_visibility_display }} diff --git a/netbox/templates/inc/searchbar.html b/netbox/templates/inc/searchbar.html new file mode 100644 index 000000000..c8ef0d548 --- /dev/null +++ b/netbox/templates/inc/searchbar.html @@ -0,0 +1,6 @@ +
+ + +
diff --git a/netbox/templates/search.html b/netbox/templates/search.html index a47b48b09..e801422c9 100644 --- a/netbox/templates/search.html +++ b/netbox/templates/search.html @@ -15,74 +15,24 @@ {% endblock tabs %} -{% block content-wrapper %} -
- {% if request.GET.q %} - {% if results %} -
-
- {% for obj_type in results %} -
-
{{ obj_type.name|bettertitle }}
-
- {% render_table obj_type.table 'inc/table.html' %} -
-
- - - {% if obj_type.table.page.has_next %} - See All {{ obj_type.table.page.paginator.count }} Results - {% else %} - Refine Search - {% endif %} - -
-
- {% endfor %} -
-
-
-
- Search Results -
-
-
- {% for obj_type in results %} - -
- {% badge obj_type.table.page.paginator.count %} -
- {{ obj_type.name|bettertitle }} - - {% endfor %} -
-
-
-
-
- {% else %} -

No results found

- {% endif %} - {% else %} -
-
-
-
-
- Search -
-
- {% render_form form %} -
-
- -
-
-
-
+{% block content %} +
+
+
+ {% render_form form %} +
+
- {% endif %} +
+
-{% endblock content-wrapper %} +
+
+
+ {% include 'htmx/table.html' %} +
+
+
+{% endblock content %} diff --git a/netbox/tenancy/search.py b/netbox/tenancy/search.py index e52b1859e..8cb3c4ccb 100644 --- a/netbox/tenancy/search.py +++ b/netbox/tenancy/search.py @@ -1,25 +1,57 @@ -import tenancy.filtersets -import tenancy.tables from netbox.search import SearchIndex, register_search -from tenancy.models import Contact, ContactAssignment, Tenant -from utilities.utils import count_related +from . import models -@register_search() -class TenantIndex(SearchIndex): - model = Tenant - queryset = Tenant.objects.prefetch_related('group') - filterset = tenancy.filtersets.TenantFilterSet - table = tenancy.tables.TenantTable - url = 'tenancy:tenant_list' - - -@register_search() +@register_search class ContactIndex(SearchIndex): - model = Contact - queryset = Contact.objects.prefetch_related('group', 'assignments').annotate( - assignment_count=count_related(ContactAssignment, 'contact') + model = models.Contact + fields = ( + ('name', 100), + ('title', 300), + ('phone', 300), + ('email', 300), + ('address', 300), + ('link', 300), + ('comments', 5000), + ) + + +@register_search +class ContactGroupIndex(SearchIndex): + model = models.ContactGroup + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class ContactRoleIndex(SearchIndex): + model = models.ContactRole + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class TenantIndex(SearchIndex): + model = models.Tenant + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ('comments', 5000), + ) + + +@register_search +class TenantGroupIndex(SearchIndex): + model = models.TenantGroup + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), ) - filterset = tenancy.filtersets.ContactFilterSet - table = tenancy.tables.ContactTable - url = 'tenancy:contact_list' diff --git a/netbox/utilities/fields.py b/netbox/utilities/fields.py index a9b851def..b2bc4d2cd 100644 --- a/netbox/utilities/fields.py +++ b/netbox/utilities/fields.py @@ -1,3 +1,6 @@ +from collections import defaultdict + +from django.contrib.contenttypes.fields import GenericForeignKey from django.core.validators import RegexValidator from django.db import models @@ -71,3 +74,70 @@ class NaturalOrderingField(models.CharField): [self.target_field], kwargs, ) + + +class RestrictedGenericForeignKey(GenericForeignKey): + + # Replicated largely from GenericForeignKey. Changes include: + # 1. Capture restrict_params from RestrictedPrefetch (hack) + # 2. If restrict_params is set, call restrict() on the queryset for + # the related model + def get_prefetch_queryset(self, instances, queryset=None): + restrict_params = {} + + # Compensate for the hack in RestrictedPrefetch + if type(queryset) is dict: + restrict_params = queryset + elif queryset is not None: + raise ValueError("Custom queryset can't be used for this lookup.") + + # For efficiency, group the instances by content type and then do one + # query per model + fk_dict = defaultdict(set) + # We need one instance for each group in order to get the right db: + instance_dict = {} + ct_attname = self.model._meta.get_field(self.ct_field).get_attname() + for instance in instances: + # We avoid looking for values if either ct_id or fkey value is None + ct_id = getattr(instance, ct_attname) + if ct_id is not None: + fk_val = getattr(instance, self.fk_field) + if fk_val is not None: + fk_dict[ct_id].add(fk_val) + instance_dict[ct_id] = instance + + ret_val = [] + for ct_id, fkeys in fk_dict.items(): + instance = instance_dict[ct_id] + ct = self.get_content_type(id=ct_id, using=instance._state.db) + if restrict_params: + # Override the default behavior to call restrict() on each model's queryset + qs = ct.model_class().objects.filter(pk__in=fkeys).restrict(**restrict_params) + ret_val.extend(qs) + else: + # Default behavior + ret_val.extend(ct.get_all_objects_for_this_type(pk__in=fkeys)) + + # For doing the join in Python, we have to match both the FK val and the + # content type, so we use a callable that returns a (fk, class) pair. + def gfk_key(obj): + ct_id = getattr(obj, ct_attname) + if ct_id is None: + return None + else: + model = self.get_content_type( + id=ct_id, using=obj._state.db + ).model_class() + return ( + model._meta.pk.get_prep_value(getattr(obj, self.fk_field)), + model, + ) + + return ( + ret_val, + lambda obj: (obj.pk, obj.__class__), + gfk_key, + True, + self.name, + False, + ) diff --git a/netbox/utilities/querysets.py b/netbox/utilities/querysets.py index 955a10d64..0e5f1cd5c 100644 --- a/netbox/utilities/querysets.py +++ b/netbox/utilities/querysets.py @@ -1,9 +1,35 @@ -from django.db.models import QuerySet +from django.db.models import Prefetch, QuerySet from users.constants import CONSTRAINT_TOKEN_USER from utilities.permissions import permission_is_exempt, qs_filter_from_constraints +class RestrictedPrefetch(Prefetch): + """ + Extend Django's Prefetch to accept a user and action to be passed to the + `restrict()` method of the related object's queryset. + """ + def __init__(self, lookup, user, action='view', queryset=None, to_attr=None): + self.restrict_user = user + self.restrict_action = action + + super().__init__(lookup, queryset=queryset, to_attr=to_attr) + + def get_current_queryset(self, level): + params = { + 'user': self.restrict_user, + 'action': self.restrict_action, + } + + if qs := super().get_current_queryset(level): + return qs.restrict(**params) + + # Bit of a hack. If no queryset is defined, pass through the dict of restrict() + # kwargs to be handled by the field. This is necessary e.g. for GenericForeignKey + # fields, which do not permit setting a queryset on a Prefetch object. + return params + + class RestrictedQuerySet(QuerySet): def restrict(self, user, action='view'): diff --git a/netbox/utilities/templates/search/searchbar.html b/netbox/utilities/templates/search/searchbar.html deleted file mode 100644 index 74d12e9b9..000000000 --- a/netbox/utilities/templates/search/searchbar.html +++ /dev/null @@ -1,50 +0,0 @@ -
- - - - - All Objects - - - -
    - {% for option in options %} - {% if option.items|length == 0 %} -
  • - -
    • - {% else %} -
  • {{ option.label }}
    • - {% endif %} - - {% for item in option.items %} -
  • - -
    • - {% endfor %} - - {% if forloop.counter != options|length %} -
    • - {% endif %} - {% endfor %} - -
- - - -
diff --git a/netbox/utilities/templatetags/search.py b/netbox/utilities/templatetags/search.py deleted file mode 100644 index ca8f3ba2a..000000000 --- a/netbox/utilities/templatetags/search.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import Dict - -from django import template - -from netbox.forms import SearchForm - -register = template.Library() -search_form = SearchForm() - - -@register.inclusion_tag("search/searchbar.html") -def search_options(request) -> Dict: - - # Provide search options to template. - return { - 'options': search_form.get_options(), - 'request': request, - } diff --git a/netbox/utilities/utils.py b/netbox/utilities/utils.py index 9f587e88d..e1fbbfe84 100644 --- a/netbox/utilities/utils.py +++ b/netbox/utilities/utils.py @@ -1,6 +1,7 @@ import datetime import decimal import json +import re from decimal import Decimal from itertools import count, groupby @@ -9,6 +10,7 @@ from django.core.serializers import serialize from django.db.models import Count, OuterRef, Subquery from django.db.models.functions import Coalesce from django.http import QueryDict +from django.utils.html import escape from jinja2.sandbox import SandboxedEnvironment from mptt.models import MPTTModel @@ -472,3 +474,23 @@ def clean_html(html, schemes): attributes=ALLOWED_ATTRIBUTES, protocols=schemes ) + + +def highlight_string(value, highlight, trim_pre=None, trim_post=None, trim_placeholder='...'): + """ + Highlight a string within a string and optionally trim the pre/post portions of the original string. + """ + # Split value on highlight string + try: + pre, match, post = re.split(fr'({highlight})', value, maxsplit=1, flags=re.IGNORECASE) + except ValueError: + # Match not found + return escape(value) + + # Trim pre/post sections to length + if trim_pre and len(pre) > trim_pre: + pre = trim_placeholder + pre[-trim_pre:] + if trim_post and len(post) > trim_post: + post = post[:trim_post] + trim_placeholder + + return f'{escape(pre)}{escape(match)}{escape(post)}' diff --git a/netbox/virtualization/search.py b/netbox/virtualization/search.py index 5b24f7fa0..184bf7049 100644 --- a/netbox/virtualization/search.py +++ b/netbox/virtualization/search.py @@ -1,33 +1,49 @@ -import virtualization.filtersets -import virtualization.tables -from dcim.models import Device from netbox.search import SearchIndex, register_search -from utilities.utils import count_related -from virtualization.models import Cluster, VirtualMachine +from . import models -@register_search() +@register_search class ClusterIndex(SearchIndex): - model = Cluster - queryset = Cluster.objects.prefetch_related('type', 'group').annotate( - device_count=count_related(Device, 'cluster'), vm_count=count_related(VirtualMachine, 'cluster') + model = models.Cluster + fields = ( + ('name', 100), + ('comments', 5000), ) - filterset = virtualization.filtersets.ClusterFilterSet - table = virtualization.tables.ClusterTable - url = 'virtualization:cluster_list' -@register_search() +@register_search +class ClusterGroupIndex(SearchIndex): + model = models.ClusterGroup + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class ClusterTypeIndex(SearchIndex): + model = models.ClusterType + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search class VirtualMachineIndex(SearchIndex): - model = VirtualMachine - queryset = VirtualMachine.objects.prefetch_related( - 'cluster', - 'tenant', - 'tenant__group', - 'platform', - 'primary_ip4', - 'primary_ip6', + model = models.VirtualMachine + fields = ( + ('name', 100), + ('comments', 5000), + ) + + +@register_search +class VMInterfaceIndex(SearchIndex): + model = models.VMInterface + fields = ( + ('name', 100), + ('description', 500), ) - filterset = virtualization.filtersets.VirtualMachineFilterSet - table = virtualization.tables.VirtualMachineTable - url = 'virtualization:virtualmachine_list' diff --git a/netbox/wireless/search.py b/netbox/wireless/search.py index 89ac23af8..55ca2977c 100644 --- a/netbox/wireless/search.py +++ b/netbox/wireless/search.py @@ -1,26 +1,32 @@ -import wireless.filtersets -import wireless.tables -from dcim.models import Interface from netbox.search import SearchIndex, register_search -from utilities.utils import count_related -from wireless.models import WirelessLAN, WirelessLink +from . import models -@register_search() +@register_search class WirelessLANIndex(SearchIndex): - model = WirelessLAN - queryset = WirelessLAN.objects.prefetch_related('group', 'vlan').annotate( - interface_count=count_related(Interface, 'wireless_lans') + model = models.WirelessLAN + fields = ( + ('ssid', 100), + ('description', 500), + ('auth_psk', 2000), ) - filterset = wireless.filtersets.WirelessLANFilterSet - table = wireless.tables.WirelessLANTable - url = 'wireless:wirelesslan_list' -@register_search() +@register_search +class WirelessLANGroupIndex(SearchIndex): + model = models.WirelessLANGroup + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search class WirelessLinkIndex(SearchIndex): - model = WirelessLink - queryset = WirelessLink.objects.prefetch_related('interface_a__device', 'interface_b__device') - filterset = wireless.filtersets.WirelessLinkFilterSet - table = wireless.tables.WirelessLinkTable - url = 'wireless:wirelesslink_list' + model = models.WirelessLink + fields = ( + ('ssid', 100), + ('description', 500), + ('auth_psk', 2000), + )