From 9628dead07ccef9608b32906aa8194bc948e5a09 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 21 Oct 2022 13:16:16 -0400 Subject: [PATCH] Closes #10560: New global search (#10676) * Initial work on new search backend * Clean up search backends * Return only the most relevant result per object * Clear any pre-existing cached entries on cache() * #6003: Implement global search functionality for custom field values * Tweak field weights & document guidance * Extend search() to accept a lookup type * Move get_registry() out of SearchBackend * Enforce object permissions when returning search results * Add indexers for remaining models * Avoid calling remove() on non-cacheable objects * Use new search backend by default * Extend search backend to filter by object type * Clean up search view form * Enable specifying lookup logic * Add indexes for value field * Remove object type selector from search bar * Introduce SearchTable and enable HTMX for results * Enable pagination * Remove legacy search backend * Cleanup * Use a UUID for CachedValue primary key * Refactoring search methods * Define max search results limit * Extend reindex command to support specifying particular models * Add clear() and size to SearchBackend * Optimize bulk caching performance * Highlight matched portion of field value * Performance improvements for reindexing * Started on search tests * Cleanup & docs * Documentation updates * Clean up SearchIndex * Flatten search registry to register by app_label.model_name * Clean up search backend classes * Clean up RestrictedGenericForeignKey and RestrictedPrefetch * Resolve migrations conflict --- docs/configuration/system.md | 8 + docs/development/search.md | 37 ++ docs/plugins/development/search.md | 15 +- docs/release-notes/version-3.4.md | 4 + mkdocs.yml | 1 + netbox/circuits/search.py | 73 +-- netbox/dcim/search.py | 422 ++++++++++++------ netbox/extras/api/serializers.py | 4 +- netbox/extras/filtersets.py | 4 +- netbox/extras/forms/bulk_import.py | 4 +- netbox/extras/forms/models.py | 4 +- netbox/extras/management/commands/reindex.py | 77 ++++ .../migrations/0079_change_jobresult_order.py | 17 - ...me.py => 0079_jobresult_scheduled_time.py} | 8 +- netbox/extras/migrations/0080_search.py | 35 ++ netbox/extras/models/__init__.py | 2 + netbox/extras/models/customfields.py | 25 +- netbox/extras/models/search.py | 50 +++ netbox/extras/plugins/__init__.py | 2 +- netbox/extras/registry.py | 2 +- netbox/extras/search.py | 15 +- netbox/extras/tables/tables.py | 4 +- netbox/extras/tests/dummy_plugin/search.py | 5 +- netbox/extras/tests/test_customfields.py | 2 + netbox/extras/tests/test_views.py | 11 +- netbox/ipam/search.py | 186 +++++--- netbox/netbox/constants.py | 3 - netbox/netbox/forms/__init__.py | 59 +-- netbox/netbox/search/__init__.py | 106 ++++- netbox/netbox/search/backends.py | 269 +++++++---- netbox/netbox/settings.py | 2 +- netbox/netbox/tables/tables.py | 41 ++ netbox/netbox/tests/test_search.py | 153 +++++++ netbox/netbox/views/__init__.py | 54 ++- netbox/project-static/dist/netbox.js | Bin 381426 -> 380899 bytes netbox/project-static/dist/netbox.js.map | Bin 354103 -> 353676 bytes netbox/project-static/src/netbox.ts | 4 +- netbox/project-static/src/search.ts | 51 +-- netbox/templates/base/layout.html | 5 +- netbox/templates/extras/customfield.html | 14 +- netbox/templates/inc/searchbar.html | 6 + netbox/templates/search.html | 88 +--- netbox/tenancy/search.py | 72 ++- netbox/utilities/fields.py | 70 +++ netbox/utilities/querysets.py | 28 +- .../utilities/templates/search/searchbar.html | 50 --- netbox/utilities/templatetags/search.py | 18 - netbox/utilities/utils.py | 22 + netbox/virtualization/search.py | 64 ++- netbox/wireless/search.py | 42 +- 50 files changed, 1571 insertions(+), 667 deletions(-) create mode 100644 docs/development/search.md create mode 100644 netbox/extras/management/commands/reindex.py delete mode 100644 netbox/extras/migrations/0079_change_jobresult_order.py rename netbox/extras/migrations/{0080_add_jobresult_scheduled_time.py => 0079_jobresult_scheduled_time.py} (64%) create mode 100644 netbox/extras/migrations/0080_search.py create mode 100644 netbox/extras/models/search.py create mode 100644 netbox/netbox/tests/test_search.py create mode 100644 netbox/templates/inc/searchbar.html delete mode 100644 netbox/utilities/templates/search/searchbar.html delete mode 100644 netbox/utilities/templatetags/search.py diff --git a/docs/configuration/system.md b/docs/configuration/system.md index 93f8fa902..3756b6a83 100644 --- a/docs/configuration/system.md +++ b/docs/configuration/system.md @@ -157,6 +157,14 @@ The file path to the location where [custom scripts](../customization/custom-scr --- +## SEARCH_BACKEND + +Default: `'netbox.search.backends.CachedValueSearchBackend'` + +The dotted path to the desired search backend class. `CachedValueSearchBackend` is currently the only search backend provided in NetBox, however this setting can be used to enable a custom backend. + +--- + ## STORAGE_BACKEND Default: None (local storage) diff --git a/docs/development/search.md b/docs/development/search.md new file mode 100644 index 000000000..02bcaa898 --- /dev/null +++ b/docs/development/search.md @@ -0,0 +1,37 @@ +# Search + +NetBox v3.4 introduced a new global search mechanism, which employs the `extras.CachedValue` model to store discrete field values from many models in a single table. + +## SearchIndex + +To enable search support for a model, declare and register a subclass of `netbox.search.SearchIndex` for it. Typically, this will be done within an app's `search.py` module. + +```python +from netbox.search import SearchIndex, register_search + +@register_search +class MyModelIndex(SearchIndex): + model = MyModel + fields = ( + ('name', 100), + ('description', 500), + ('comments', 5000), + ) +``` + +A SearchIndex subclass defines both its model and a list of two-tuples specifying which model fields to be indexed and the weight (precedence) associated with each. Guidance on weight assignment for fields is provided below. + +### Field Weight Guidance + +| Weight | Field Role | Examples | +|--------|--------------------------------------------------|----------------------------------------------------| +| 50 | Unique serialized attribute | Device.asset_tag | +| 60 | Unique serialized attribute (per related object) | Device.serial | +| 100 | Primary human identifier | Device.name, Circuit.cid, Cable.label | +| 110 | Slug | Site.slug | +| 200 | Secondary identifier | Provider.account, DeviceType.part_number | +| 300 | Highly unique descriptive attribute | CircuitTermination.xconnect_id, IPAddress.dns_name | +| 500 | Description | Site.description | +| 1000 | Custom field default | - | +| 2000 | Other discrete attribute | CircuitTermination.port_speed | +| 5000 | Comment field | Site.comments | diff --git a/docs/plugins/development/search.md b/docs/plugins/development/search.md index 13edd4527..e3b861f00 100644 --- a/docs/plugins/development/search.md +++ b/docs/plugins/development/search.md @@ -4,17 +4,16 @@ Plugins can define and register their own models to extend NetBox's core search ```python # search.py -from netbox.search import SearchMixin -from .filters import MyModelFilterSet -from .tables import MyModelTable +from netbox.search import SearchIndex from .models import MyModel -class MyModelIndex(SearchMixin): +class MyModelIndex(SearchIndex): model = MyModel - queryset = MyModel.objects.all() - filterset = MyModelFilterSet - table = MyModelTable - url = 'plugins:myplugin:mymodel_list' + fields = ( + ('name', 100), + ('description', 500), + ('comments', 5000), + ) ``` To register one or more indexes with NetBox, define a list named `indexes` at the end of this file: diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 8a9c61e22..80b94d6c2 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -11,6 +11,10 @@ ### New Features +#### New Global Search ([#10560](https://github.com/netbox-community/netbox/issues/10560)) + +NetBox's global search functionality has been completely overhauled and replaced by a new cache-based lookup. + #### Top-Level Plugin Navigation Menus ([#9071](https://github.com/netbox-community/netbox/issues/9071)) A new `PluginMenu` class has been introduced, which enables a plugin to inject a top-level menu in NetBox's navigation menu. This menu can have one or more groups of menu items, just like core items. Backward compatibility with the existing `menu_items` has been maintained. diff --git a/mkdocs.yml b/mkdocs.yml index fc3a40632..011d4414f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -245,6 +245,7 @@ nav: - Adding Models: 'development/adding-models.md' - Extending Models: 'development/extending-models.md' - Signals: 'development/signals.md' + - Search: 'development/search.md' - Application Registry: 'development/application-registry.md' - User Preferences: 'development/user-preferences.md' - Web UI: 'development/web-ui.md' diff --git a/netbox/circuits/search.py b/netbox/circuits/search.py index 5adfb97fb..673f6308f 100644 --- a/netbox/circuits/search.py +++ b/netbox/circuits/search.py @@ -1,34 +1,55 @@ -import circuits.filtersets -import circuits.tables -from circuits.models import Circuit, Provider, ProviderNetwork from netbox.search import SearchIndex, register_search -from utilities.utils import count_related +from . import models -@register_search() -class ProviderIndex(SearchIndex): - model = Provider - queryset = Provider.objects.annotate(count_circuits=count_related(Circuit, 'provider')) - filterset = circuits.filtersets.ProviderFilterSet - table = circuits.tables.ProviderTable - url = 'circuits:provider_list' - - -@register_search() +@register_search class CircuitIndex(SearchIndex): - model = Circuit - queryset = Circuit.objects.prefetch_related( - 'type', 'provider', 'tenant', 'tenant__group', 'terminations__site' + model = models.Circuit + fields = ( + ('cid', 100), + ('description', 500), + ('comments', 5000), ) - filterset = circuits.filtersets.CircuitFilterSet - table = circuits.tables.CircuitTable - url = 'circuits:circuit_list' -@register_search() +@register_search +class CircuitTerminationIndex(SearchIndex): + model = models.CircuitTermination + fields = ( + ('xconnect_id', 300), + ('pp_info', 300), + ('description', 500), + ('port_speed', 2000), + ('upstream_speed', 2000), + ) + + +@register_search +class CircuitTypeIndex(SearchIndex): + model = models.CircuitType + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class ProviderIndex(SearchIndex): + model = models.Provider + fields = ( + ('name', 100), + ('account', 200), + ('comments', 5000), + ) + + +@register_search class ProviderNetworkIndex(SearchIndex): - model = ProviderNetwork - queryset = ProviderNetwork.objects.prefetch_related('provider') - filterset = circuits.filtersets.ProviderNetworkFilterSet - table = circuits.tables.ProviderNetworkTable - url = 'circuits:providernetwork_list' + model = models.ProviderNetwork + fields = ( + ('name', 100), + ('service_id', 200), + ('description', 500), + ('comments', 5000), + ) diff --git a/netbox/dcim/search.py b/netbox/dcim/search.py index b179402ce..d34a78888 100644 --- a/netbox/dcim/search.py +++ b/netbox/dcim/search.py @@ -1,143 +1,293 @@ -import dcim.filtersets -import dcim.tables -from dcim.models import ( - Cable, - Device, - DeviceType, - Location, - Module, - ModuleType, - PowerFeed, - Rack, - RackReservation, - Site, - VirtualChassis, -) from netbox.search import SearchIndex, register_search -from utilities.utils import count_related +from . import models -@register_search() -class SiteIndex(SearchIndex): - model = Site - queryset = Site.objects.prefetch_related('region', 'tenant', 'tenant__group') - filterset = dcim.filtersets.SiteFilterSet - table = dcim.tables.SiteTable - url = 'dcim:site_list' - - -@register_search() -class RackIndex(SearchIndex): - model = Rack - queryset = Rack.objects.prefetch_related('site', 'location', 'tenant', 'tenant__group', 'role').annotate( - device_count=count_related(Device, 'rack') - ) - filterset = dcim.filtersets.RackFilterSet - table = dcim.tables.RackTable - url = 'dcim:rack_list' - - -@register_search() -class RackReservationIndex(SearchIndex): - model = RackReservation - queryset = RackReservation.objects.prefetch_related('rack', 'user') - filterset = dcim.filtersets.RackReservationFilterSet - table = dcim.tables.RackReservationTable - url = 'dcim:rackreservation_list' - - -@register_search() -class LocationIndex(SearchIndex): - model = Location - queryset = Location.objects.add_related_count( - Location.objects.add_related_count(Location.objects.all(), Device, 'location', 'device_count', cumulative=True), - Rack, - 'location', - 'rack_count', - cumulative=True, - ).prefetch_related('site') - filterset = dcim.filtersets.LocationFilterSet - table = dcim.tables.LocationTable - url = 'dcim:location_list' - - -@register_search() -class DeviceTypeIndex(SearchIndex): - model = DeviceType - queryset = DeviceType.objects.prefetch_related('manufacturer').annotate( - instance_count=count_related(Device, 'device_type') - ) - filterset = dcim.filtersets.DeviceTypeFilterSet - table = dcim.tables.DeviceTypeTable - url = 'dcim:devicetype_list' - - -@register_search() -class DeviceIndex(SearchIndex): - model = Device - queryset = Device.objects.prefetch_related( - 'device_type__manufacturer', - 'device_role', - 'tenant', - 'tenant__group', - 'site', - 'rack', - 'primary_ip4', - 'primary_ip6', - ) - filterset = dcim.filtersets.DeviceFilterSet - table = dcim.tables.DeviceTable - url = 'dcim:device_list' - - -@register_search() -class ModuleTypeIndex(SearchIndex): - model = ModuleType - queryset = ModuleType.objects.prefetch_related('manufacturer').annotate( - instance_count=count_related(Module, 'module_type') - ) - filterset = dcim.filtersets.ModuleTypeFilterSet - table = dcim.tables.ModuleTypeTable - url = 'dcim:moduletype_list' - - -@register_search() -class ModuleIndex(SearchIndex): - model = Module - queryset = Module.objects.prefetch_related( - 'module_type__manufacturer', - 'device', - 'module_bay', - ) - filterset = dcim.filtersets.ModuleFilterSet - table = dcim.tables.ModuleTable - url = 'dcim:module_list' - - -@register_search() -class VirtualChassisIndex(SearchIndex): - model = VirtualChassis - queryset = VirtualChassis.objects.prefetch_related('master').annotate( - member_count=count_related(Device, 'virtual_chassis') - ) - filterset = dcim.filtersets.VirtualChassisFilterSet - table = dcim.tables.VirtualChassisTable - url = 'dcim:virtualchassis_list' - - -@register_search() +@register_search class CableIndex(SearchIndex): - model = Cable - queryset = Cable.objects.all() - filterset = dcim.filtersets.CableFilterSet - table = dcim.tables.CableTable - url = 'dcim:cable_list' + model = models.Cable + fields = ( + ('label', 100), + ) -@register_search() +@register_search +class ConsolePortIndex(SearchIndex): + model = models.ConsolePort + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ('speed', 2000), + ) + + +@register_search +class ConsoleServerPortIndex(SearchIndex): + model = models.ConsoleServerPort + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ('speed', 2000), + ) + + +@register_search +class DeviceIndex(SearchIndex): + model = models.Device + fields = ( + ('asset_tag', 50), + ('serial', 60), + ('name', 100), + ('comments', 5000), + ) + + +@register_search +class DeviceBayIndex(SearchIndex): + model = models.DeviceBay + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ) + + +@register_search +class DeviceRoleIndex(SearchIndex): + model = models.DeviceRole + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class DeviceTypeIndex(SearchIndex): + model = models.DeviceType + fields = ( + ('model', 100), + ('part_number', 200), + ('comments', 5000), + ) + + +@register_search +class FrontPortIndex(SearchIndex): + model = models.FrontPort + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ) + + +@register_search +class InterfaceIndex(SearchIndex): + model = models.Interface + fields = ( + ('name', 100), + ('label', 200), + ('mac_address', 300), + ('wwn', 300), + ('description', 500), + ('mtu', 2000), + ('speed', 2000), + ) + + +@register_search +class InventoryItemIndex(SearchIndex): + model = models.InventoryItem + fields = ( + ('asset_tag', 50), + ('serial', 60), + ('name', 100), + ('label', 200), + ('description', 500), + ('part_id', 2000), + ) + + +@register_search +class LocationIndex(SearchIndex): + model = models.Location + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class ManufacturerIndex(SearchIndex): + model = models.Manufacturer + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class ModuleIndex(SearchIndex): + model = models.Module + fields = ( + ('asset_tag', 50), + ('serial', 60), + ('comments', 5000), + ) + + +@register_search +class ModuleBayIndex(SearchIndex): + model = models.ModuleBay + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ) + + +@register_search +class ModuleTypeIndex(SearchIndex): + model = models.ModuleType + fields = ( + ('model', 100), + ('part_number', 200), + ('comments', 5000), + ) + + +@register_search +class PlatformIndex(SearchIndex): + model = models.Platform + fields = ( + ('name', 100), + ('slug', 110), + ('napalm_driver', 300), + ('description', 500), + ) + + +@register_search class PowerFeedIndex(SearchIndex): - model = PowerFeed - queryset = PowerFeed.objects.all() - filterset = dcim.filtersets.PowerFeedFilterSet - table = dcim.tables.PowerFeedTable - url = 'dcim:powerfeed_list' + model = models.PowerFeed + fields = ( + ('name', 100), + ('comments', 5000), + ) + + +@register_search +class PowerOutletIndex(SearchIndex): + model = models.PowerOutlet + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ) + + +@register_search +class PowerPanelIndex(SearchIndex): + model = models.PowerPanel + fields = ( + ('name', 100), + ) + + +@register_search +class PowerPortIndex(SearchIndex): + model = models.PowerPort + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ('maximum_draw', 2000), + ('allocated_draw', 2000), + ) + + +@register_search +class RackIndex(SearchIndex): + model = models.Rack + fields = ( + ('asset_tag', 50), + ('serial', 60), + ('name', 100), + ('facility_id', 200), + ('comments', 5000), + ) + + +@register_search +class RackReservationIndex(SearchIndex): + model = models.RackReservation + fields = ( + ('description', 500), + ) + + +@register_search +class RackRoleIndex(SearchIndex): + model = models.RackRole + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class RearPortIndex(SearchIndex): + model = models.RearPort + fields = ( + ('name', 100), + ('label', 200), + ('description', 500), + ) + + +@register_search +class RegionIndex(SearchIndex): + model = models.Region + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500) + ) + + +@register_search +class SiteIndex(SearchIndex): + model = models.Site + fields = ( + ('name', 100), + ('facility', 100), + ('slug', 110), + ('description', 500), + ('physical_address', 2000), + ('shipping_address', 2000), + ('comments', 5000), + ) + + +@register_search +class SiteGroupIndex(SearchIndex): + model = models.SiteGroup + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500) + ) + + +@register_search +class VirtualChassisIndex(SearchIndex): + model = models.VirtualChassis + fields = ( + ('name', 100), + ('domain', 300) + ) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index b34f5fba3..99f4dd02b 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -92,8 +92,8 @@ class CustomFieldSerializer(ValidatedModelSerializer): model = CustomField fields = [ 'id', 'url', 'display', 'content_types', 'type', 'object_type', 'data_type', 'name', 'label', 'group_name', - 'description', 'required', 'filter_logic', 'ui_visibility', 'default', 'weight', 'validation_minimum', - 'validation_maximum', 'validation_regex', 'choices', 'created', 'last_updated', + 'description', 'required', 'search_weight', 'filter_logic', 'ui_visibility', 'default', 'weight', + 'validation_minimum', 'validation_maximum', 'validation_regex', 'choices', 'created', 'last_updated', ] def get_data_type(self, obj): diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index 8c9c58a13..1b1b049c7 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -73,8 +73,8 @@ class CustomFieldFilterSet(BaseFilterSet): class Meta: model = CustomField fields = [ - 'id', 'content_types', 'name', 'group_name', 'required', 'filter_logic', 'ui_visibility', 'weight', - 'description', + 'id', 'content_types', 'name', 'group_name', 'required', 'search_weight', 'filter_logic', 'ui_visibility', + 'weight', 'description', ] def search(self, queryset, name, value): diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index e83cac3b9..0303dae30 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -46,8 +46,8 @@ class CustomFieldCSVForm(CSVModelForm): class Meta: model = CustomField fields = ( - 'name', 'label', 'group_name', 'type', 'content_types', 'object_type', 'required', 'description', 'weight', - 'filter_logic', 'default', 'choices', 'weight', 'validation_minimum', 'validation_maximum', + 'name', 'label', 'group_name', 'type', 'content_types', 'object_type', 'required', 'description', + 'search_weight', 'filter_logic', 'default', 'choices', 'weight', 'validation_minimum', 'validation_maximum', 'validation_regex', 'ui_visibility', ) diff --git a/netbox/extras/forms/models.py b/netbox/extras/forms/models.py index bea1fbcc1..eca93849b 100644 --- a/netbox/extras/forms/models.py +++ b/netbox/extras/forms/models.py @@ -41,9 +41,9 @@ class CustomFieldForm(BootstrapMixin, forms.ModelForm): fieldsets = ( ('Custom Field', ( - 'content_types', 'name', 'label', 'group_name', 'type', 'object_type', 'weight', 'required', 'description', + 'content_types', 'name', 'label', 'group_name', 'type', 'object_type', 'required', 'description', )), - ('Behavior', ('filter_logic', 'ui_visibility')), + ('Behavior', ('search_weight', 'filter_logic', 'ui_visibility', 'weight')), ('Values', ('default', 'choices')), ('Validation', ('validation_minimum', 'validation_maximum', 'validation_regex')), ) diff --git a/netbox/extras/management/commands/reindex.py b/netbox/extras/management/commands/reindex.py new file mode 100644 index 000000000..6dc9bbb2d --- /dev/null +++ b/netbox/extras/management/commands/reindex.py @@ -0,0 +1,77 @@ +from django.contrib.contenttypes.models import ContentType +from django.core.management.base import BaseCommand, CommandError + +from extras.registry import registry +from netbox.search.backends import search_backend + + +class Command(BaseCommand): + help = 'Reindex objects for search' + + def add_arguments(self, parser): + parser.add_argument( + 'args', + metavar='app_label[.ModelName]', + nargs='*', + help='One or more apps or models to reindex', + ) + + def _get_indexers(self, *model_names): + indexers = {} + + # No models specified; pull in all registered indexers + if not model_names: + for idx in registry['search'].values(): + indexers[idx.model] = idx + + # Return only indexers for the specified models + else: + for label in model_names: + try: + app_label, model_name = label.lower().split('.') + except ValueError: + raise CommandError( + f"Invalid model: {label}. Model names must be in the format .." + ) + try: + idx = registry['search'][f'{app_label}.{model_name}'] + indexers[idx.model] = idx + except KeyError: + raise CommandError(f"No indexer registered for {label}") + + return indexers + + def handle(self, *model_labels, **kwargs): + + # Determine which models to reindex + indexers = self._get_indexers(*model_labels) + if not indexers: + raise CommandError("No indexers found!") + self.stdout.write(f'Reindexing {len(indexers)} models.') + + # Clear all cached values for the specified models + self.stdout.write('Clearing cached values... ', ending='') + self.stdout.flush() + content_types = [ + ContentType.objects.get_for_model(model) for model in indexers.keys() + ] + deleted_count = search_backend.clear(content_types) + self.stdout.write(f'{deleted_count} entries deleted.') + + # Index models + self.stdout.write('Indexing models') + for model, idx in indexers.items(): + app_label = model._meta.app_label + model_name = model._meta.model_name + self.stdout.write(f' {app_label}.{model_name}... ', ending='') + self.stdout.flush() + i = search_backend.cache(model.objects.iterator(), remove_existing=False) + if i: + self.stdout.write(f'{i} entries cached.') + else: + self.stdout.write(f'None found.') + + msg = f'Completed.' + if total_count := search_backend.size: + msg += f' Total entries: {total_count}' + self.stdout.write(msg, self.style.SUCCESS) diff --git a/netbox/extras/migrations/0079_change_jobresult_order.py b/netbox/extras/migrations/0079_change_jobresult_order.py deleted file mode 100644 index 12e35bf67..000000000 --- a/netbox/extras/migrations/0079_change_jobresult_order.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 4.1.1 on 2022-10-09 18:37 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('extras', '0078_unique_constraints'), - ] - - operations = [ - migrations.AlterModelOptions( - name='jobresult', - options={'ordering': ['-created']}, - ), - ] diff --git a/netbox/extras/migrations/0080_add_jobresult_scheduled_time.py b/netbox/extras/migrations/0079_jobresult_scheduled_time.py similarity index 64% rename from netbox/extras/migrations/0080_add_jobresult_scheduled_time.py rename to netbox/extras/migrations/0079_jobresult_scheduled_time.py index fddde4bc5..c9646f13c 100644 --- a/netbox/extras/migrations/0080_add_jobresult_scheduled_time.py +++ b/netbox/extras/migrations/0079_jobresult_scheduled_time.py @@ -1,12 +1,10 @@ -# Generated by Django 4.1.1 on 2022-10-16 09:52 - from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('extras', '0079_change_jobresult_order'), + ('extras', '0078_unique_constraints'), ] operations = [ @@ -15,4 +13,8 @@ class Migration(migrations.Migration): name='scheduled_time', field=models.DateTimeField(blank=True, null=True), ), + migrations.AlterModelOptions( + name='jobresult', + options={'ordering': ['-created']}, + ), ] diff --git a/netbox/extras/migrations/0080_search.py b/netbox/extras/migrations/0080_search.py new file mode 100644 index 000000000..7a133e84b --- /dev/null +++ b/netbox/extras/migrations/0080_search.py @@ -0,0 +1,35 @@ +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('extras', '0079_jobresult_scheduled_time'), + ] + + operations = [ + migrations.AddField( + model_name='customfield', + name='search_weight', + field=models.PositiveSmallIntegerField(default=1000), + ), + migrations.CreateModel( + name='CachedValue', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('object_id', models.PositiveBigIntegerField()), + ('field', models.CharField(max_length=200)), + ('type', models.CharField(max_length=30)), + ('value', models.TextField(db_index=True)), + ('weight', models.PositiveSmallIntegerField(default=1000)), + ('object_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='contenttypes.contenttype')), + ], + options={ + 'ordering': ('weight', 'object_type', 'object_id'), + }, + ), + ] diff --git a/netbox/extras/models/__init__.py b/netbox/extras/models/__init__.py index 3cb6372be..e3a4be3fe 100644 --- a/netbox/extras/models/__init__.py +++ b/netbox/extras/models/__init__.py @@ -2,9 +2,11 @@ from .change_logging import ObjectChange from .configcontexts import ConfigContext, ConfigContextModel from .customfields import CustomField from .models import * +from .search import * from .tags import Tag, TaggedItem __all__ = ( + 'CachedValue', 'ConfigContext', 'ConfigContextModel', 'ConfigRevision', diff --git a/netbox/extras/models/customfields.py b/netbox/extras/models/customfields.py index c3c298a44..2de806ca6 100644 --- a/netbox/extras/models/customfields.py +++ b/netbox/extras/models/customfields.py @@ -16,6 +16,7 @@ from extras.choices import * from extras.utils import FeatureQuery from netbox.models import ChangeLoggedModel from netbox.models.features import CloningMixin, ExportTemplatesMixin, WebhooksMixin +from netbox.search import FieldTypes from utilities import filters from utilities.forms import ( CSVChoiceField, CSVMultipleChoiceField, DatePicker, DynamicModelChoiceField, DynamicModelMultipleChoiceField, @@ -30,6 +31,15 @@ __all__ = ( 'CustomFieldManager', ) +SEARCH_TYPES = { + CustomFieldTypeChoices.TYPE_TEXT: FieldTypes.STRING, + CustomFieldTypeChoices.TYPE_LONGTEXT: FieldTypes.STRING, + CustomFieldTypeChoices.TYPE_INTEGER: FieldTypes.INTEGER, + CustomFieldTypeChoices.TYPE_DECIMAL: FieldTypes.FLOAT, + CustomFieldTypeChoices.TYPE_DATE: FieldTypes.STRING, + CustomFieldTypeChoices.TYPE_URL: FieldTypes.STRING, +} + class CustomFieldManager(models.Manager.from_queryset(RestrictedQuerySet)): use_in_migrations = True @@ -94,6 +104,11 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge help_text='If true, this field is required when creating new objects ' 'or editing an existing object.' ) + search_weight = models.PositiveSmallIntegerField( + default=1000, + help_text='Weighting for search. Lower values are considered more important. ' + 'Fields with a search weight of zero will be ignored.' + ) filter_logic = models.CharField( max_length=50, choices=CustomFieldFilterLogicChoices, @@ -109,6 +124,7 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge ) weight = models.PositiveSmallIntegerField( default=100, + verbose_name='Display weight', help_text='Fields with higher weights appear lower in a form.' ) validation_minimum = models.IntegerField( @@ -148,8 +164,9 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge objects = CustomFieldManager() clone_fields = ( - 'content_types', 'type', 'object_type', 'group_name', 'description', 'required', 'filter_logic', 'default', - 'weight', 'validation_minimum', 'validation_maximum', 'validation_regex', 'choices', 'ui_visibility', + 'content_types', 'type', 'object_type', 'group_name', 'description', 'required', 'search_weight', + 'filter_logic', 'default', 'weight', 'validation_minimum', 'validation_maximum', 'validation_regex', 'choices', + 'ui_visibility', ) class Meta: @@ -167,6 +184,10 @@ class CustomField(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogge # Cache instance's original name so we can check later whether it has changed self._name = self.name + @property + def search_type(self): + return SEARCH_TYPES.get(self.type) + def populate_initial_data(self, content_types): """ Populate initial custom field data upon either a) the creation of a new CustomField, or diff --git a/netbox/extras/models/search.py b/netbox/extras/models/search.py new file mode 100644 index 000000000..b7bb104e7 --- /dev/null +++ b/netbox/extras/models/search.py @@ -0,0 +1,50 @@ +import uuid + +from django.contrib.contenttypes.models import ContentType +from django.db import models + +from utilities.fields import RestrictedGenericForeignKey + +__all__ = ( + 'CachedValue', +) + + +class CachedValue(models.Model): + id = models.UUIDField( + primary_key=True, + default=uuid.uuid4, + editable=False + ) + timestamp = models.DateTimeField( + auto_now_add=True, + editable=False + ) + object_type = models.ForeignKey( + to=ContentType, + on_delete=models.CASCADE, + related_name='+' + ) + object_id = models.PositiveBigIntegerField() + object = RestrictedGenericForeignKey( + ct_field='object_type', + fk_field='object_id' + ) + field = models.CharField( + max_length=200 + ) + type = models.CharField( + max_length=30 + ) + value = models.TextField( + db_index=True + ) + weight = models.PositiveSmallIntegerField( + default=1000 + ) + + class Meta: + ordering = ('weight', 'object_type', 'object_id') + + def __str__(self): + return f'{self.object_type} {self.object_id}: {self.field}={self.value}' diff --git a/netbox/extras/plugins/__init__.py b/netbox/extras/plugins/__init__.py index ea88c0f51..f855027e2 100644 --- a/netbox/extras/plugins/__init__.py +++ b/netbox/extras/plugins/__init__.py @@ -75,7 +75,7 @@ class PluginConfig(AppConfig): try: search_indexes = import_string(f"{self.__module__}.{self.search_indexes}") for idx in search_indexes: - register_search()(idx) + register_search(idx) except ImportError: pass diff --git a/netbox/extras/registry.py b/netbox/extras/registry.py index f89499842..76886e791 100644 --- a/netbox/extras/registry.py +++ b/netbox/extras/registry.py @@ -29,5 +29,5 @@ registry['model_features'] = { feature: collections.defaultdict(set) for feature in EXTRAS_FEATURES } registry['denormalized_fields'] = collections.defaultdict(list) -registry['search'] = collections.defaultdict(dict) +registry['search'] = dict() registry['views'] = collections.defaultdict(dict) diff --git a/netbox/extras/search.py b/netbox/extras/search.py index ae6c9e7b9..da4aa1c84 100644 --- a/netbox/extras/search.py +++ b/netbox/extras/search.py @@ -1,14 +1,11 @@ -import extras.filtersets -import extras.tables -from extras.models import JournalEntry from netbox.search import SearchIndex, register_search +from . import models -@register_search() +@register_search class JournalEntryIndex(SearchIndex): - model = JournalEntry - queryset = JournalEntry.objects.prefetch_related('assigned_object', 'created_by') - filterset = extras.filtersets.JournalEntryFilterSet - table = extras.tables.JournalEntryTable - url = 'extras:journalentry_list' + model = models.JournalEntry + fields = ( + ('comments', 5000), + ) category = 'Journal' diff --git a/netbox/extras/tables/tables.py b/netbox/extras/tables/tables.py index 8f365a58b..73d3e98b2 100644 --- a/netbox/extras/tables/tables.py +++ b/netbox/extras/tables/tables.py @@ -34,8 +34,8 @@ class CustomFieldTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = CustomField fields = ( - 'pk', 'id', 'name', 'content_types', 'label', 'type', 'group_name', 'required', 'weight', 'default', - 'description', 'filter_logic', 'ui_visibility', 'choices', 'created', 'last_updated', + 'pk', 'id', 'name', 'content_types', 'label', 'type', 'group_name', 'required', 'default', 'description', + 'search_weight', 'filter_logic', 'ui_visibility', 'weight', 'choices', 'created', 'last_updated', ) default_columns = ('pk', 'name', 'content_types', 'label', 'group_name', 'type', 'required', 'description') diff --git a/netbox/extras/tests/dummy_plugin/search.py b/netbox/extras/tests/dummy_plugin/search.py index 4a1b7e666..4b1c6f10e 100644 --- a/netbox/extras/tests/dummy_plugin/search.py +++ b/netbox/extras/tests/dummy_plugin/search.py @@ -4,8 +4,9 @@ from .models import DummyModel class DummyModelIndex(SearchIndex): model = DummyModel - queryset = DummyModel.objects.all() - url = 'plugins:dummy_plugin:dummy_models' + fields = ( + ('name', 100), + ) indexes = ( diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index 6080ce2e5..c6ba96a82 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -292,6 +292,7 @@ class CustomFieldTest(TestCase): cf = CustomField.objects.create( name='object_field', type=CustomFieldTypeChoices.TYPE_OBJECT, + object_type=ContentType.objects.get_for_model(VLAN), required=False ) cf.content_types.set([self.object_type]) @@ -323,6 +324,7 @@ class CustomFieldTest(TestCase): cf = CustomField.objects.create( name='object_field', type=CustomFieldTypeChoices.TYPE_MULTIOBJECT, + object_type=ContentType.objects.get_for_model(VLAN), required=False ) cf.content_types.set([self.object_type]) diff --git a/netbox/extras/tests/test_views.py b/netbox/extras/tests/test_views.py index 936213cbf..9634038c1 100644 --- a/netbox/extras/tests/test_views.py +++ b/netbox/extras/tests/test_views.py @@ -32,6 +32,7 @@ class CustomFieldTestCase(ViewTestCases.PrimaryObjectViewTestCase): 'label': 'Field X', 'type': 'text', 'content_types': [site_ct.pk], + 'search_weight': 2000, 'filter_logic': CustomFieldFilterLogicChoices.FILTER_EXACT, 'default': None, 'weight': 200, @@ -40,11 +41,11 @@ class CustomFieldTestCase(ViewTestCases.PrimaryObjectViewTestCase): } cls.csv_data = ( - 'name,label,type,content_types,object_type,weight,filter_logic,choices,validation_minimum,validation_maximum,validation_regex,ui_visibility', - 'field4,Field 4,text,dcim.site,,100,exact,,,,[a-z]{3},read-write', - 'field5,Field 5,integer,dcim.site,,100,exact,,1,100,,read-write', - 'field6,Field 6,select,dcim.site,,100,exact,"A,B,C",,,,read-write', - 'field7,Field 7,object,dcim.site,dcim.region,100,exact,,,,,read-write', + 'name,label,type,content_types,object_type,weight,search_weight,filter_logic,choices,validation_minimum,validation_maximum,validation_regex,ui_visibility', + 'field4,Field 4,text,dcim.site,,100,1000,exact,,,,[a-z]{3},read-write', + 'field5,Field 5,integer,dcim.site,,100,2000,exact,,1,100,,read-write', + 'field6,Field 6,select,dcim.site,,100,3000,exact,"A,B,C",,,,read-write', + 'field7,Field 7,object,dcim.site,dcim.region,100,4000,exact,,,,,read-write', ) cls.bulk_edit_data = { diff --git a/netbox/ipam/search.py b/netbox/ipam/search.py index 2f4599321..d1d25da76 100644 --- a/netbox/ipam/search.py +++ b/netbox/ipam/search.py @@ -1,69 +1,139 @@ -import ipam.filtersets -import ipam.tables -from ipam.models import ASN, VLAN, VRF, Aggregate, IPAddress, Prefix, Service +from . import models from netbox.search import SearchIndex, register_search -@register_search() -class VRFIndex(SearchIndex): - model = VRF - queryset = VRF.objects.prefetch_related('tenant', 'tenant__group') - filterset = ipam.filtersets.VRFFilterSet - table = ipam.tables.VRFTable - url = 'ipam:vrf_list' - - -@register_search() +@register_search class AggregateIndex(SearchIndex): - model = Aggregate - queryset = Aggregate.objects.prefetch_related('rir') - filterset = ipam.filtersets.AggregateFilterSet - table = ipam.tables.AggregateTable - url = 'ipam:aggregate_list' - - -@register_search() -class PrefixIndex(SearchIndex): - model = Prefix - queryset = Prefix.objects.prefetch_related( - 'site', 'vrf__tenant', 'tenant', 'tenant__group', 'vlan', 'role' + model = models.Aggregate + fields = ( + ('prefix', 100), + ('description', 500), + ('date_added', 2000), ) - filterset = ipam.filtersets.PrefixFilterSet - table = ipam.tables.PrefixTable - url = 'ipam:prefix_list' -@register_search() -class IPAddressIndex(SearchIndex): - model = IPAddress - queryset = IPAddress.objects.prefetch_related('vrf__tenant', 'tenant', 'tenant__group') - filterset = ipam.filtersets.IPAddressFilterSet - table = ipam.tables.IPAddressTable - url = 'ipam:ipaddress_list' - - -@register_search() -class VLANIndex(SearchIndex): - model = VLAN - queryset = VLAN.objects.prefetch_related('site', 'group', 'tenant', 'tenant__group', 'role') - filterset = ipam.filtersets.VLANFilterSet - table = ipam.tables.VLANTable - url = 'ipam:vlan_list' - - -@register_search() +@register_search class ASNIndex(SearchIndex): - model = ASN - queryset = ASN.objects.prefetch_related('rir', 'tenant', 'tenant__group') - filterset = ipam.filtersets.ASNFilterSet - table = ipam.tables.ASNTable - url = 'ipam:asn_list' + model = models.ASN + fields = ( + ('asn', 100), + ('description', 500), + ) -@register_search() +@register_search +class FHRPGroupIndex(SearchIndex): + model = models.FHRPGroup + fields = ( + ('name', 100), + ('group_id', 2000), + ('description', 500), + ) + + +@register_search +class IPAddressIndex(SearchIndex): + model = models.IPAddress + fields = ( + ('address', 100), + ('dns_name', 300), + ('description', 500), + ) + + +@register_search +class IPRangeIndex(SearchIndex): + model = models.IPRange + fields = ( + ('start_address', 100), + ('end_address', 300), + ('description', 500), + ) + + +@register_search +class L2VPNIndex(SearchIndex): + model = models.L2VPN + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class PrefixIndex(SearchIndex): + model = models.Prefix + fields = ( + ('prefix', 100), + ('description', 500), + ) + + +@register_search +class RIRIndex(SearchIndex): + model = models.RIR + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class RoleIndex(SearchIndex): + model = models.Role + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ) + + +@register_search +class RouteTargetIndex(SearchIndex): + model = models.RouteTarget + fields = ( + ('name', 100), + ('description', 500), + ) + + +@register_search class ServiceIndex(SearchIndex): - model = Service - queryset = Service.objects.prefetch_related('device', 'virtual_machine') - filterset = ipam.filtersets.ServiceFilterSet - table = ipam.tables.ServiceTable - url = 'ipam:service_list' + model = models.Service + fields = ( + ('name', 100), + ('description', 500), + ) + + +@register_search +class VLANIndex(SearchIndex): + model = models.VLAN + fields = ( + ('name', 100), + ('vid', 100), + ('description', 500), + ) + + +@register_search +class VLANGroupIndex(SearchIndex): + model = models.VLANGroup + fields = ( + ('name', 100), + ('slug', 110), + ('description', 500), + ('max_vid', 2000), + ) + + +@register_search +class VRFIndex(SearchIndex): + model = models.VRF + fields = ( + ('name', 100), + ('rd', 200), + ('description', 500), + ) diff --git a/netbox/netbox/constants.py b/netbox/netbox/constants.py index 776938a97..c8054b3b0 100644 --- a/netbox/netbox/constants.py +++ b/netbox/netbox/constants.py @@ -1,5 +1,2 @@ # Prefix for nested serializers NESTED_SERIALIZER_PREFIX = 'Nested' - -# Max results per object type -SEARCH_MAX_RESULTS = 15 diff --git a/netbox/netbox/forms/__init__.py b/netbox/netbox/forms/__init__.py index eb1311d98..dd1fb7726 100644 --- a/netbox/netbox/forms/__init__.py +++ b/netbox/netbox/forms/__init__.py @@ -1,38 +1,45 @@ from django import forms +from django.utils.translation import gettext as _ -from netbox.search.backends import default_search_engine -from utilities.forms import BootstrapMixin +from netbox.search import LookupTypes +from netbox.search.backends import search_backend +from utilities.forms import BootstrapMixin, StaticSelect, StaticSelectMultiple from .base import * - -def build_options(choices): - options = [{"label": choices[0][1], "items": []}] - - for label, choices in choices[1:]: - items = [] - - for value, choice_label in choices: - items.append({"label": choice_label, "value": value}) - - options.append({"label": label, "items": items}) - return options +LOOKUP_CHOICES = ( + ('', _('Partial match')), + (LookupTypes.EXACT, _('Exact match')), + (LookupTypes.STARTSWITH, _('Starts with')), + (LookupTypes.ENDSWITH, _('Ends with')), +) class SearchForm(BootstrapMixin, forms.Form): - q = forms.CharField(label='Search') - options = None + q = forms.CharField( + label='Search', + widget=forms.TextInput( + attrs={ + 'hx-get': '', + 'hx-target': '#object_list', + 'hx-trigger': 'keyup[target.value.length >= 3] changed delay:500ms', + } + ) + ) + obj_types = forms.MultipleChoiceField( + choices=[], + required=False, + label='Object type(s)', + widget=StaticSelectMultiple() + ) + lookup = forms.ChoiceField( + choices=LOOKUP_CHOICES, + initial=LookupTypes.PARTIAL, + required=False, + widget=StaticSelect() + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields["obj_type"] = forms.ChoiceField( - choices=default_search_engine.get_search_choices(), - required=False, - label='Type' - ) - def get_options(self): - if not self.options: - self.options = build_options(default_search_engine.get_search_choices()) - - return self.options + self.fields['obj_types'].choices = search_backend.get_object_types() diff --git a/netbox/netbox/search/__init__.py b/netbox/netbox/search/__init__.py index 0664dc6ca..568bf8652 100644 --- a/netbox/netbox/search/__init__.py +++ b/netbox/netbox/search/__init__.py @@ -1,5 +1,24 @@ +from collections import namedtuple + +from django.db import models + from extras.registry import registry +ObjectFieldValue = namedtuple('ObjectFieldValue', ('name', 'type', 'weight', 'value')) + + +class FieldTypes: + FLOAT = 'float' + INTEGER = 'int' + STRING = 'str' + + +class LookupTypes: + PARTIAL = 'icontains' + EXACT = 'iexact' + STARTSWITH = 'istartswith' + ENDSWITH = 'iendswith' + class SearchIndex: """ @@ -7,27 +26,90 @@ class SearchIndex: Attrs: model: The model class for which this index is used. + category: The label of the group under which this indexer is categorized (for form field display). If none, + the name of the model's app will be used. + fields: An iterable of two-tuples defining the model fields to be indexed and the weight associated with each. """ model = None + category = None + fields = () + + @staticmethod + def get_field_type(instance, field_name): + """ + Return the data type of the specified model field. + """ + field_cls = instance._meta.get_field(field_name).__class__ + if issubclass(field_cls, (models.FloatField, models.DecimalField)): + return FieldTypes.FLOAT + if issubclass(field_cls, models.IntegerField): + return FieldTypes.INTEGER + return FieldTypes.STRING + + @staticmethod + def get_field_value(instance, field_name): + """ + Return the value of the specified model field as a string. + """ + return str(getattr(instance, field_name)) @classmethod def get_category(cls): + return cls.category or cls.model._meta.app_config.verbose_name + + @classmethod + def to_cache(cls, instance, custom_fields=None): """ - Return the title of the search category under which this model is registered. + Return a list of ObjectFieldValue representing the instance fields to be cached. + + Args: + instance: The instance being cached. + custom_fields: An iterable of CustomFields to include when caching the instance. If None, all custom fields + defined for the model will be included. (This can also be provided during bulk caching to avoid looking + up the available custom fields for each instance.) """ - if hasattr(cls, 'category'): - return cls.category - return cls.model._meta.app_config.verbose_name + values = [] + + # Capture built-in fields + for name, weight in cls.fields: + type_ = cls.get_field_type(instance, name) + value = cls.get_field_value(instance, name) + if type_ and value: + values.append( + ObjectFieldValue(name, type_, weight, value) + ) + + # Capture custom fields + if getattr(instance, 'custom_field_data', None): + if custom_fields is None: + custom_fields = instance.get_custom_fields().keys() + for cf in custom_fields: + type_ = cf.search_type + value = instance.custom_field_data.get(cf.name) + weight = cf.search_weight + if type_ and value and weight: + values.append( + ObjectFieldValue(f'cf_{cf.name}', type_, weight, value) + ) + + return values -def register_search(): - def _wrapper(cls): - model = cls.model - app_label = model._meta.app_label - model_name = model._meta.model_name +def get_indexer(model): + """ + Get the SearchIndex class for the given model. + """ + label = f'{model._meta.app_label}.{model._meta.model_name}' - registry['search'][app_label][model_name] = cls + return registry['search'][label] - return cls - return _wrapper +def register_search(cls): + """ + Decorator for registering a SearchIndex class. + """ + model = cls.model + label = f'{model._meta.app_label}.{model._meta.model_name}' + registry['search'][label] = cls + + return cls diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index b6cead5bd..f1e00b86b 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -1,125 +1,236 @@ from collections import defaultdict -from importlib import import_module from django.conf import settings +from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ImproperlyConfigured -from django.urls import reverse +from django.db.models import F, Window +from django.db.models.functions import window +from django.db.models.signals import post_delete, post_save +from django.utils.module_loading import import_string +from extras.models import CachedValue, CustomField from extras.registry import registry -from netbox.constants import SEARCH_MAX_RESULTS +from utilities.querysets import RestrictedPrefetch +from utilities.templatetags.builtins.filters import bettertitle +from . import FieldTypes, LookupTypes, get_indexer -# The cache for the initialized backend. -_backends_cache = {} - - -class SearchEngineError(Exception): - """Something went wrong with a search engine.""" - pass +DEFAULT_LOOKUP_TYPE = LookupTypes.PARTIAL +MAX_RESULTS = 1000 class SearchBackend: - """A search engine capable of performing multi-table searches.""" - _search_choice_options = tuple() + """ + Base class for search backends. Subclasses must extend the `cache()`, `remove()`, and `clear()` methods below. + """ + _object_types = None - def get_registry(self): - r = {} - for app_label, models in registry['search'].items(): - r.update(**models) - - return r - - def get_search_choices(self): - """Return the set of choices for individual object types, organized by category.""" - if not self._search_choice_options: + def get_object_types(self): + """ + Return a list of all registered object types, organized by category, suitable for populating a form's + ChoiceField. + """ + if not self._object_types: # Organize choices by category categories = defaultdict(dict) - for app_label, models in registry['search'].items(): - for name, cls in models.items(): - title = cls.model._meta.verbose_name.title() - categories[cls.get_category()][name] = title + for label, idx in registry['search'].items(): + title = bettertitle(idx.model._meta.verbose_name) + categories[idx.get_category()][label] = title # Compile a nested tuple of choices for form rendering results = ( ('', 'All Objects'), - *[(category, choices.items()) for category, choices in categories.items()] + *[(category, list(choices.items())) for category, choices in categories.items()] ) - self._search_choice_options = results + self._object_types = results - return self._search_choice_options + return self._object_types - def search(self, request, value, **kwargs): - """Execute a search query for the given value.""" + def search(self, value, user=None, object_types=None, lookup=DEFAULT_LOOKUP_TYPE): + """ + Search cached object representations for the given value. + """ raise NotImplementedError - def cache(self, instance): - """Create or update the cached copy of an instance.""" + def caching_handler(self, sender, instance, **kwargs): + """ + Receiver for the post_save signal, responsible for caching object creation/changes. + """ + self.cache(instance) + + def removal_handler(self, sender, instance, **kwargs): + """ + Receiver for the post_delete signal, responsible for caching object deletion. + """ + self.remove(instance) + + def cache(self, instances, indexer=None, remove_existing=True): + """ + Create or update the cached representation of an instance. + """ raise NotImplementedError + def remove(self, instance): + """ + Delete any cached representation of an instance. + """ + raise NotImplementedError -class FilterSetSearchBackend(SearchBackend): - """ - Legacy search backend. Performs a discrete database query for each registered object type, using the FilterSet - class specified by the index for each. - """ - def search(self, request, value, **kwargs): - results = [] + def clear(self, object_types=None): + """ + Delete *all* cached data. + """ + raise NotImplementedError - search_registry = self.get_registry() - for obj_type in search_registry.keys(): + @property + def size(self): + """ + Return a total number of cached entries. The meaning of this value will be + backend-dependent. + """ + return None - queryset = search_registry[obj_type].queryset - url = search_registry[obj_type].url - # Restrict the queryset for the current user - if hasattr(queryset, 'restrict'): - queryset = queryset.restrict(request.user, 'view') +class CachedValueSearchBackend(SearchBackend): - filterset = getattr(search_registry[obj_type], 'filterset', None) - if not filterset: - # This backend requires a FilterSet class for the model - continue + def search(self, value, user=None, object_types=None, lookup=DEFAULT_LOOKUP_TYPE): - table = getattr(search_registry[obj_type], 'table', None) - if not table: - # This backend requires a Table class for the model - continue + # Define the search parameters + params = { + f'value__{lookup}': value + } + if lookup != LookupTypes.EXACT: + # Partial matches are valid only on string values + params['type'] = FieldTypes.STRING + if object_types: + params['object_type__in'] = object_types - # Construct the results table for this object type - filtered_queryset = filterset({'q': value}, queryset=queryset).qs - table = table(filtered_queryset, orderable=False) - table.paginate(per_page=SEARCH_MAX_RESULTS) + # Construct the base queryset to retrieve matching results + queryset = CachedValue.objects.filter(**params).annotate( + # Annotate the rank of each result for its object according to its weight + row_number=Window( + expression=window.RowNumber(), + partition_by=[F('object_type'), F('object_id')], + order_by=[F('weight').asc()], + ) + )[:MAX_RESULTS] - if table.page: - results.append({ - 'name': queryset.model._meta.verbose_name_plural, - 'table': table, - 'url': f"{reverse(url)}?q={value}" - }) + # Construct a Prefetch to pre-fetch only those related objects for which the + # user has permission to view. + if user: + prefetch = (RestrictedPrefetch('object', user, 'view'), 'object_type') + else: + prefetch = ('object', 'object_type') - return results + # Wrap the base query to return only the lowest-weight result for each object + # Hat-tip to https://blog.oyam.dev/django-filter-by-window-function/ for the solution + sql, params = queryset.query.sql_with_params() + results = CachedValue.objects.prefetch_related(*prefetch).raw( + f"SELECT * FROM ({sql}) t WHERE row_number = 1", + params + ) - def cache(self, instance): - # This backend does not utilize a cache - pass + # Omit any results pertaining to an object the user does not have permission to view + return [ + r for r in results if r.object is not None + ] + + def cache(self, instances, indexer=None, remove_existing=True): + content_type = None + custom_fields = None + + # Convert a single instance to an iterable + if not hasattr(instances, '__iter__'): + instances = [instances] + + buffer = [] + counter = 0 + for instance in instances: + + # First item + if not counter: + + # Determine the indexer + if indexer is None: + try: + indexer = get_indexer(instance) + except KeyError: + break + + # Prefetch any associated custom fields + content_type = ContentType.objects.get_for_model(indexer.model) + custom_fields = CustomField.objects.filter(content_types=content_type).exclude(search_weight=0) + + # Wipe out any previously cached values for the object + if remove_existing: + self.remove(instance) + + # Generate cache data + for field in indexer.to_cache(instance, custom_fields=custom_fields): + buffer.append( + CachedValue( + object_type=content_type, + object_id=instance.pk, + field=field.name, + type=field.type, + weight=field.weight, + value=field.value + ) + ) + + # Check whether the buffer needs to be flushed + if len(buffer) >= 2000: + counter += len(CachedValue.objects.bulk_create(buffer)) + buffer = [] + + # Final buffer flush + if buffer: + counter += len(CachedValue.objects.bulk_create(buffer)) + + return counter + + def remove(self, instance): + # Avoid attempting to query for non-cacheable objects + try: + get_indexer(instance) + except KeyError: + return + + ct = ContentType.objects.get_for_model(instance) + qs = CachedValue.objects.filter(object_type=ct, object_id=instance.pk) + + # Call _raw_delete() on the queryset to avoid first loading instances into memory + return qs._raw_delete(using=qs.db) + + def clear(self, object_types=None): + qs = CachedValue.objects.all() + if object_types: + qs = qs.filter(object_type__in=object_types) + + # Call _raw_delete() on the queryset to avoid first loading instances into memory + return qs._raw_delete(using=qs.db) + + @property + def size(self): + return CachedValue.objects.count() def get_backend(): - """Initializes and returns the configured search backend.""" - backend_name = settings.SEARCH_BACKEND - - # Load the backend class - backend_module_name, backend_cls_name = backend_name.rsplit('.', 1) - backend_module = import_module(backend_module_name) + """ + Initializes and returns the configured search backend. + """ try: - backend_cls = getattr(backend_module, backend_cls_name) + backend_cls = import_string(settings.SEARCH_BACKEND) except AttributeError: - raise ImproperlyConfigured(f"Could not find a class named {backend_module_name} in {backend_cls_name}") + raise ImproperlyConfigured(f"Failed to import configured SEARCH_BACKEND: {settings.SEARCH_BACKEND}") # Initialize and return the backend instance return backend_cls() -default_search_engine = get_backend() -search = default_search_engine.search +search_backend = get_backend() + +# Connect handlers to the appropriate model signals +post_save.connect(search_backend.caching_handler) +post_delete.connect(search_backend.removal_handler) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 980d3dc7e..b1b285283 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -116,7 +116,7 @@ REMOTE_AUTH_GROUP_SEPARATOR = getattr(configuration, 'REMOTE_AUTH_GROUP_SEPARATO REPORTS_ROOT = getattr(configuration, 'REPORTS_ROOT', os.path.join(BASE_DIR, 'reports')).rstrip('/') RQ_DEFAULT_TIMEOUT = getattr(configuration, 'RQ_DEFAULT_TIMEOUT', 300) SCRIPTS_ROOT = getattr(configuration, 'SCRIPTS_ROOT', os.path.join(BASE_DIR, 'scripts')).rstrip('/') -SEARCH_BACKEND = getattr(configuration, 'SEARCH_BACKEND', 'netbox.search.backends.FilterSetSearchBackend') +SEARCH_BACKEND = getattr(configuration, 'SEARCH_BACKEND', 'netbox.search.backends.CachedValueSearchBackend') SENTRY_DSN = getattr(configuration, 'SENTRY_DSN', DEFAULT_SENTRY_DSN) SENTRY_ENABLED = getattr(configuration, 'SENTRY_ENABLED', False) SENTRY_SAMPLE_RATE = getattr(configuration, 'SENTRY_SAMPLE_RATE', 1.0) diff --git a/netbox/netbox/tables/tables.py b/netbox/netbox/tables/tables.py index 38399b5fe..9b86b2ed3 100644 --- a/netbox/netbox/tables/tables.py +++ b/netbox/netbox/tables/tables.py @@ -4,16 +4,21 @@ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.core.exceptions import FieldDoesNotExist from django.db.models.fields.related import RelatedField +from django.utils.safestring import mark_safe +from django.utils.translation import gettext as _ from django_tables2.data import TableQuerysetData from extras.models import CustomField, CustomLink from extras.choices import CustomFieldVisibilityChoices from netbox.tables import columns from utilities.paginator import EnhancedPaginator, get_paginate_count +from utilities.templatetags.builtins.filters import bettertitle +from utilities.utils import highlight_string __all__ = ( 'BaseTable', 'NetBoxTable', + 'SearchTable', ) @@ -192,3 +197,39 @@ class NetBoxTable(BaseTable): ]) super().__init__(*args, extra_columns=extra_columns, **kwargs) + + +class SearchTable(tables.Table): + object_type = columns.ContentTypeColumn( + verbose_name=_('Type') + ) + object = tables.Column( + linkify=True + ) + field = tables.Column() + value = tables.Column() + + trim_length = 30 + + class Meta: + attrs = { + 'class': 'table table-hover object-list', + } + empty_text = _('No results found') + + def __init__(self, data, highlight=None, **kwargs): + self.highlight = highlight + super().__init__(data, **kwargs) + + def render_field(self, value, record): + if hasattr(record.object, value): + return bettertitle(record.object._meta.get_field(value).verbose_name) + return value + + def render_value(self, value): + if not self.highlight: + return value + + value = highlight_string(value, self.highlight, trim_pre=self.trim_length, trim_post=self.trim_length) + + return mark_safe(value) diff --git a/netbox/netbox/tests/test_search.py b/netbox/netbox/tests/test_search.py new file mode 100644 index 000000000..1b6fe9eac --- /dev/null +++ b/netbox/netbox/tests/test_search.py @@ -0,0 +1,153 @@ +from django.contrib.contenttypes.models import ContentType +from django.test import TestCase + +from dcim.models import Site +from dcim.search import SiteIndex +from extras.models import CachedValue +from netbox.search.backends import search_backend + + +class SearchBackendTestCase(TestCase): + + @classmethod + def setUpTestData(cls): + # Create sites with a value for each cacheable field defined on SiteIndex + sites = ( + Site( + name='Site 1', + slug='site-1', + facility='Alpha', + description='First test site', + physical_address='123 Fake St Lincoln NE 68588', + shipping_address='123 Fake St Lincoln NE 68588', + comments='Lorem ipsum etcetera' + ), + Site( + name='Site 2', + slug='site-2', + facility='Bravo', + description='Second test site', + physical_address='725 Cyrus Valleys Suite 761 Douglasfort NE 57761', + shipping_address='725 Cyrus Valleys Suite 761 Douglasfort NE 57761', + comments='Lorem ipsum etcetera' + ), + Site( + name='Site 3', + slug='site-3', + facility='Charlie', + description='Third test site', + physical_address='2321 Dovie Dale East Cristobal AK 71959', + shipping_address='2321 Dovie Dale East Cristobal AK 71959', + comments='Lorem ipsum etcetera' + ), + ) + Site.objects.bulk_create(sites) + + def test_cache_single_object(self): + """ + Test that a single object is cached appropriately + """ + site = Site.objects.first() + search_backend.cache(site) + + content_type = ContentType.objects.get_for_model(Site) + self.assertEqual( + CachedValue.objects.filter(object_type=content_type, object_id=site.pk).count(), + len(SiteIndex.fields) + ) + for field_name, weight in SiteIndex.fields: + self.assertTrue( + CachedValue.objects.filter( + object_type=content_type, + object_id=site.pk, + field=field_name, + value=getattr(site, field_name), + weight=weight + ), + ) + + def test_cache_multiple_objects(self): + """ + Test that multiples objects are cached appropriately + """ + sites = Site.objects.all() + search_backend.cache(sites) + + content_type = ContentType.objects.get_for_model(Site) + self.assertEqual( + CachedValue.objects.filter(object_type=content_type).count(), + len(SiteIndex.fields) * sites.count() + ) + for site in sites: + for field_name, weight in SiteIndex.fields: + self.assertTrue( + CachedValue.objects.filter( + object_type=content_type, + object_id=site.pk, + field=field_name, + value=getattr(site, field_name), + weight=weight + ), + ) + + def test_cache_on_save(self): + """ + Test that an object is automatically cached on calling save(). + """ + site = Site( + name='Site 4', + slug='site-4', + facility='Delta', + description='Fourth test site', + physical_address='7915 Lilla Plains West Ladariusport TX 19429', + shipping_address='7915 Lilla Plains West Ladariusport TX 19429', + comments='Lorem ipsum etcetera' + ) + site.save() + + content_type = ContentType.objects.get_for_model(Site) + self.assertEqual( + CachedValue.objects.filter(object_type=content_type, object_id=site.pk).count(), + len(SiteIndex.fields) + ) + + def test_remove_on_delete(self): + """ + Test that any cached value for an object are automatically removed on delete(). + """ + site = Site.objects.first() + site.delete() + + content_type = ContentType.objects.get_for_model(Site) + self.assertFalse( + CachedValue.objects.filter(object_type=content_type, object_id=site.pk).exists() + ) + + def test_clear_all(self): + """ + Test that calling clear() on the backend removes all cached entries. + """ + sites = Site.objects.all() + search_backend.cache(sites) + self.assertTrue( + CachedValue.objects.exists() + ) + + search_backend.clear() + self.assertFalse( + CachedValue.objects.exists() + ) + + def test_search(self): + """ + Test various searches. + """ + sites = Site.objects.all() + search_backend.cache(sites) + + results = search_backend.search('site') + self.assertEqual(len(results), 3) + results = search_backend.search('first') + self.assertEqual(len(results), 1) + results = search_backend.search('xxxxx') + self.assertEqual(len(results), 0) diff --git a/netbox/netbox/views/__init__.py b/netbox/netbox/views/__init__.py index d880ba64c..0f35dab49 100644 --- a/netbox/netbox/views/__init__.py +++ b/netbox/netbox/views/__init__.py @@ -2,15 +2,16 @@ import platform import sys from django.conf import settings +from django.contrib.contenttypes.models import ContentType from django.core.cache import cache from django.http import HttpResponseServerError from django.shortcuts import redirect, render from django.template import loader from django.template.exceptions import TemplateDoesNotExist -from django.urls import reverse from django.views.decorators.csrf import requires_csrf_token from django.views.defaults import ERROR_500_TEMPLATE_NAME, page_not_found from django.views.generic import View +from django_tables2 import RequestConfig from packaging import version from sentry_sdk import capture_message @@ -21,10 +22,13 @@ from dcim.models import ( from extras.models import ObjectChange from extras.tables import ObjectChangeTable from ipam.models import Aggregate, IPAddress, IPRange, Prefix, VLAN, VRF -from netbox.constants import SEARCH_MAX_RESULTS from netbox.forms import SearchForm -from netbox.search.backends import default_search_engine +from netbox.search import LookupTypes +from netbox.search.backends import search_backend +from netbox.tables import SearchTable from tenancy.models import Tenant +from utilities.htmx import is_htmx +from utilities.paginator import EnhancedPaginator, get_paginate_count from virtualization.models import Cluster, VirtualMachine from wireless.models import WirelessLAN, WirelessLink @@ -149,22 +153,48 @@ class HomeView(View): class SearchView(View): def get(self, request): - form = SearchForm(request.GET) results = [] + highlight = None + + # Initialize search form + form = SearchForm(request.GET) if 'q' in request.GET else SearchForm() if form.is_valid(): - search_registry = default_search_engine.get_registry() - # If an object type has been specified, redirect to the dedicated view for it - if form.cleaned_data['obj_type']: - object_type = form.cleaned_data['obj_type'] - url = reverse(search_registry[object_type].url) - return redirect(f"{url}?q={form.cleaned_data['q']}") - results = default_search_engine.search(request, form.cleaned_data['q']) + # Restrict results by object type + object_types = [] + for obj_type in form.cleaned_data['obj_types']: + app_label, model_name = obj_type.split('.') + object_types.append(ContentType.objects.get_by_natural_key(app_label, model_name)) + + lookup = form.cleaned_data['lookup'] or LookupTypes.PARTIAL + results = search_backend.search( + form.cleaned_data['q'], + user=request.user, + object_types=object_types, + lookup=lookup + ) + + if form.cleaned_data['lookup'] != LookupTypes.EXACT: + highlight = form.cleaned_data['q'] + + table = SearchTable(results, highlight=highlight) + + # Paginate the table results + RequestConfig(request, { + 'paginator_class': EnhancedPaginator, + 'per_page': get_paginate_count(request) + }).configure(table) + + # If this is an HTMX request, return only the rendered table HTML + if is_htmx(request): + return render(request, 'htmx/table.html', { + 'table': table, + }) return render(request, 'search.html', { 'form': form, - 'results': results, + 'table': table, }) diff --git a/netbox/project-static/dist/netbox.js b/netbox/project-static/dist/netbox.js index 1213d719fba8177f84c950cec2ee3e75190db002..19cdae0bd318381cbc0701ff151e7f87ad3ae9ee 100644 GIT binary patch delta 8423 zcmahudt6l2*1xs)*@}`6L_|adMuZtX43OqE@Q65|fG{8+3Pd;z0}Kr3%!@Zv+P%H6 zX{YYiva~FF`3&u-J$%&dmd~`zURGM_wXb|+dAmKob!L#defRhK{+YGcW3RLJdhEUS zo^RGge!eDh%RNt!yvE-9o)`~gextf!oq<#|s&B7*0vxLE$y)HM??3rTh*gnFWusA6 zUwz6sx+#cwQ^i`hNt)T!YV*eCluA}LWBsf!XR6rHPmM2?n$+d%n?O;2TE7f@>b*~I z1ep6GN@5|H{A*y z>XgkpAiMXg&1C@9y_T&N043^UFUa9?K>s{<-;$bfUT;kAaDa9-Wt%^?(;4tVX{wml zue%0i6r)M(D3oON^=-F8p_;Y*dB{{RZTEvkb-bud%uf})&AOYs&1OftTk-mRQddup zDP4D4ad1gl>c3x1gwEbLO$6|&)*W$Uyo$Nr;TN-8Bpfx^e4?*eANd@9wQ6Uix^_p1 z;8jFFU;Y6^s;}*c#c5yc5b-x`=cI8uM>h4C(u_rllvQXn`204%!(b8%3M-4$MLP|o zNZ~6!fKfPe^uYW=qhA(_)U7)cLy8nrhF?aZ&vwQ~BGP9kDpZ=$7_tAKf*PK(6}Q%ZtI<8~@4@U`?1H&HPj{M5w!8 zwV_Dd?ok99)#-bef~WVTy&ixZHR-i|5U*Z(O%Cw{zcWXzd)*DQ)f2BTMn6seI|Ep# z&U~W^%GA|w#2A|NAT*~)9v#-FNSVzDEzMG8mD%0mYY?3ZkDX3O_FjJD9Ng+uj43E> zbEn}b)g;zAU4DmGN3J)CKBd{A52u(!N2|lzp^vBPGIFgxydgNum1`aCIzH{%^kPS& z!|vBfe==Q*ZfQ#(V-}=}SpgpzT6~UJAOD|Z2vuv|x}BA$iUobH`1LIg^LqLI=|J*Q z)uRVS4Y1-;(sR_zy^-qXrw6K`2jj68Qx5(QsYq4F9r|;W!y}gUd)|{GHTZDJIp~7k zU5AnYW&e5jTabHC9?1nPzr>^KvHT7nO&lQWdMi+GggX3K7ub65I~EHe3xmn+Qn$Sm zH>lHzs_dCkWA7*LY=RIa7y`Tc;BghJ)A-)%s5+0R)L*N8cc!EpD>i3J%DR*90o1CW zoQTAaUOjOD-0Hsf7qe!M=&lb0U8t?ca-8)+A~f|je4r3$Rp0;U!>A6A*wL?WPG+#G zW@lnm9r(BxvejEp7GSX+I$1-pSW`~LfUFjsdIZ|kuTD+C`W$}xF(_3xpN>SW|2Q26 ztk5II2S2EO^LaRQssp|##Jno|qBt@~kruR=Olh8aDJ~^ghB@_8j=K8`=GjiDwe=BL!zB zg=Kog+5PKrTW}qBoms;y9 zFbw$?3`5!ZasjJo%hyw}Ue0`d3=*v#(bak_07VwbXTty#TO>sfM584b5L+t`Gs6V_ zsFe=t9el|IkwqS{u76r5s@9XCu2ns{cdS}>SwihkUY-V6jURo}n(Fb0+5N(E(JDPH z=2;}K{;fq8$))4oI;mNQdHNccUJPp-QR{ZNhn4>0q|0eDL7r(y} z*Xa?n`YmhCm+ZO)HThCWE5eR^tO5E%0>vF-($DupS?{i&Zv$AMrvJJR%GFTr~@exKC2XlBz5Lk`yO4;oH z`%yjVY;DT}@Np=b{MX0e0ch1?*II-m7CL~G@n;06FEzkf5B#v&Jc0a-S6c^&>1uN*7O=30kc zsdZ#1{N2rP1zNN>x4<-lHvIYm7-1;RumxsdE*KYRH*AAS0ye(lMKGhidtQVYx}87o zfM0OxPdnjyKu0FO1l>j`;u}7P<1m*m{Q~YD-JpnW#qD-@=Q?DE*XCEeX-14&HJ@+> zPLC^cN|mKvvA`=@yke%eN^EyZdRyc58^pQXa26&*1Gk>V3Mk^PvoK@yTxXT3N35(e ziA_Ge53nn;-R3tI@jYiD8|G>Q&cTHNL)?l~VDmSao1L;)tic7i1)!MUb`hou#ZIG_ ze{CjXwRbN9_8ZOGhnHY6wklfs6-dWTQfvDb7{RdjTm=a;^`)y21#W))DrRb{_VZP+ z0~%xd9_~k(%ikl~tS$IA${<(y0bC>;KOxJL6xo!l$lcv3Vn#$-*CN~fPDPHL$6*M07-d%vA=8Cs}XwivqG6P$3J~D!MMix0uCb3$UYW0L`P%<*@X$^c;1o<8X9vDS7 zU|DBG5{{10PK+j&VNkC{8wsXFGhZ}`+=HI@d=e31V5?$u%NcGyN+eFO^4mmW8ftY; zZk8*vY*k{OB3Zd6l8LxkoD|9RBdpG(0=aULQk9e&1a&+viG(A4a*#f`pFSsv#E!H& z#eAPBE%=oJTbjiuT6rgOsMXoS<0g^_{z4KthOYdJnasuvc+*Vc5PWGS7tu=}CX-6& z(59u3e+W>m{W6vO2x;gwmuyON$$Dz}9qs;Xm!sJs`!fQJKM+l6l~sM|oiT4Ab7+=tl;5aTF5-are%@m zAWu7$MZyFq;a74<27)QMIHg2;ESG#gbkcmMBsKg! z^t9;ip3nCzB2h4(f4GReHRLx=Lc)B$rJPJg&d23sG=lHSF$VK_bOl)eZQN6VA8q4X zE66{gO?$YKTqbDteK(W(kcf6gGRWu@gIJ0qx7X2X5DR%oJEH zkVt-e8`(ElCwS#bMXutfqS;9P$XGU5Yi=i#2!3H@Ct94TUFal7@xy#iH;F{>em8MM z>ho+EM3>Fy%Xj+xW?OBoF;n56c4JmJd&o2><;5u6SX zr)*wLi~*S?_mVhWcGzAr;cu(SVE*dsY!JVG4cW>(PUAc~|Jq0meDGQli)Au-E%{{t z&Ytu%*@bc7XPzNv5k2@Ud0`;3Z{14H;41)++eX59doStKE2Sbe9zy=4Vrr==@3q-2 zWE5X)pc8oe0aDs06~UWpNGQK>fP{jB|8Rg9A)Ze-Na7LX93+pI#OsmjXb%PkBN4!} z5y!|m`p{ww#I-;M_l-~SXD$P1(Bd;t|_@gt%1 zTPV<8zmDF_aL=4Df>xpZOGeQB!l;74cJYU@FW|OFItmJSdnBzu@J=Kx3PDMH2k9~C zc^J-Tj;8go#icr<+wYkk^STFrDKx+#ipiKCzQ zBby9#DCF{23^W!S_D>9S{H=5JK)07li<=dEB~b9`A-*d>EA6MX`1?qONa~MoKq&JX zMd=r^ASsw60g3z|8mRUFAqxq?YS+;rVnH8{DKm%PHGyu0+5GX-BtcsoPwx`YZRd^j zL5jrrw_qX{hSC;3D2WaoG}{@xLwk5^5*>rhczP1O89MpHN%UcC)Ne4;Y`tGM(^>?d znd!5*@OzS}c~Ic0YHyW_wY|yI2++j8Oreu-W<)C8Uly;#I_1~~o5@*c%yL%7D^++I zZ?NeDli%CXRil%|ctz}^^BL2wVyKR zp9kT3mgLfRgCLwc^XN*lP~lKQMsrsl4ddcG`U46l=F^*W;Xf~+rv`fBr4FZYHea4h zNAaa)G(0TuW>sDnyt+GcC9Af+jGhME|MbQ`SCaXPMfAs@ScfktmaU6DT29pg*i5G` zrmG0T12t3&4e=>PMK(KJ{4p0bYdd6mWgzl4w9{+pdW2%Yv_151$hYpL zmvy;WZ_tl1j@r04=|q5B&GHt-*SuW5eLuZ>TrRfLvS`p#4j&>BbNmkDYY);$?ma-8 zbUFEVod2AKTjc+zN<&roFu|dux8a+e(q=AF+Tbnu4pCb(ks$ zwj9Qi4}g(8<_H}+!Vzepe`}gu3NI=l5nAaH8qKf}-0xB!PWa+onvRQ&Jx;B95-ab~ zm5{|lPSC&d58kJ1NM{F!*M$k%%nzv@_<@h;LzqXlk7>{Nyj-*2;nTY^Q^tZ^9i;i? zz{&2ArE>n$$MgZb2;6^?-U;(~k-GRPy~OaE z_1PI}6v}cd7xL@QQYV!2&1Y#0f9NcYLPa>9iHa_qr8ZPlbdHvgyj=d&Ihu$HwZrG= z{Xi;mdG&dEH(Gf9Je7t!tbvN^y9*+7+LtuW6r^StCZwdK7}5+A>gwtYfhwEZA^EN8 z{kIIicH5UU=@F>YHvL5yEFj!@w=i}%CeVAUgnJ0uZCfq41-vF6TPqCF6YH}lgj9-f z#Cjns1mVI>!X2Pz-lv;|*Ywm{w^g`Phm(5+d}wUa$abL<9~^nli^9W$%JkGXnYvtB ztR|=ggRyO=a3j=e7j_Ei3{iQvFp(GT7RCq1Q}L3rZnv-sF=>yGh1k+P!ZdL6s=Y!o z|K}dTkJl6&pGJXM`-FuER_+tTNVgS}SHDgvUZdWJ`Q=!pF1Cq(v`;ASlfmY`<~6~q zUz?8qT}Vahngc=+P)O1H@PoqjP|7D95*Fahjzhv!?0_~M62^sB z|IQwK6$mW%lS9Hb6oZ2-f9=Aknbn6MnThu4o`?ltjWjtT$7mydJr2oHoqxz>43ILHv6`?XL2(83YM z^PLxjvq)Qe5hFID+*#QsSK;e&KAvUO@?@!8d*_mHS4ddCRhp8L@>_Xjao4|bLW{QS z-{>DK2Kpm@3gK5j33yL!(zXHXqGDBEb&#t&d`4}g5_;&to? z#&oWgUl_q=f>lcjX9Y0QDofRNqroZLy@7kJ!IW<0%SN(Tl-N9yVMk@Gk7S1+mKf(P}nEzy~oaKQ)fc@1sQVg7NH4ea-m2F){32oNzIQWgtk7WxtHa zm6ym#xF&mx*NZp$0=(Zhq+8`Uv=x0YFOE$fXO*kOoc|_=`D@kGIQ9>M>MG;e%Lsm) z$P7GfB1;R7WB7v;SzU14h2CsSU^j#yiryT>Uo*2v^u5`JPhoPcrNQZ{HI@V{y zC_OxxeIAaUL>uw%Kzw_s^`*OIBf2pQ{ilUm*v^5d?C*JOJ}wa*#gE?4Mrz6P*lPmP zuN1P9NYw9g$o06Yx$@-6rY>xqLh-xX{%c#a#oyRF*K zKGuz9YMpIt2O6YJ>0r}!+||uQebTC1*s}w5i^MpqmU}CE249)AZ*F7v!H7DgvP9N< z$yUW#8=I1lP{)@qWf@^eZ~YxTqmF;JlvU%RXWh<@Y~n0uW{^I%W@ICw3dxWi#}Y-2*Eqb$tTz(#veZ_grhs7pJC}F*UGo{ zQXO?Y!)7Dc{0yrk)_!X2vut{hIhuQ(W%?l%`W$Nl7eBv&rD#t-ht3bd6R#GV2N(bR zdG?ErH3Tq}@SuF80& z^a3l#o!5fNWpt_b()0U^yX@QBDmP`Dra)HaeoO8Jk-sEGN z9@dz4oVGb-mSr!WVI8$H&Bj!GWZ7tzQ>ONsG#$&DrpbSudy%Y}@B2T#@2gm6SI2EZ_ zHklN4_cM;+1wq7{Bv!l4()9KwtH+RCEIHNG4KqTmNn%|uHKtf9P*-elf=m5r!*Z~x z4?MdG{OTRgTZ>I(<_bw5p3YsiB!kVtua?j(ka0V>T~`O0{eA zA}CTPZP@|MU0-i00chz;-&O`ttgd}Y4)X-opX2V?QUl6$MRg4ZNK+HH`wa6P0UM+z ziL-mxu0kHgWENZVC6D^n_C=7ZX1x3&WT=;4_CcO%e?=LWlO%c?bTfGx66`H*#pClz z?VX+G6y0p~!7W+Tf4ve1^Sh!o5zyk69nm8kNG=1z06sZfzJ@=Zps&+4-q&0=1Dd7)am z(?|*xzVZVYhLXek=H{DxvRJ5Y+Zh*9sF+iIGBSO#GbRF&9zAhEr2&;eEs$~Ku9=3$ zpytjb(I42IzMF!4DaV7>a5YHt)y`c}P~Y|Vu5knk)P1ilguJQzcXL0dCLtY>dChj!o04j ze`Np*)am;xV3zvW{wQOC?u3R!DO-oNiBehv!t@5IyduHv_tuGu!VOdD(5@@{KZiw% zVoF4At2-G-NoKLe;quu%I&zy?^ePQ@eK^T1+MDd2R((8K=aH-R;qAd;j$CbT(eWua z#0%~9cAHNp{my*DyT$%M#LP(&GXgd;`n`68m;YlT3{a~M{(+SyiFrMyxPFks+%A4- zDiC|4dc1pBA17`lC0k9~8^NQVVj=1+hsT1fCLaC=ksH;~NA8aFG>Rp?rf*D?>b$t; zY_vhwt|Re!7=3Z{TTr^r9LoXBzqsQYF#nDokL%;n7cRqv!_~p>wu8Ux!FLT1QWA{r z8g=`7(f#H-a4B1w)Y$d+_clX_H|T?Eb=CVSrl;v|k43s0MWyye^1IU{U0bmsP4cck z^EZG7_3tMmz^h(6*$qwVo2M4Csz%XW8}PbNU5oiR z0#|+e(@zgd5>pTW|leLfciXQL?p8DN-@ zpDvZCv0ugo-H@A(ZYcS3fq-eW_2OjAsS6k1g}A~-(baUr0R`!j*NP4(M5sHWK0W9V zYZDJmfU*2>3+>m{|FRh(8XLu$Ua>Y@S`UVrCiVE= z9j~Kz;4t~S6#Q-a?)Uin-gi~_JLTG@@Oh33^heKuQ`8hlotf$n*H+_6u=4v==)%|^ z4&V{+%@0?j=QWBMy_z-UN;X}Cs$8k431Mq4<^cUMmf{gH{yz`Ftgc=ESqv~oO}YLi zl&U{p9~BwcNq%ndxQc0#EPj+r%as~mo&;lA{tZq4t_NwF)eI|WXq$qkqTHEQlaOw;)!C&s zPU7Jcb6t!WQ*|R2Tl1u5-I|$sQs7js%###7UW)RhY;8aSECq1$mspU>e^pMedElxOwTB{Vo>v0j5qa2H@E zE&c{33-u0@hhIz}qqGy>09F{a+Ud)%kU*oBaurelVe_}Z2sH7|Yar=ydkrF?fxmwZ zBehEV&o!_CDr5Zt9zvcgKOkDG&HESf;H>f^xJVd&LYBuXvU#E+cXT9*t;zM;rJuk` z&?W`fVI`{h>2a~PKJ?bde#mNCyhf39cHswDNEIQz`2#wG+VNpKN?Pc zK!%5hk&WQiQX>dQLue<5lk`E*q>VHY42fD^I-V>;OPm`|MCedqpR2!x`#ybn@tNB%Hq#Pu@jS{xN~f#0WT$K%xB?PIk$9X!-0dzD$?B!7lq!0~5a?nv=^bdcr%o zRGaiWvO*82x2BSpDV_ms)5s_Ud>Wa7%b%J?&I&MF+m%M@0p{{?>BNuVFX_Yr+5D?? zG6%6K8RP|+tDVgtp#mO+SF=eff=M|j(yXn`AwS$YzRV%X_GUw#)h)JQv4YJ)Ft;&L z&Fy6ly^qktqNAgX?Ne(UqFktrkq?MsP=<*lG>2)Vns5_XcVKEk0ZCo-eeThct|bEr)deb1pByj zZF(Kqh7}yY>?8{yUz_bB$B9s^Bjnv!*l&dKZa+Y z^uX=Iuk0Za{J{|{gg?+scJNZq6UAxpqj^o$) zo(>X$;8X{(N9b}^^rXw`_2xRfz65J^wJBZUA9rAAI6BD`NaO1|$%ybYB{g43Mjd56 z;t>m|d#sZfNwLD$c9N(eX-e`;Z*V?eNyTBB7IPQ53}~D1#bh;sTy67Gl1yo?V&<+-Urp|W814SmWC_r@4ik5;K?ldIA${E*Z(|A0rtkvgI*i3h*p>fJE!OgZ7fKk32^D^DS?(e*Cs4$TpVkFwL>?izaI1 z{nrr#rpbhLYGB5gRn+J)6j zfV2=ZYUy?ou3L5W%j6zJqh28+DDf$L&m-h^&ApSf_JM5et2fB<{yKF~B%Q%`9U>k5 zX5-0Q=kS_S_@39uFuu@8$MTkLQryE8&Ks)80Dh^P3;-Yhv74A6i;q1_Vi05>CQlY+ zInbD`EkVbiCjxjT;^;Yh4_b(hxZ%jayw4>qtWj)!uj7V9B}(-8E{jh)LPm`YFg7C@ zy;Q9<<1O#-HaNUqT$6i`kQ|Z`+#JWvMe~~DBvc!Bl+0te&Gz@mV8W*kpecO+`{XG! zr{ix3GWabgNu(}^+y7-_Aw2&i5qrqP2FvCIJ;qFCvpJ>CVam|fpCslX81F~EBujN) z6kH({dJ(tzDyb1#@Qs6SyNadd z9zQmKehWF;Ter|V8J?75htLYtddU!aNFX_wM~8(l$l=xqIt)Qe1T91GUIZ-&$x-l* z#6LaY!ua&zv=(PJ45u-=*h(u&(*{P;T0$%e-&#Q?@Fz#oRhZOMM$t;96LH^>d}}m4 z*Nbd6(p$m8UpG<%R_1><(lLu>>K<(=mKHWB_yC~bt3phBfL7c~^ZR;8`AF(@WPmB{ z21V&*N=H&K8Uh@-K{Sxo0YU~6f@!Uz1I4@^9Ajf9zi%wv25tPwPf4t{FoxbIK(_X! ziLRnZoO>5WZ2kc1=l$a8t^L{@!R|PV8{+8*th!U;>76j2uZgE?up+-bfoAH3c>=9Q z@JRxF9t!xfi8P^Kpb51#N%h*^iPWUmS6?O4@hBOdL=Tl@DF%mZsI!_KHKq(ld6rUv zZFZeiADDff*7ho$EM_TU53LicI9r{5akU58dzd>rJM~wm${JB;*KWU!ehK)EsL6Cm zpZQ8hhiSgHVKV(mpe2gLJ7<8AzkLUtJE%lSkgZLQTD{(kjV;mQlIimd<&LN7Dr!Ha z(!2ZNewO6W6G0Hh9kc0bQlfAuBEz|BHVx(C9Qq?N$K}%B>&$o0qi6eO#Yn9V(@ef% zA|1w;me8=!z;o1snqX7z$dR1dh7$TI^b5QhT5}{1Us+239b{|u2H7%ow#OGxwGUXe zq=ocRg0Q=aY6C*7ib;_Z>@L36MH93gGQHXt=QcLc+Cf;&k6Ta2Yt!$cA5tWaSVpBG z2umNMc71`}YiW&u5Sh$_V4jx$5+^>mW*dMU+<)$7B)bkI?ni0aKaN)-fK zk7CLPz)&7_jNUrL7pRqgsg7+5FDN46TJbSDl3^maPf#xkoIgQRaI=Q@sYTzc@;AB~ z@_5Ke`ln%SIlR{Y9J1T*yO~RWBtQQL`rbT*kxrG=jBRR#!A@c=_PEsUdhTneCY}_k1jdcnCbB-9wRE4 z5cr_vO~RsAU&d(s^)kjZm&=e}r9XUnZ{jx^&3xHO`oiz#=a|gxHHycimo7fppwt*F zvdMV!R)Yq?&za3P7-fEZ2aOzUbU6&u4OM=h4;88QC~p1Eig)5S<=r16A;a*|XU`+P6xv-4jX;8REa0}R^?^`Pj)DMB<>x3kV zkUlMBgdoh`DBJ`3De=xGVV|DTuLH0iSy<+SQkYHobq8w+n0f&C`!5v$@@- zg}yAP1QX{OO}GQw%8xZ6g&|t`x-gEl#NZKRJM{xfhL5yg!p!;>-|2o3>;XT5F9vqDhxk6^`}2Om5|(Mf$I+hE3%=t{f6(KtA7kNJzxan~w;$4o5j- zwbkP^>Vb}pP7q1;^T&?}`B>^9rw?Cws*VaP2j>Su&)gGwenq_;8qS9w69(FTStZ40 z_1ZCsagWC2UQSaF#T=ZL71{njXRGZsR=>;lKLk7ZrDxb^-0g;ALNes?v&V$Qu#B6G z5;$ne^LWfV!U{afUw#K;&%(cXNB9C?r%oLg9uC8#bY2h+GsLI=4g+PM407UXdet%;PmxTMQyV0(clu!M`bmHGBV+ok zlYcyzO$VnoCY0sDP^TlV6WuWrLkEw)t2c z1i0`Gj5~(2IGkKLoQ*`VX*l}`!Lmp;akvvtRQ!h^9^F!=EFu=;kXmrSk4Lim1YF3; zD@L$LWJf}{-ck>XL@uATD)5glDLzRovY9gVYsTc{ua01e#3}Pje_$h-X@Lhx14pvi zC|x>|mFY9fE7%BzkCZt4@Buc0UmVG%^w1;u^ik}9zH$8MfYIy(ihVMgr6MqlVLu~? zielj*PQ1-pWZe0*C^liVQ?3xRWvNJx#~s`J9uMANd3cf4Vca^kyQA3K1O;=A>@@^m zk7Y(aVJu4yj-&V=$FiE>xE*a+7sGB3K@@E`jBgiN1X?`7im#G#b$Xq{Rc$H?sP?hQ zqLCZo*|{*ZAATFJF?$&{*ggU2 z=kr)m1g`J0%eB6`lv(nG3FdYzgK;xX&0=A#dYl}X%Z}iu9w}tcGOVblm$F3|MY~Jc zeSovG%2*bHI%W(k-W+hle~H(p00vth962Z6N;i!5VDhf}-M$U0C% z&FN=5P!}zxnN8L4iZ&+dLd!bX^L=$?#Av6M+{vB;d;vap7kdzf)G6gfvR>#nDUNDG zVr*<3U$U5`h9bS`CVFZeKfah%;(n8sutFgO7sD_UGx;rPY^1i~kF0Vi&h=TtEV`v8 zKF*@J?{Rh!$5m@t4uXwqamP-rdo3e)^JqucvGHuE)AHNv9}mr2Hn0>@VBy=kXc!(4 z6&u)01nV}ia#Gk!4SkkP4NeZ{RnM|Vp`QQrEOUa5pMHTQYO9|^FNENQSB+(Yjh}d) z{j6hk0Sq}BQ8xEQHfepYbnVVkZ4w8`8xR@zT}Ocd!oKB71hR`*b{K5Bs+c zGxxLm@rj!sJHX7S?Y|DNxd`SQWPT*>J%}M<;r~3yDi9PLV)x>U$f-l@cZdz{W)=jM a-7Fr7f9htx$M==5x=}TR(MQ;u%cC%>=ZWA{i89 delta 479 zcmYk2y-EX75QT}G2M{Ekpx8Q~;!kYPo$Q7+EbFqaG3BzTNn90wKur^ih>d;lJpyU1 z6R=L>yEt=Kqs=h)yEA9b%-ggw{;WI?N`r-wvoh(og9B;-mrQja)6ijJ9OWK`NL&R0 z`9QPCGH};udTwLUme7s?S8K{oHC$V@gr4TuD#w%pDcuOYw-rt5*WsQK0G}Gbg`oh5 zH0@~0RrMl&q8TfdsS;Cp^CSP-;uHPPC;?hx8X;xCAw|HQk;rXpEG?vcxVPr7bM;@V z?6zgM`YCN7**3$qiSyFWS*@=dal=)kLz*hrQlWiA53