8927 refactor search

This commit is contained in:
Arthur 2022-09-27 08:34:29 -07:00 committed by jeremystretch
parent 4fd183573f
commit 085883a5ff
11 changed files with 205 additions and 338 deletions

View File

@ -7,32 +7,29 @@ from utilities.utils import count_related
class ProviderIndex(SearchMixin):
def __init__(self):
self.model = Provider
self.queryset = Provider.objects.annotate(count_circuits=count_related(Circuit, 'provider'))
self.filterset = circuits.filtersets.ProviderFilterSet
self.table = circuits.tables.ProviderTable
self.url = 'circuits:provider_list'
model = Provider
queryset = Provider.objects.annotate(count_circuits=count_related(Circuit, 'provider'))
filterset = circuits.filtersets.ProviderFilterSet
table = circuits.tables.ProviderTable
url = 'circuits:provider_list'
class CircuitIndex(SearchMixin):
def __init__(self):
self.model = Circuit
self.queryset = Circuit.objects.prefetch_related(
'type', 'provider', 'tenant', 'tenant__group', 'terminations__site'
)
self.filterset = circuits.filtersets.CircuitFilterSet
self.table = circuits.tables.CircuitTable
self.url = 'circuits:circuit_list'
model = Circuit
queryset = Circuit.objects.prefetch_related(
'type', 'provider', 'tenant', 'tenant__group', 'terminations__site'
)
filterset = circuits.filtersets.CircuitFilterSet
table = circuits.tables.CircuitTable
url = 'circuits:circuit_list'
class ProviderNetworkIndex(SearchMixin):
def __init__(self):
self.model = ProviderNetwork
self.queryset = ProviderNetwork.objects.prefetch_related('provider')
self.filterset = circuits.filtersets.ProviderNetworkFilterSet
self.table = circuits.tables.ProviderNetworkTable
self.url = 'circuits:providernetwork_list'
model = ProviderNetwork
queryset = ProviderNetwork.objects.prefetch_related('provider')
filterset = circuits.filtersets.ProviderNetworkFilterSet
table = circuits.tables.ProviderNetworkTable
url = 'circuits:providernetwork_list'
CIRCUIT_SEARCH_TYPES = {

View File

@ -20,140 +20,118 @@ from utilities.utils import count_related
class SiteIndex(SearchMixin):
def __init__(self):
self.model = Site
self.queryset = Site.objects.prefetch_related('region', 'tenant', 'tenant__group')
self.filterset = dcim.filtersets.SiteFilterSet
self.table = dcim.tables.SiteTable
self.url = 'dcim:site_list'
model = Site
queryset = Site.objects.prefetch_related('region', 'tenant', 'tenant__group')
filterset = dcim.filtersets.SiteFilterSet
table = dcim.tables.SiteTable
url = 'dcim:site_list'
class RackIndex(SearchMixin):
def __init__(self):
self.model = Rack
self.queryset = Rack.objects.prefetch_related('site', 'location', 'tenant', 'tenant__group', 'role').annotate(
device_count=count_related(Device, 'rack')
)
self.filterset = dcim.filtersets.RackFilterSet
self.table = dcim.tables.RackTable
self.url = 'dcim:rack_list'
model = Rack
queryset = Rack.objects.prefetch_related('site', 'location', 'tenant', 'tenant__group', 'role').annotate(
device_count=count_related(Device, 'rack')
)
filterset = dcim.filtersets.RackFilterSet
table = dcim.tables.RackTable
url = 'dcim:rack_list'
class RackReservationIndex(SearchMixin):
def __init__(self):
self.model = RackReservation
self.queryset = RackReservation.objects.prefetch_related('rack', 'user')
self.filterset = dcim.filtersets.RackReservationFilterSet
self.table = dcim.tables.RackReservationTable
self.url = 'dcim:rackreservation_list'
model = RackReservation
queryset = RackReservation.objects.prefetch_related('rack', 'user')
filterset = dcim.filtersets.RackReservationFilterSet
table = dcim.tables.RackReservationTable
url = 'dcim:rackreservation_list'
class LocationIndex(SearchMixin):
def __init__(self):
self.model = Site
self.queryset = Location.objects.add_related_count(
Location.objects.add_related_count(Location.objects.all(), Device, 'location', 'device_count', cumulative=True),
Rack,
'location',
'rack_count',
cumulative=True,
).prefetch_related('site')
self.filterset = dcim.filtersets.LocationFilterSet
self.table = dcim.tables.LocationTable
self.url = 'dcim:location_list'
model = Site
queryset = Location.objects.add_related_count(
Location.objects.add_related_count(Location.objects.all(), Device, 'location', 'device_count', cumulative=True),
Rack,
'location',
'rack_count',
cumulative=True,
).prefetch_related('site')
filterset = dcim.filtersets.LocationFilterSet
table = dcim.tables.LocationTable
url = 'dcim:location_list'
class DeviceTypeIndex(SearchMixin):
def __init__(self):
self.model = DeviceType
self.queryset = DeviceType.objects.prefetch_related('manufacturer').annotate(
instance_count=count_related(Device, 'device_type')
)
self.filterset = dcim.filtersets.DeviceTypeFilterSet
self.table = dcim.tables.DeviceTypeTable
self.url = 'dcim:devicetype_list'
model = DeviceType
queryset = DeviceType.objects.prefetch_related('manufacturer').annotate(
instance_count=count_related(Device, 'device_type')
)
filterset = dcim.filtersets.DeviceTypeFilterSet
table = dcim.tables.DeviceTypeTable
url = 'dcim:devicetype_list'
class DeviceIndex(SearchMixin):
def __init__(self):
self.model = DeviceIndex
self.queryset = Device.objects.prefetch_related(
'device_type__manufacturer',
'device_role',
'tenant',
'tenant__group',
'site',
'rack',
'primary_ip4',
'primary_ip6',
)
self.filterset = dcim.filtersets.DeviceFilterSet
self.table = dcim.tables.DeviceTable
self.url = 'dcim:device_list'
model = Device
queryset = Device.objects.prefetch_related(
'device_type__manufacturer',
'device_role',
'tenant',
'tenant__group',
'site',
'rack',
'primary_ip4',
'primary_ip6',
)
filterset = dcim.filtersets.DeviceFilterSet
table = dcim.tables.DeviceTable
url = 'dcim:device_list'
class ModuleTypeIndex(SearchMixin):
def __init__(self):
self.model = ModuleType
self.queryset = ModuleType.objects.prefetch_related('manufacturer').annotate(
instance_count=count_related(Module, 'module_type')
)
self.filterset = dcim.filtersets.ModuleTypeFilterSet
self.table = dcim.tables.ModuleTypeTable
self.url = 'dcim:moduletype_list'
model = ModuleType
queryset = ModuleType.objects.prefetch_related('manufacturer').annotate(
instance_count=count_related(Module, 'module_type')
)
filterset = dcim.filtersets.ModuleTypeFilterSet
table = dcim.tables.ModuleTypeTable
url = 'dcim:moduletype_list'
class ModuleIndex(SearchMixin):
def __init__(self):
self.model = Module
self.queryset = Module.objects.prefetch_related(
'module_type__manufacturer',
'device',
'module_bay',
)
self.filterset = dcim.filtersets.ModuleFilterSet
self.table = dcim.tables.ModuleTable
self.url = 'dcim:module_list'
model = Module
queryset = Module.objects.prefetch_related(
'module_type__manufacturer',
'device',
'module_bay',
)
filterset = dcim.filtersets.ModuleFilterSet
table = dcim.tables.ModuleTable
url = 'dcim:module_list'
class VirtualChassisIndex(SearchMixin):
def __init__(self):
self.model = VirtualChassis
self.queryset = VirtualChassis.objects.prefetch_related('master').annotate(
member_count=count_related(Device, 'virtual_chassis')
)
self.filterset = dcim.filtersets.VirtualChassisFilterSet
self.table = dcim.tables.VirtualChassisTable
self.url = 'dcim:virtualchassis_list'
model = VirtualChassis
queryset = VirtualChassis.objects.prefetch_related('master').annotate(
member_count=count_related(Device, 'virtual_chassis')
)
filterset = dcim.filtersets.VirtualChassisFilterSet
table = dcim.tables.VirtualChassisTable
url = 'dcim:virtualchassis_list'
class CableIndex(SearchMixin):
def __init__(self):
self.model = Cable
self.queryset = Cable.objects.all()
self.filterset = dcim.filtersets.CableFilterSet
self.table = dcim.tables.CableTable
self.url = 'dcim:cable_list'
model = Cable
queryset = Cable.objects.all()
filterset = dcim.filtersets.CableFilterSet
table = dcim.tables.CableTable
url = 'dcim:cable_list'
class PowerFeedIndex(SearchMixin):
def __init__(self):
self.model = PowerFeed
self.queryset = PowerFeed.objects.all()
self.filterset = dcim.filtersets.PowerFeedFilterSet
self.table = dcim.tables.PowerFeedTable
self.url = 'dcim:powerfeed_list'
model = PowerFeed
queryset = PowerFeed.objects.all()
filterset = dcim.filtersets.PowerFeedFilterSet
table = dcim.tables.PowerFeedTable
url = 'dcim:powerfeed_list'
DCIM_SEARCH_TYPES = {

View File

@ -7,12 +7,11 @@ from utilities.utils import count_related
class JournalEntryIndex(SearchMixin):
def __init__(self):
self.model = JournalEntry
self.queryset = JournalEntry.objects.prefetch_related('assigned_object', 'created_by')
self.filterset = extras.filtersets.JournalEntryFilterSet
self.table = extras.tables.JournalEntryTable
self.url = 'extras:journalentry_list'
model = JournalEntry
queryset = JournalEntry.objects.prefetch_related('assigned_object', 'created_by')
filterset = extras.filtersets.JournalEntryFilterSet
table = extras.tables.JournalEntryTable
url = 'extras:journalentry_list'
JOURNAL_SEARCH_TYPES = {

View File

@ -7,68 +7,61 @@ from utilities.utils import count_related
class VRFIndex(SearchMixin):
def __init__(self):
self.model = VRF
self.queryset = VRF.objects.prefetch_related('tenant', 'tenant__group')
self.filterset = ipam.filtersets.VRFFilterSet
self.table = ipam.tables.VRFTable
self.url = 'ipam:vrf_list'
model = VRF
queryset = VRF.objects.prefetch_related('tenant', 'tenant__group')
filterset = ipam.filtersets.VRFFilterSet
table = ipam.tables.VRFTable
url = 'ipam:vrf_list'
class AggregateIndex(SearchMixin):
def __init__(self):
self.model = Aggregate
self.queryset = Aggregate.objects.prefetch_related('rir')
self.filterset = ipam.filtersets.AggregateFilterSet
self.table = ipam.tables.AggregateTable
self.url = 'ipam:aggregate_list'
model = Aggregate
queryset = Aggregate.objects.prefetch_related('rir')
filterset = ipam.filtersets.AggregateFilterSet
table = ipam.tables.AggregateTable
url = 'ipam:aggregate_list'
class PrefixIndex(SearchMixin):
def __init__(self):
self.model = Prefix
self.queryset = Prefix.objects.prefetch_related(
'site', 'vrf__tenant', 'tenant', 'tenant__group', 'vlan', 'role'
)
self.filterset = ipam.filtersets.PrefixFilterSet
self.table = ipam.tables.PrefixTable
self.url = 'ipam:prefix_list'
model = Prefix
queryset = Prefix.objects.prefetch_related(
'site', 'vrf__tenant', 'tenant', 'tenant__group', 'vlan', 'role'
)
filterset = ipam.filtersets.PrefixFilterSet
table = ipam.tables.PrefixTable
url = 'ipam:prefix_list'
class IPAddressIndex(SearchMixin):
def __init__(self):
self.model = IPAddress
self.queryset = IPAddress.objects.prefetch_related('vrf__tenant', 'tenant', 'tenant__group')
self.filterset = ipam.filtersets.IPAddressFilterSet
self.table = ipam.tables.IPAddressTable
self.url = 'ipam:ipaddress_list'
model = IPAddress
queryset = IPAddress.objects.prefetch_related('vrf__tenant', 'tenant', 'tenant__group')
filterset = ipam.filtersets.IPAddressFilterSet
table = ipam.tables.IPAddressTable
url = 'ipam:ipaddress_list'
class VLANIndex(SearchMixin):
def __init__(self):
self.model = VLAN
self.queryset = VLAN.objects.prefetch_related('site', 'group', 'tenant', 'tenant__group', 'role')
self.filterset = ipam.filtersets.VLANFilterSet
self.table = ipam.tables.VLANTable
self.url = 'ipam:vlan_list'
model = VLAN
queryset = VLAN.objects.prefetch_related('site', 'group', 'tenant', 'tenant__group', 'role')
filterset = ipam.filtersets.VLANFilterSet
table = ipam.tables.VLANTable
url = 'ipam:vlan_list'
class ASNIndex(SearchMixin):
def __init__(self):
self.model = ASN
self.queryset = ASN.objects.prefetch_related('rir', 'tenant', 'tenant__group')
self.filterset = ipam.filtersets.ASNFilterSet
self.table = ipam.tables.ASNTable
self.url = 'ipam:asn_list'
model = ASN
queryset = ASN.objects.prefetch_related('rir', 'tenant', 'tenant__group')
filterset = ipam.filtersets.ASNFilterSet
table = ipam.tables.ASNTable
url = 'ipam:asn_list'
class ServiceIndex(SearchMixin):
def __init__(self):
self.model = Service
self.queryset = Service.objects.prefetch_related('device', 'virtual_machine')
self.filterset = ipam.filtersets.ServiceFilterSet
self.table = ipam.tables.ServiceTable
self.url = 'ipam:service_list'
model = Service
queryset = Service.objects.prefetch_related('device', 'virtual_machine')
filterset = ipam.filtersets.ServiceFilterSet
table = ipam.tables.ServiceTable
url = 'ipam:service_list'
IPAM_SEARCH_TYPES = {

View File

@ -1,7 +1,7 @@
from django import forms
from utilities.forms import BootstrapMixin
from netbox.search import SEARCH_TYPE_HIERARCHY
from search.backends import default_search_engine
from .base import *
@ -9,7 +9,7 @@ from .base import *
def build_search_choices():
result = list()
result.append(('', 'All Objects'))
for category, items in SEARCH_TYPE_HIERARCHY.items():
for category, items in default_search_engine.get_registry().items():
subcategories = list()
for slug, obj in items.items():
name = obj.queryset.model._meta.verbose_name_plural

View File

@ -1,30 +0,0 @@
from circuits.search_indexes import CIRCUIT_SEARCH_TYPES
from dcim.search_indexes import DCIM_SEARCH_TYPES
from extras.search_indexes import JOURNAL_SEARCH_TYPES
from ipam.search_indexes import IPAM_SEARCH_TYPES
from tenancy.search_indexes import TENANCY_SEARCH_TYPES
from virtualization.search_indexes import VIRTUALIZATION_SEARCH_TYPES
from wireless.search_indexes import WIRELESS_SEARCH_TYPES
SEARCH_TYPE_HIERARCHY = {
'Circuits': CIRCUIT_SEARCH_TYPES,
'DCIM': DCIM_SEARCH_TYPES,
'IPAM': IPAM_SEARCH_TYPES,
'Tenancy': TENANCY_SEARCH_TYPES,
'Virtualization': VIRTUALIZATION_SEARCH_TYPES,
'Wireless': WIRELESS_SEARCH_TYPES,
'Journal': JOURNAL_SEARCH_TYPES,
}
def build_search_types():
result = dict()
for app_types in SEARCH_TYPE_HIERARCHY.values():
for name, items in app_types.items():
result[name] = items
return result
SEARCH_TYPES = build_search_types()

View File

@ -23,7 +23,7 @@ from extras.tables import ObjectChangeTable
from ipam.models import Aggregate, IPAddress, IPRange, Prefix, VLAN, VRF
from netbox.constants import SEARCH_MAX_RESULTS
from netbox.forms import SearchForm
from netbox.search import SEARCH_TYPES
from search.backends import default_search_engine
from tenancy.models import Tenant
from virtualization.models import Cluster, VirtualMachine
from wireless.models import WirelessLAN, WirelessLink

View File

@ -1,3 +1,5 @@
import inspect
import sys
from django.apps import AppConfig
from django.apps import apps
@ -27,7 +29,18 @@ class SearchConfig(AppConfig):
verbose_name = "search"
def ready(self):
from .backends import default_search_engine
from .hierarchy import SEARCH_TYPES
for name, module in SEARCH_TYPES.items():
default_search_engine.register(name, module)
for name, module in get_app_modules():
submodule_name = "search_indexes"
if module_has_submodule(module, submodule_name):
print(f"{name}.{submodule_name}")
module_name = f"{name}.{submodule_name}"
for cls_name, cls_obj in inspect.getmembers(sys.modules[module_name]):
if inspect.isclass(cls_obj) and getattr(cls_obj, "search_index", False) and getattr(cls_obj, "model", None):
cls_name = cls_obj.model.__name__.lower()
if not default_search_engine.is_registered(cls_name, cls_obj):
default_search_engine.register(cls_name, cls_obj)

View File

@ -10,30 +10,6 @@ from django.db.models.signals import post_save, pre_delete
_backends_cache = {}
def get_backend(backend_name=None):
"""Initializes and returns the search backend."""
global _backends_cache
if not backend_name:
backend_name = getattr(settings, "SEARCH_BACKEND", "search.backends.PostgresIcontainsSearchBackend")
# Try to use the cached backend.
if backend_name in _backends_cache:
return _backends_cache[backend_name]
# Load the backend class.
backend_module_name, backend_cls_name = backend_name.rsplit(".", 1)
backend_module = import_module(backend_module_name)
try:
backend_cls = getattr(backend_module, backend_cls_name)
except AttributeError:
raise ImproperlyConfigured(f"Could not find a class named {backend_module_name} in {backend_cls_name}")
# Initialize the backend.
backend = backend_cls()
_backends_cache[backend_name] = backend
return backend
class SearchEngineError(Exception):
"""Something went wrong with a search engine."""
@ -63,11 +39,11 @@ class SearchBackend(object):
# Store a reference to this engine.
self.__class__._created_engines[engine_slug] = self
def is_registered(self, model):
def is_registered(self, key, model):
"""Checks whether the given model is registered with this search engine."""
return model in self._registered_models
return key in self._registered_models
def register(self, model):
def register(self, key, model):
"""
Registers the given model with this search engine.
@ -75,14 +51,19 @@ class SearchBackend(object):
RegistrationError will be raised.
"""
# Check for existing registration.
if self.is_registered(model):
if self.is_registered(key, model):
raise RegistrationError(f"{model} is already registered with this search engine")
self._registered_models[key] = model
# Connect to the signalling framework.
if self._use_hooks():
post_save.connect(self._post_save_receiver, model)
pre_delete.connect(self._pre_delete_receiver, model)
def get_registry(self):
return self._registered_models
# Signalling hooks.
def _use_hooks(self):
@ -132,6 +113,30 @@ class PostgresIcontainsSearchBackend(SearchBackend):
return results
def get_backend(backend_name=None):
"""Initializes and returns the search backend."""
global _backends_cache
if not backend_name:
backend_name = getattr(settings, "SEARCH_BACKEND", "search.backends.PostgresIcontainsSearchBackend")
# Try to use the cached backend.
if backend_name in _backends_cache:
return _backends_cache[backend_name]
# Load the backend class.
backend_module_name, backend_cls_name = backend_name.rsplit(".", 1)
backend_module = import_module(backend_module_name)
try:
backend_cls = getattr(backend_module, backend_cls_name)
except AttributeError:
raise ImproperlyConfigured(f"Could not find a class named {backend_module_name} in {backend_cls_name}")
# Initialize the backend.
backend = backend_cls("default")
_backends_cache[backend_name] = backend
return backend
# The main search methods.
default_search_engine = SearchBackend("default")
default_search_engine = get_backend()
search = default_search_engine.search

View File

@ -6,90 +6,4 @@ class SearchMixin(object):
"""
Base class for building search indexes.
"""
def __init__(self, model=None, queryset=None, filterset=None, table=None, url=None):
self.model = model
self.queryset = queryset
self.filterset = filterset
self.table = table
self.url = url
def get_model(self):
"""
Should return the ``Model`` class (not an instance) that the rest of the
``SearchIndex`` should use.
This method is required & you must override it to return the correct class.
"""
if self.model is not None:
model = self.model
else:
raise ImproperlyConfigured(
f"{self.__class__.__name__}s is missing a Model. Set model in init or override"
f"{self.__class__.__name__}s.get_model()."
)
return model
def get_queryset(self):
"""
Should return the ``QuerySet`` class (not an instance) that the rest of the
``SearchIndex`` should use.
This method is required & you must override it to return the correct class.
"""
if self.queryset is not None:
queryset = self.queryset
else:
raise ImproperlyConfigured(
f"{self.__class__.__name__}s is missing a QuerySet. Set queryset in init or override "
f"{self.__class__.__name__}s.get_queryset()."
)
return queryset
def get_filterset(self):
"""
Should return the ``FilterSet`` class (not an instance) that the rest of the
``SearchIndex`` should use.
This method is required & you must override it to return the correct class.
"""
if self.filterset is not None:
filterset = self.filterset
else:
raise ImproperlyConfigured(
f"{self.__class__.__name__}s is missing a FilterSet. Set filterset in init or override "
f"{self.__class__.__name__}s.get_filterset()."
)
return filterset
def get_table(self):
"""
Should return the ``Table`` class (not an instance) that the rest of the
``SearchIndex`` should use.
This method is required & you must override it to return the correct class.
"""
if self.table is not None:
table = self.table
else:
raise ImproperlyConfigured(
f"{self.__class__.__name__}s is missing a Table. Set table in init or override "
f"{self.__class__.__name__}s.get_table()."
)
return table
def get_url(self):
"""
Should return the ``URL`` class (not an instance) that the rest of the
``SearchIndex`` should use.
This method is required & you must override it to return the correct class.
"""
if self.url is not None:
url = self.url
else:
raise ImproperlyConfigured(
f"{self.__class__.__name__}s is missing a URL. Set url in init or override "
f"{self.__class__.__name__}s.get_url()."
)
return url
search_index = True

View File

@ -8,23 +8,21 @@ from wireless.models import WirelessLAN, WirelessLink
class WirelessLANIndex(SearchMixin):
def __init__(self):
self.model = WirelessLAN
self.queryset = WirelessLAN.objects.prefetch_related('group', 'vlan').annotate(
interface_count=count_related(Interface, 'wireless_lans')
)
self.filterset = wireless.filtersets.WirelessLANFilterSet
self.table = wireless.tables.WirelessLANTable
self.url = 'wireless:wirelesslan_list'
model = WirelessLAN
queryset = WirelessLAN.objects.prefetch_related('group', 'vlan').annotate(
interface_count=count_related(Interface, 'wireless_lans')
)
filterset = wireless.filtersets.WirelessLANFilterSet
table = wireless.tables.WirelessLANTable
url = 'wireless:wirelesslan_list'
class WirelessLinkIndex(SearchMixin):
def __init__(self):
self.model = WirelessLink
self.queryset = WirelessLink.objects.prefetch_related('interface_a__device', 'interface_b__device')
self.filterset = wireless.filtersets.WirelessLinkFilterSet
self.table = wireless.tables.WirelessLinkTable
self.url = 'wireless:wirelesslink_list'
model = WirelessLink
queryset = WirelessLink.objects.prefetch_related('interface_a__device', 'interface_b__device')
filterset = wireless.filtersets.WirelessLinkFilterSet
table = wireless.tables.WirelessLinkTable
url = 'wireless:wirelesslink_list'
WIRELESS_SEARCH_TYPES = {