NeoAnd/netbox
1
0
Fork 0
mirror of https://github.com/netbox-community/netbox.git synced 2025-08-25 08:46:10 -06:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

Performance improvements for reindexing

Browse Source
This commit is contained in:
jeremystretch 2022-10-20 09:07:05 -04:00
parent 6eb2983ccd
commit 530f5180b8
2 changed files with 29 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
netbox/extras/management/commands/reindex.py
View File

@ -52,6 +52,7 @@ class Command(BaseCommand):
# Clear all cached values for the specified models # Clear all cached values for the specified models
self.stdout.write('Clearing cached values... ', ending='') self.stdout.write('Clearing cached values... ', ending='')
self.stdout.flush()
content_types = [ content_types = [
ContentType.objects.get_for_model(model) for model in indexers.keys() ContentType.objects.get_for_model(model) for model in indexers.keys()
] ]
@ -59,11 +60,13 @@ class Command(BaseCommand):
self.stdout.write(f'{deleted_count} entries deleted.') self.stdout.write(f'{deleted_count} entries deleted.')
# Index models # Index models
self.stdout.write('Indexing models')
for model, idx in indexers.items(): for model, idx in indexers.items():
app_label = model._meta.app_label app_label = model._meta.app_label
model_name = model._meta.model_name model_name = model._meta.model_name
self.stdout.write(f'Reindexing {app_label}.{model_name}... ', ending='') self.stdout.write(f' {app_label}.{model_name}... ', ending='')
i = search_backend.cache(model.objects.iterator()) self.stdout.flush()
i = search_backend.cache(model.objects.iterator(), remove_existing=False)
if i: if i:
self.stdout.write(f'{i} entries cached.') self.stdout.write(f'{i} entries cached.')
else: else:
@ -72,4 +75,4 @@ class Command(BaseCommand):
msg = f'Completed.' msg = f'Completed.'
if total_count := search_backend.size: if total_count := search_backend.size:
msg += f' Total entries: {total_count}' msg += f' Total entries: {total_count}'
self.stdout.write(msg) self.stdout.write(msg, self.style.SUCCESS)

38
netbox/netbox/search/backends.py
View File

@ -156,20 +156,24 @@ class CachedValueSearchBackend(SearchBackend):
counter = 0 counter = 0
for instance in instances: for instance in instances:
# First item
if not counter:
# Determine the indexer
if indexer is None:
try:
indexer = get_indexer(instance)
except KeyError:
break
# Prefetch any associated custom fields
content_type = ContentType.objects.get_for_model(indexer.model)
custom_fields = CustomField.objects.filter(content_types=content_type).exclude(search_weight=0)
# Wipe out any previously cached values for the object # Wipe out any previously cached values for the object
if remove_existing: if remove_existing:
cls.remove(instance) cls.remove(instance)
# Determine the indexer
if indexer is None:
try:
indexer = get_indexer(instance)
content_type = ContentType.objects.get_for_model(indexer.model)
custom_fields = CustomField.objects.filter(content_types=content_type).exclude(search_weight=0)
except KeyError:
# No indexer has been registered for this model
continue
# Generate cache data # Generate cache data
for field in indexer.to_cache(instance, custom_fields=custom_fields): for field in indexer.to_cache(instance, custom_fields=custom_fields):
buffer.append( buffer.append(
@ -203,15 +207,19 @@ class CachedValueSearchBackend(SearchBackend):
return return
ct = ContentType.objects.get_for_model(instance) ct = ContentType.objects.get_for_model(instance)
CachedValue.objects.filter(object_type=ct, object_id=instance.pk).delete() qs = CachedValue.objects.filter(object_type=ct, object_id=instance.pk)
# Call _raw_delete() on the queryset to avoid first loading instances into memory
return qs._raw_delete(using=qs.db)
@classmethod @classmethod
def clear(cls, object_types=None): def clear(cls, object_types=None):
qs = CachedValue.objects.all()
if object_types: if object_types:
del_count, _ = CachedValue.objects.filter(object_type__in=object_types).delete() qs = qs.filter(object_type__in=object_types)
else:
del_count, _ = CachedValue.objects.all().delete() # Call _raw_delete() on the queryset to avoid first loading instances into memory
return del_count return qs._raw_delete(using=qs.db)
@property @property
def size(self): def size(self):