From 557248c53f30680bdbf651b21f5929c50b9a09e1 Mon Sep 17 00:00:00 2001 From: kkthxbye <> Date: Thu, 24 Mar 2022 13:42:07 +0100 Subject: [PATCH 1/3] Save old JobResults --- netbox/extras/api/views.py | 4 ++-- netbox/extras/management/commands/runscript.py | 7 ------- netbox/extras/reports.py | 9 --------- netbox/extras/scripts.py | 11 +---------- netbox/extras/views.py | 4 ++-- netbox/netbox/views/__init__.py | 11 +---------- 6 files changed, 6 insertions(+), 40 deletions(-) diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index 579e39d86..688f3c7ab 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -179,7 +179,7 @@ class ReportViewSet(ViewSet): for r in JobResult.objects.filter( obj_type=report_content_type, status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES - ).defer('data') + ).order_by('name', '-created').distinct('name').defer('data') } # Iterate through all available Reports. @@ -271,7 +271,7 @@ class ScriptViewSet(ViewSet): for r in JobResult.objects.filter( obj_type=script_content_type, status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES - ).defer('data').order_by('created') + ).order_by('name', '-created').distinct('name').defer('data') } flat_list = [] diff --git a/netbox/extras/management/commands/runscript.py b/netbox/extras/management/commands/runscript.py index 0d1dc5cea..12188619f 100644 --- a/netbox/extras/management/commands/runscript.py +++ b/netbox/extras/management/commands/runscript.py @@ -113,13 +113,6 @@ class Command(BaseCommand): script_content_type = ContentType.objects.get(app_label='extras', model='script') - # Delete any previous terminal state results - JobResult.objects.filter( - obj_type=script_content_type, - name=script.full_name, - status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES - ).delete() - # Create the job result job_result = JobResult.objects.create( name=script.full_name, diff --git a/netbox/extras/reports.py b/netbox/extras/reports.py index 0bdc4847e..0a8a8d89b 100644 --- a/netbox/extras/reports.py +++ b/netbox/extras/reports.py @@ -84,15 +84,6 @@ def run_report(job_result, *args, **kwargs): job_result.save() logging.error(f"Error during execution of report {job_result.name}") - # Delete any previous terminal state results - JobResult.objects.filter( - obj_type=job_result.obj_type, - name=job_result.name, - status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES - ).exclude( - pk=job_result.pk - ).delete() - class Report(object): """ diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 5351bca8a..4eacddbeb 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -481,15 +481,6 @@ def run_script(data, request, commit=True, *args, **kwargs): else: _run_script() - # Delete any previous terminal state results - JobResult.objects.filter( - obj_type=job_result.obj_type, - name=job_result.name, - status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES - ).exclude( - pk=job_result.pk - ).delete() - def get_scripts(use_names=False): """ @@ -497,7 +488,7 @@ def get_scripts(use_names=False): defined name in place of the actual module name. """ scripts = OrderedDict() - # Iterate through all modules within the reports path. These are the user-created files in which reports are + # Iterate through all modules within the scripts path. These are the user-created files in which reports are # defined. for importer, module_name, _ in pkgutil.iter_modules([settings.SCRIPTS_ROOT]): # Remove cached module to ensure consistency with filesystem diff --git a/netbox/extras/views.py b/netbox/extras/views.py index 785c5eb5a..9825d10de 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -524,7 +524,7 @@ class ReportListView(ContentTypePermissionRequiredMixin, View): for r in JobResult.objects.filter( obj_type=report_content_type, status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES - ).defer('data') + ).order_by('name', '-created').distinct('name').defer('data') } ret = [] @@ -656,7 +656,7 @@ class ScriptListView(ContentTypePermissionRequiredMixin, View): for r in JobResult.objects.filter( obj_type=script_content_type, status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES - ).defer('data') + ).order_by('name', '-created').distinct('name').defer('data') } for _scripts in scripts.values(): diff --git a/netbox/netbox/views/__init__.py b/netbox/netbox/views/__init__.py index 5d388be35..fad347c36 100644 --- a/netbox/netbox/views/__init__.py +++ b/netbox/netbox/views/__init__.py @@ -19,8 +19,7 @@ from circuits.models import Circuit, Provider from dcim.models import ( Cable, ConsolePort, Device, DeviceType, Interface, PowerPanel, PowerFeed, PowerPort, Rack, Site, ) -from extras.choices import JobResultStatusChoices -from extras.models import ObjectChange, JobResult +from extras.models import ObjectChange from extras.tables import ObjectChangeTable from ipam.models import Aggregate, IPAddress, IPRange, Prefix, VLAN, VRF from netbox.constants import SEARCH_MAX_RESULTS, SEARCH_TYPES @@ -48,13 +47,6 @@ class HomeView(View): pk__lt=F('_path__destination_id') ) - # Report Results - report_content_type = ContentType.objects.get(app_label='extras', model='report') - report_results = JobResult.objects.filter( - obj_type=report_content_type, - status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES - ).defer('data')[:10] - def build_stats(): org = ( ("dcim.view_site", "Sites", Site.objects.restrict(request.user, 'view').count), @@ -150,7 +142,6 @@ class HomeView(View): return render(request, self.template_name, { 'search_form': SearchForm(), 'stats': build_stats(), - 'report_results': report_results, 'changelog_table': changelog_table, 'new_release': new_release, }) From 56a8a860432e3cb8d8549761025a47387dd8c036 Mon Sep 17 00:00:00 2001 From: kkthxbye <> Date: Fri, 25 Mar 2022 09:00:02 +0100 Subject: [PATCH 2/3] Add dynamic config JOBRESULT_RETENTION and cleanup functionality to the housekeeping script --- docs/administration/housekeeping.md | 1 + docs/configuration/dynamic-settings.md | 12 ++++++++ netbox/extras/admin.py | 2 +- .../management/commands/housekeeping.py | 28 +++++++++++++++++++ netbox/netbox/config/parameters.py | 7 +++++ 5 files changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/administration/housekeeping.md b/docs/administration/housekeeping.md index bbb03dc27..1989e41c0 100644 --- a/docs/administration/housekeeping.md +++ b/docs/administration/housekeeping.md @@ -4,6 +4,7 @@ NetBox includes a `housekeeping` management command that should be run nightly. * Clearing expired authentication sessions from the database * Deleting changelog records older than the configured [retention time](../configuration/dynamic-settings.md#changelog_retention) +* Deleting job result records older than the configured [retention time](../configuration/dynamic-settings.md#jobresult_retention) This command can be invoked directly, or by using the shell script provided at `/opt/netbox/contrib/netbox-housekeeping.sh`. This script can be linked from your cron scheduler's daily jobs directory (e.g. `/etc/cron.daily`) or referenced directly within the cron configuration file. diff --git a/docs/configuration/dynamic-settings.md b/docs/configuration/dynamic-settings.md index 5649eb9be..4a12726ba 100644 --- a/docs/configuration/dynamic-settings.md +++ b/docs/configuration/dynamic-settings.md @@ -43,6 +43,18 @@ changes in the database indefinitely. --- +## JOBRESULT_RETENTION + +Default: 0 + +The number of days to retain job results (scripts and reports). Set this to `0` to retain +job results in the database indefinitely. + +!!! warning + If enabling indefinite job results retention, it is recommended to periodically delete old entries. Otherwise, the database may eventually exceed capacity. + +--- + ## CUSTOM_VALIDATORS This is a mapping of models to [custom validators](../customization/custom-validation.md) that have been defined locally to enforce custom validation logic. An example is provided below: diff --git a/netbox/extras/admin.py b/netbox/extras/admin.py index 64c224cb1..28902c323 100644 --- a/netbox/extras/admin.py +++ b/netbox/extras/admin.py @@ -40,7 +40,7 @@ class ConfigRevisionAdmin(admin.ModelAdmin): 'fields': ('DEFAULT_USER_PREFERENCES',), }), ('Miscellaneous', { - 'fields': ('MAINTENANCE_MODE', 'GRAPHQL_ENABLED', 'CHANGELOG_RETENTION', 'MAPS_URL'), + 'fields': ('MAINTENANCE_MODE', 'GRAPHQL_ENABLED', 'CHANGELOG_RETENTION', 'JOBRESULT_RETENTION', 'MAPS_URL'), }), ('Config Revision', { 'fields': ('comment',), diff --git a/netbox/extras/management/commands/housekeeping.py b/netbox/extras/management/commands/housekeeping.py index 0607a16c2..51d50d7e1 100644 --- a/netbox/extras/management/commands/housekeeping.py +++ b/netbox/extras/management/commands/housekeeping.py @@ -9,6 +9,7 @@ from django.db import DEFAULT_DB_ALIAS from django.utils import timezone from packaging import version +from extras.models import JobResult from extras.models import ObjectChange from netbox.config import Config @@ -63,6 +64,33 @@ class Command(BaseCommand): f"\tSkipping: No retention period specified (CHANGELOG_RETENTION = {config.CHANGELOG_RETENTION})" ) + # Delete expired JobResults + if options['verbosity']: + self.stdout.write("[*] Checking for expired jobresult records") + if config.JOBRESULT_RETENTION: + cutoff = timezone.now() - timedelta(days=config.JOBRESULT_RETENTION) + if options['verbosity'] >= 2: + self.stdout.write(f"\tRetention period: {config.JOBRESULT_RETENTION} days") + self.stdout.write(f"\tCut-off time: {cutoff}") + expired_records = JobResult.objects.filter(created__lt=cutoff).count() + if expired_records: + if options['verbosity']: + self.stdout.write( + f"\tDeleting {expired_records} expired records... ", + self.style.WARNING, + ending="" + ) + self.stdout.flush() + JobResult.objects.filter(created__lt=cutoff)._raw_delete(using=DEFAULT_DB_ALIAS) + if options['verbosity']: + self.stdout.write("Done.", self.style.SUCCESS) + elif options['verbosity']: + self.stdout.write("\tNo expired records found.", self.style.SUCCESS) + elif options['verbosity']: + self.stdout.write( + f"\tSkipping: No retention period specified (JOBRESULT_RETENTION = {config.JOBRESULT_RETENTION})" + ) + # Check for new releases (if enabled) if options['verbosity']: self.stdout.write("[*] Checking for latest release") diff --git a/netbox/netbox/config/parameters.py b/netbox/netbox/config/parameters.py index 89de94674..9bbf45ceb 100644 --- a/netbox/netbox/config/parameters.py +++ b/netbox/netbox/config/parameters.py @@ -187,6 +187,13 @@ PARAMS = ( description="Days to retain changelog history (set to zero for unlimited)", field=forms.IntegerField ), + ConfigParam( + name='JOBRESULT_RETENTION', + label='Job result retention', + default=0, + description="Days to retain job result history (set to zero for unlimited)", + field=forms.IntegerField + ), ConfigParam( name='MAPS_URL', label='Maps URL', From 9515cf1efd3d34f9cb05f570512916aa44b95d1a Mon Sep 17 00:00:00 2001 From: kkthxbye-code Date: Fri, 8 Apr 2022 22:10:17 +0200 Subject: [PATCH 3/3] Change default JOBRESULT_RETENTION from 0 to 90 --- docs/configuration/dynamic-settings.md | 2 +- netbox/netbox/config/parameters.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration/dynamic-settings.md b/docs/configuration/dynamic-settings.md index 4a12726ba..2fa046fcf 100644 --- a/docs/configuration/dynamic-settings.md +++ b/docs/configuration/dynamic-settings.md @@ -45,7 +45,7 @@ changes in the database indefinitely. ## JOBRESULT_RETENTION -Default: 0 +Default: 90 The number of days to retain job results (scripts and reports). Set this to `0` to retain job results in the database indefinitely. diff --git a/netbox/netbox/config/parameters.py b/netbox/netbox/config/parameters.py index 9bbf45ceb..68c96b38a 100644 --- a/netbox/netbox/config/parameters.py +++ b/netbox/netbox/config/parameters.py @@ -190,7 +190,7 @@ PARAMS = ( ConfigParam( name='JOBRESULT_RETENTION', label='Job result retention', - default=0, + default=90, description="Days to retain job result history (set to zero for unlimited)", field=forms.IntegerField ),