From ccb09b0f7b38ad5e474cebc47d60478e83454a33 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 27 Mar 2023 16:18:45 -0400 Subject: [PATCH] Reference database object by GFK when running scripts & reports via UI --- netbox/core/models/data.py | 7 +- netbox/core/models/jobs.py | 35 ++++++++++ netbox/extras/api/views.py | 1 + netbox/extras/models/reports.py | 7 ++ netbox/extras/reports.py | 16 ++--- netbox/extras/scripts.py | 10 +-- netbox/extras/urls.py | 4 +- netbox/extras/views.py | 65 +++++++++---------- .../templates/extras/htmx/report_result.html | 24 +++---- .../templates/extras/htmx/script_result.html | 26 ++++---- netbox/templates/extras/report.html | 2 +- netbox/templates/extras/report_list.html | 2 +- netbox/templates/extras/report_result.html | 6 +- netbox/templates/extras/script_list.html | 2 +- netbox/templates/extras/script_result.html | 10 +-- 15 files changed, 129 insertions(+), 88 deletions(-) diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py index a4422ac79..53b9602ee 100644 --- a/netbox/core/models/data.py +++ b/netbox/core/models/data.py @@ -118,11 +118,10 @@ class DataSource(PrimaryModel): DataSource.objects.filter(pk=self.pk).update(status=self.status) # Enqueue a sync job - job_result = Job.enqueue_job( + job_result = Job.enqueue( import_string('core.jobs.sync_datasource'), - name=self.name, - obj_type=ContentType.objects.get_for_model(DataSource), - user=request.user, + instance=self, + user=request.user ) return job_result diff --git a/netbox/core/models/jobs.py b/netbox/core/models/jobs.py index 0ae626a74..cd2ac8ab5 100644 --- a/netbox/core/models/jobs.py +++ b/netbox/core/models/jobs.py @@ -194,6 +194,41 @@ class Job(models.Model): return job + @classmethod + def enqueue(cls, func, instance, name=None, user=None, schedule_at=None, interval=None, **kwargs): + """ + Create a Job instance and enqueue a job using the given callable + + Args: + func: The callable object to be enqueued for execution + instance: The NetBox object to which this job pertains + name: Name for the job (optional) + user: The user responsible for running the job + schedule_at: Schedule the job to be executed at the passed date and time + interval: Recurrence interval (in minutes) + """ + object_type = ContentType.objects.get_for_model(instance, for_concrete_model=False) + rq_queue_name = get_queue_for_model(object_type.model) + queue = django_rq.get_queue(rq_queue_name) + status = JobStatusChoices.STATUS_SCHEDULED if schedule_at else JobStatusChoices.STATUS_PENDING + job = Job.objects.create( + object_type=object_type, + object_id=instance.pk, + name=name, + status=status, + scheduled=schedule_at, + interval=interval, + user=user, + job_id=uuid.uuid4() + ) + + if schedule_at: + queue.enqueue_at(schedule_at, func, job_id=str(job.job_id), job_result=job, **kwargs) + else: + queue.enqueue(func, job_id=str(job.job_id), job_result=job, **kwargs) + + return job + def trigger_webhooks(self, event): from extras.models import Webhook diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index bd775c668..6b8309b28 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -1,5 +1,6 @@ from django.contrib.contenttypes.models import ContentType from django.http import Http404 +from django.shortcuts import get_object_or_404 from django_rq.queues import get_connection from rest_framework import status from rest_framework.decorators import action diff --git a/netbox/extras/models/reports.py b/netbox/extras/models/reports.py index 68174ef1a..3c62d29ed 100644 --- a/netbox/extras/models/reports.py +++ b/netbox/extras/models/reports.py @@ -1,6 +1,7 @@ import inspect from functools import cached_property +from django.contrib.contenttypes.fields import GenericRelation from django.db import models from django.urls import reverse @@ -35,6 +36,12 @@ class ReportModule(PythonModuleMixin, ManagedFile): """ Proxy model for report module files. """ + jobs = GenericRelation( + to='core.Job', + content_type_field='object_type', + object_id_field='object_id' + ) + objects = ReportModuleManager() class Meta: diff --git a/netbox/extras/reports.py b/netbox/extras/reports.py index ec7836329..086bd0977 100644 --- a/netbox/extras/reports.py +++ b/netbox/extras/reports.py @@ -28,24 +28,24 @@ def run_report(job_result, *args, **kwargs): Helper function to call the run method on a report. This is needed to get around the inability to pickle an instance method for queueing into the background processor. """ - module_name, report_name = job_result.name.split('.', 1) - report = get_report(module_name, report_name)() + job_result.start() + + module = ReportModule.objects.get(pk=job_result.object_id) + report = module.reports.get(job_result.name)() try: - job_result.start() report.run(job_result) except Exception: job_result.terminate(status=JobStatusChoices.STATUS_ERRORED) logging.error(f"Error during execution of report {job_result.name}") finally: # Schedule the next job if an interval has been set - start_time = job_result.scheduled or job_result.started - if start_time and job_result.interval: - new_scheduled_time = start_time + timedelta(minutes=job_result.interval) - Job.enqueue_job( + if job_result.interval: + new_scheduled_time = job_result.scheduled + timedelta(minutes=job_result.interval) + Job.enqueue( run_report, + instance=job_result.object, name=job_result.name, - obj_type=job_result.obj_type, user=job_result.user, job_timeout=report.job_timeout, schedule_at=new_scheduled_time, diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 061384e6a..69fd88aaa 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -444,10 +444,10 @@ def run_script(data, request, commit=True, *args, **kwargs): job_result = kwargs.pop('job_result') job_result.start() - module_name, script_name = job_result.name.split('.', 1) - script = get_script(module_name, script_name)() + module = ScriptModule.objects.get(pk=job_result.object_id) + script = module.scripts.get(job_result.name)() - logger = logging.getLogger(f"netbox.scripts.{module_name}.{script_name}") + logger = logging.getLogger(f"netbox.scripts.{script.full_name}") logger.info(f"Running script (commit={commit})") # Add files to form data @@ -500,10 +500,10 @@ def run_script(data, request, commit=True, *args, **kwargs): # Schedule the next job if an interval has been set if job_result.interval: new_scheduled_time = job_result.scheduled + timedelta(minutes=job_result.interval) - Job.enqueue_job( + Job.enqueue( run_script, + instance=job_result.object, name=job_result.name, - obj_type=job_result.obj_type, user=job_result.user, schedule_at=new_scheduled_time, interval=job_result.interval, diff --git a/netbox/extras/urls.py b/netbox/extras/urls.py index bc3e33b01..2bc6e709c 100644 --- a/netbox/extras/urls.py +++ b/netbox/extras/urls.py @@ -95,14 +95,14 @@ urlpatterns = [ # Reports path('reports/', views.ReportListView.as_view(), name='report_list'), path('reports/add/', views.ReportModuleCreateView.as_view(), name='reportmodule_add'), - path('reports/results//', views.ReportResultView.as_view(), name='report_result'), + path('reports/results//', views.ReportResultView.as_view(), name='report_result'), path('reports//', include(get_model_urls('extras', 'reportmodule'))), path('reports/./', views.ReportView.as_view(), name='report'), # Scripts path('scripts/', views.ScriptListView.as_view(), name='script_list'), path('scripts/add/', views.ScriptModuleCreateView.as_view(), name='scriptmodule_add'), - path('scripts/results//', views.ScriptResultView.as_view(), name='script_result'), + path('scripts/results//', views.ScriptResultView.as_view(), name='script_result'), path('scripts//', include(get_model_urls('extras', 'scriptmodule'))), path('scripts/./', views.ScriptView.as_view(), name='script'), diff --git a/netbox/extras/views.py b/netbox/extras/views.py index fc5f0aeea..c4640b890 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -845,10 +845,11 @@ class ReportView(ContentTypePermissionRequiredMixin, View): module = get_object_or_404(ReportModule.objects.restrict(request.user), file_path=f'{module}.py') report = module.reports[name]() - report_content_type = ContentType.objects.get(app_label='extras', model='report') + object_type = ContentType.objects.get(app_label='extras', model='reportmodule') report.result = Job.objects.filter( - object_type=report_content_type, - name=report.full_name, + object_type=object_type, + object_id=module.pk, + name=report.name, status__in=JobStatusChoices.TERMINAL_STATE_CHOICES ).first() @@ -876,17 +877,17 @@ class ReportView(ContentTypePermissionRequiredMixin, View): }) # Run the Report. A new Job is created. - job_result = Job.enqueue_job( + job = Job.enqueue( run_report, - name=report.full_name, - obj_type=ContentType.objects.get_for_model(Report), + instance=module, + name=report.class_name, user=request.user, schedule_at=form.cleaned_data.get('schedule_at'), interval=form.cleaned_data.get('interval'), job_timeout=report.job_timeout ) - return redirect('extras:report_result', job_result_pk=job_result.pk) + return redirect('extras:report_result', job_pk=job.pk) return render(request, 'extras/report.html', { 'module': module, @@ -902,28 +903,26 @@ class ReportResultView(ContentTypePermissionRequiredMixin, View): def get_required_permission(self): return 'extras.view_report' - def get(self, request, job_result_pk): - report_content_type = ContentType.objects.get(app_label='extras', model='report') - result = get_object_or_404(Job.objects.all(), pk=job_result_pk, object_type=report_content_type) + def get(self, request, job_pk): + object_type = ContentType.objects.get_by_natural_key(app_label='extras', model='reportmodule') + job = get_object_or_404(Job.objects.all(), pk=job_pk, object_type=object_type) - # Retrieve the Report and attach the Job to it - module, report_name = result.name.split('.', maxsplit=1) - report = get_report(module, report_name) - report.result = result + module = job.object + report = module.reports[job.name] # If this is an HTMX request, return only the result HTML if is_htmx(request): response = render(request, 'extras/htmx/report_result.html', { 'report': report, - 'result': result, + 'job': job, }) - if result.completed or not result.started: + if job.completed or not job.started: response.status_code = 286 return response return render(request, 'extras/report_result.html', { 'report': report, - 'result': result, + 'job': job, }) @@ -982,9 +981,11 @@ class ScriptView(ContentTypePermissionRequiredMixin, View): form = script.as_form(initial=normalize_querydict(request.GET)) # Look for a pending Job (use the latest one by creation timestamp) + object_type = ContentType.objects.get(app_label='extras', model='scriptmodule') script.result = Job.objects.filter( - object_type=ContentType.objects.get_for_model(Script), - name=script.full_name, + object_type=object_type, + object_id=module.pk, + name=script.name, ).exclude( status__in=JobStatusChoices.TERMINAL_STATE_CHOICES ).first() @@ -1008,10 +1009,10 @@ class ScriptView(ContentTypePermissionRequiredMixin, View): messages.error(request, "Unable to run script: RQ worker process not running.") elif form.is_valid(): - job_result = Job.enqueue_job( + job = Job.enqueue( run_script, - name=script.full_name, - obj_type=ContentType.objects.get_for_model(Script), + instance=module, + name=script.class_name, user=request.user, schedule_at=form.cleaned_data.pop('_schedule_at'), interval=form.cleaned_data.pop('_interval'), @@ -1021,7 +1022,7 @@ class ScriptView(ContentTypePermissionRequiredMixin, View): commit=form.cleaned_data.pop('_commit') ) - return redirect('extras:script_result', job_result_pk=job_result.pk) + return redirect('extras:script_result', job_pk=job.pk) return render(request, 'extras/script.html', { 'module': module, @@ -1035,28 +1036,26 @@ class ScriptResultView(ContentTypePermissionRequiredMixin, View): def get_required_permission(self): return 'extras.view_script' - def get(self, request, job_result_pk): - script_content_type = ContentType.objects.get(app_label='extras', model='script') - result = get_object_or_404(Job.objects.all(), pk=job_result_pk, object_type=script_content_type) + def get(self, request, job_pk): + object_type = ContentType.objects.get_by_natural_key(app_label='extras', model='scriptmodule') + job = get_object_or_404(Job.objects.all(), pk=job_pk, object_type=object_type) - module_name, script_name = result.name.split('.', 1) - module = get_object_or_404(ScriptModule.objects.restrict(request.user), file_path=f'{module_name}.py') - script = module.scripts[script_name]() + module = job.object + script = module.scripts[job.name]() # If this is an HTMX request, return only the result HTML if is_htmx(request): response = render(request, 'extras/htmx/script_result.html', { 'script': script, - 'result': result, + 'job': job, }) - if result.completed or not result.started: + if job.completed or not job.started: response.status_code = 286 return response return render(request, 'extras/script_result.html', { 'script': script, - 'result': result, - 'class_name': script.__class__.__name__ + 'job': job, }) diff --git a/netbox/templates/extras/htmx/report_result.html b/netbox/templates/extras/htmx/report_result.html index fcf8cae68..d15898c3d 100644 --- a/netbox/templates/extras/htmx/report_result.html +++ b/netbox/templates/extras/htmx/report_result.html @@ -2,24 +2,24 @@ {% load helpers %}

- {% if result.started %} - Started: {{ result.started|annotated_date }} - {% elif result.scheduled %} - Scheduled for: {{ result.scheduled|annotated_date }} ({{ result.scheduled|naturaltime }}) + {% if job.started %} + Started: {{ job.started|annotated_date }} + {% elif job.scheduled %} + Scheduled for: {{ job.scheduled|annotated_date }} ({{ job.scheduled|naturaltime }}) {% else %} - Created: {{ result.created|annotated_date }} + Created: {{ job.created|annotated_date }} {% endif %} - {% if result.completed %} - Duration: {{ result.duration }} + {% if job.completed %} + Duration: {{ job.duration }} {% endif %} - {% badge result.get_status_display result.get_status_color %} + {% badge job.get_status_display job.get_status_color %}

-{% if result.completed %} +{% if job.completed %}
Report Methods
- {% for method, data in result.data.items %} + {% for method, data in job.data.items %} - {% for method, data in result.data.items %} + {% for method, data in job.data.items %}
{{ method }} @@ -46,7 +46,7 @@
{{ method }} @@ -75,6 +75,6 @@
-{% elif result.started %} +{% elif job.started %} {% include 'extras/inc/result_pending.html' %} {% endif %} diff --git a/netbox/templates/extras/htmx/script_result.html b/netbox/templates/extras/htmx/script_result.html index 6037c3052..5b2ac8cf3 100644 --- a/netbox/templates/extras/htmx/script_result.html +++ b/netbox/templates/extras/htmx/script_result.html @@ -3,19 +3,19 @@ {% load log_levels %}

- {% if result.started %} - Started: {{ result.started|annotated_date }} - {% elif result.scheduled %} - Scheduled for: {{ result.scheduled|annotated_date }} ({{ result.scheduled|naturaltime }}) + {% if job.started %} + Started: {{ job.started|annotated_date }} + {% elif job.scheduled %} + Scheduled for: {{ job.scheduled|annotated_date }} ({{ job.scheduled|naturaltime }}) {% else %} - Created: {{ result.created|annotated_date }} + Created: {{ job.created|annotated_date }} {% endif %} - {% if result.completed %} - Duration: {{ result.duration }} + {% if job.completed %} + Duration: {{ job.duration }} {% endif %} - {% badge result.get_status_display result.get_status_color %} + {% badge job.get_status_display job.get_status_color %}

-{% if result.completed %} +{% if job.completed %}
Script Log
@@ -25,7 +25,7 @@ Level Message - {% for log in result.data.log %} + {% for log in job.data.log %} {{ forloop.counter }} {% log_level log.status %} @@ -47,11 +47,11 @@ {% endif %}

Output

- {% if result.data.output %} - 
{{ result.data.output }}
+ {% if job.data.output %} + 
{{ job.data.output }}
{% else %}

None

{% endif %} -{% elif result.started %} +{% elif job.started %} {% include 'extras/inc/result_pending.html' %} {% endif %} diff --git a/netbox/templates/extras/report.html b/netbox/templates/extras/report.html index 382c0669f..6cc52d914 100644 --- a/netbox/templates/extras/report.html +++ b/netbox/templates/extras/report.html @@ -55,7 +55,7 @@
{% if report.result %} - Last run: + Last run: {{ report.result.created|annotated_date }} {% endif %} diff --git a/netbox/templates/extras/report_list.html b/netbox/templates/extras/report_list.html index f2c527013..bc7469d3c 100644 --- a/netbox/templates/extras/report_list.html +++ b/netbox/templates/extras/report_list.html @@ -58,7 +58,7 @@ {{ report.description|markdown|placeholder }} {% if last_result %} - {{ last_result.created|annotated_date }} + {{ last_result.created|annotated_date }} {% badge last_result.get_status_display last_result.get_status_color %} diff --git a/netbox/templates/extras/report_result.html b/netbox/templates/extras/report_result.html index ffa52f9b7..9358af364 100644 --- a/netbox/templates/extras/report_result.html +++ b/netbox/templates/extras/report_result.html @@ -4,7 +4,7 @@ {% block content-wrapper %}
-
+
{% include 'extras/htmx/report_result.html' %}
@@ -13,8 +13,8 @@ {% block controls %}
- {% if request.user|can_delete:result %} - {% delete_button result %} + {% if request.user|can_delete:job %} + {% delete_button job %} {% endif %}
diff --git a/netbox/templates/extras/script_list.html b/netbox/templates/extras/script_list.html index 7377d5e8a..cfc26106b 100644 --- a/netbox/templates/extras/script_list.html +++ b/netbox/templates/extras/script_list.html @@ -58,7 +58,7 @@ {% if last_result %} - {{ last_result.created|annotated_date }} + {{ last_result.created|annotated_date }} {% badge last_result.get_status_display last_result.get_status_color %} diff --git a/netbox/templates/extras/script_result.html b/netbox/templates/extras/script_result.html index bff3fc61e..4dfd7482a 100644 --- a/netbox/templates/extras/script_result.html +++ b/netbox/templates/extras/script_result.html @@ -16,8 +16,8 @@
  1. Scripts
  2. {{ script.module|bettertitle }}
    3. -
  3. {{ script }}
    4. -
  4. {{ result.created|annotated_date }}
    5. +
  5. {{ script }}
    6. +
  6. {{ job.created|annotated_date }}
@@ -28,8 +28,8 @@ {% block controls %}
- {% if request.user|can_delete:result %} - {% delete_button result %} + {% if request.user|can_delete:job %} + {% delete_button job %} {% endif %}
@@ -47,7 +47,7 @@
-
+
{% include 'extras/htmx/script_result.html' %}