From da77b7c41a6f96785cc270f1487f1c1142544390 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 2 Jul 2025 10:46:47 -0400 Subject: [PATCH 1/5] Initial work on #19589 --- netbox/netbox/jobs.py | 35 +++++++++++++++ netbox/netbox/middleware.py | 12 +----- netbox/netbox/views/generic/bulk_views.py | 52 +++++++++++++++++------ netbox/templates/generic/bulk_import.html | 3 ++ netbox/utilities/forms/bulk_import.py | 5 +++ netbox/utilities/request.py | 19 +++++++++ 6 files changed, 104 insertions(+), 22 deletions(-) diff --git a/netbox/netbox/jobs.py b/netbox/netbox/jobs.py index 3af3af554..73cc2f4e8 100644 --- a/netbox/netbox/jobs.py +++ b/netbox/netbox/jobs.py @@ -8,11 +8,15 @@ from django_pglocks import advisory_lock from rq.timeouts import JobTimeoutException from core.choices import JobStatusChoices +from core.events import JOB_COMPLETED, JOB_FAILED from core.models import Job, ObjectType +from extras.models import Notification from netbox.constants import ADVISORY_LOCK_KEYS from netbox.registry import registry +from utilities.request import apply_request_processors __all__ = ( + 'AsyncViewJob', 'JobRunner', 'system_job', ) @@ -154,3 +158,34 @@ class JobRunner(ABC): job.delete() return cls.enqueue(instance=instance, schedule_at=schedule_at, interval=interval, *args, **kwargs) + + +class AsyncViewJob(JobRunner): + """ + Execute a view as a background job. + """ + class Meta: + name = 'Async View' + + def run(self, view_cls, request, **kwargs): + view = view_cls.as_view() + + # Apply all registered request processors (e.g. event_tracking) + with apply_request_processors(request): + result, errors = view(request) + + self.job.data = { + 'result': result, + 'errors': errors, + } + # TODO: Figure out how to mark a job as "failed" + # if errors: + # self.job.terminate(status=JobStatusChoices.STATUS_FAILED, error=errors[0]) + + # Notify the user + notification = Notification( + user=request.user, + object=self.job, + event_type=JOB_COMPLETED if not errors else JOB_FAILED, + ) + notification.save() diff --git a/netbox/netbox/middleware.py b/netbox/netbox/middleware.py index 3a9cb0f78..145141615 100644 --- a/netbox/netbox/middleware.py +++ b/netbox/netbox/middleware.py @@ -1,8 +1,5 @@ -from contextlib import ExitStack - import logging import uuid -import warnings from django.conf import settings from django.contrib import auth, messages @@ -13,10 +10,10 @@ from django.db.utils import InternalError from django.http import Http404, HttpResponseRedirect from netbox.config import clear_config, get_config -from netbox.registry import registry from netbox.views import handler_500 from utilities.api import is_api_request from utilities.error_handlers import handle_rest_api_exception +from utilities.request import apply_request_processors __all__ = ( 'CoreMiddleware', @@ -36,12 +33,7 @@ class CoreMiddleware: request.id = uuid.uuid4() # Apply all registered request processors - with ExitStack() as stack: - for request_processor in registry['request_processors']: - try: - stack.enter_context(request_processor(request)) - except Exception as e: - warnings.warn(f'Failed to initialize request processor {request_processor}: {e}') + with apply_request_processors(request): response = self.get_response(request) # Check if language cookie should be renewed diff --git a/netbox/netbox/views/generic/bulk_views.py b/netbox/netbox/views/generic/bulk_views.py index 08b060634..075599693 100644 --- a/netbox/netbox/views/generic/bulk_views.py +++ b/netbox/netbox/views/generic/bulk_views.py @@ -22,6 +22,7 @@ from core.models import ObjectType from core.signals import clear_events from extras.choices import CustomFieldUIEditableChoices from extras.models import CustomField, ExportTemplate +from netbox.jobs import AsyncViewJob from netbox.object_actions import AddObject, BulkDelete, BulkEdit, BulkExport, BulkImport from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortRequest, AbortTransaction, PermissionsViolation @@ -30,7 +31,7 @@ from utilities.forms.bulk_import import BulkImportForm from utilities.htmx import htmx_partial from utilities.permissions import get_permission_for_model from utilities.query import reapply_model_ordering -from utilities.request import safe_for_redirect +from utilities.request import copy_safe_request, safe_for_redirect from utilities.tables import get_table_configs from utilities.views import GetReturnURLMixin, get_viewname from .base import BaseMultiObjectView @@ -500,25 +501,38 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView): if form.is_valid(): logger.debug("Import form validation was successful") + redirect_url = reverse(get_viewname(model, action='list')) + new_objects = [] + + # Defer the request to a background job? + if form.cleaned_data['background_job'] and not getattr(request, '_background', False): + + # Create a serializable copy of the original request + request_copy = copy_safe_request(request) + request_copy._background = True + + # Enqueue a job to perform the work in the background + job = AsyncViewJob.enqueue( + user=request.user, + view_cls=self.__class__, + request=request_copy, + ) + msg = _("Background job enqueued: {job}").format(job=job.pk) + logger.info(msg) + messages.info(request, msg) + + # Redirect to the model's list view + return redirect(redirect_url) try: # Iterate through data and bind each record to a new model form instance. with transaction.atomic(using=router.db_for_write(model)): - new_objs = self.create_and_update_objects(form, request) + new_objects = self.create_and_update_objects(form, request) # Enforce object-level permissions - if self.queryset.filter(pk__in=[obj.pk for obj in new_objs]).count() != len(new_objs): + if self.queryset.filter(pk__in=[obj.pk for obj in new_objects]).count() != len(new_objects): raise PermissionsViolation - if new_objs: - msg = f"Imported {len(new_objs)} {model._meta.verbose_name_plural}" - logger.info(msg) - messages.success(request, msg) - - view_name = get_viewname(model, action='list') - results_url = f"{reverse(view_name)}?modified_by_request={request.id}" - return redirect(results_url) - except (AbortTransaction, ValidationError): clear_events.send(sender=self) @@ -527,6 +541,20 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView): form.add_error(None, e.message) clear_events.send(sender=self) + # If this request was executed via a background job, return the raw data for logging + if getattr(request, '_background', False): + result = [ + _('Created {object}').format(object=str(obj)) + for obj in new_objects + ] + return result, form.errors + + if new_objects: + msg = f"Imported {len(new_objects)} {model._meta.verbose_name_plural}" + logger.info(msg) + messages.success(request, msg) + return redirect(f"{redirect_url}?modified_by_request={request.id}") + else: logger.debug("Form validation failed") diff --git a/netbox/templates/generic/bulk_import.html b/netbox/templates/generic/bulk_import.html index 3a652d3e9..8e890969c 100644 --- a/netbox/templates/generic/bulk_import.html +++ b/netbox/templates/generic/bulk_import.html @@ -50,6 +50,7 @@ Context: {% render_field form.data %} {% render_field form.format %} {% render_field form.csv_delimiter %} + {% render_field form.background_job %}
{% if return_url %} @@ -72,6 +73,7 @@ Context: {% render_field form.upload_file %} {% render_field form.format %} {% render_field form.csv_delimiter %} + {% render_field form.background_job %}
{% if return_url %} @@ -94,6 +96,7 @@ Context: {% render_field form.data_file %} {% render_field form.format %} {% render_field form.csv_delimiter %} + {% render_field form.background_job %}
{% if return_url %} diff --git a/netbox/utilities/forms/bulk_import.py b/netbox/utilities/forms/bulk_import.py index 4ee6f09bd..0fa41f570 100644 --- a/netbox/utilities/forms/bulk_import.py +++ b/netbox/utilities/forms/bulk_import.py @@ -37,6 +37,11 @@ class BulkImportForm(SyncedDataMixin, forms.Form): help_text=_("The character which delimits CSV fields. Applies only to CSV format."), required=False ) + background_job = forms.BooleanField( + label=_('Background job'), + help_text=_("Enqueue a background job to complete the bulk import/update."), + required=False, + ) data_field = 'data' diff --git a/netbox/utilities/request.py b/netbox/utilities/request.py index 8af55e868..eadbe0f5c 100644 --- a/netbox/utilities/request.py +++ b/netbox/utilities/request.py @@ -1,13 +1,17 @@ +import warnings +from contextlib import ExitStack, contextmanager from urllib.parse import urlparse from django.utils.http import url_has_allowed_host_and_scheme from django.utils.translation import gettext_lazy as _ from netaddr import AddrFormatError, IPAddress +from netbox.registry import registry from .constants import HTTP_REQUEST_META_SAFE_COPY __all__ = ( 'NetBoxFakeRequest', + 'apply_request_processors', 'copy_safe_request', 'get_client_ip', 'safe_for_redirect', @@ -48,6 +52,7 @@ def copy_safe_request(request): 'GET': request.GET, 'FILES': request.FILES, 'user': request.user, + 'method': request.method, 'path': request.path, 'id': getattr(request, 'id', None), # UUID assigned by middleware }) @@ -87,3 +92,17 @@ def safe_for_redirect(url): Returns True if the given URL is safe to use as an HTTP redirect; otherwise returns False. """ return url_has_allowed_host_and_scheme(url, allowed_hosts=None) + + +@contextmanager +def apply_request_processors(request): + """ + A context manager with applies all registered request processors (such as event_tracking). + """ + with ExitStack() as stack: + for request_processor in registry['request_processors']: + try: + stack.enter_context(request_processor(request)) + except Exception as e: + warnings.warn(f'Failed to initialize request processor {request_processor.__name__}: {e}') + yield From bcb0c6ccbcec39c887495ad92de4f3f536b86da8 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 2 Jul 2025 15:43:19 -0400 Subject: [PATCH 2/5] Add tooling for handling background requests --- netbox/netbox/jobs.py | 15 ++++---- netbox/netbox/views/generic/bulk_views.py | 47 +++++++++-------------- netbox/utilities/jobs.py | 45 ++++++++++++++++++++++ 3 files changed, 72 insertions(+), 35 deletions(-) create mode 100644 netbox/utilities/jobs.py diff --git a/netbox/netbox/jobs.py b/netbox/netbox/jobs.py index 73cc2f4e8..b7a6a6db1 100644 --- a/netbox/netbox/jobs.py +++ b/netbox/netbox/jobs.py @@ -172,20 +172,21 @@ class AsyncViewJob(JobRunner): # Apply all registered request processors (e.g. event_tracking) with apply_request_processors(request): - result, errors = view(request) + data = view(request) self.job.data = { - 'result': result, - 'errors': errors, + 'log': data.log, + 'errors': data.errors, } - # TODO: Figure out how to mark a job as "failed" - # if errors: - # self.job.terminate(status=JobStatusChoices.STATUS_FAILED, error=errors[0]) # Notify the user notification = Notification( user=request.user, object=self.job, - event_type=JOB_COMPLETED if not errors else JOB_FAILED, + event_type=JOB_COMPLETED if not data.errors else JOB_FAILED, ) notification.save() + + # TODO: Waiting on fix for bug #19806 + # if errors: + # raise JobFailed() diff --git a/netbox/netbox/views/generic/bulk_views.py b/netbox/netbox/views/generic/bulk_views.py index 075599693..23577e7fb 100644 --- a/netbox/netbox/views/generic/bulk_views.py +++ b/netbox/netbox/views/generic/bulk_views.py @@ -22,16 +22,16 @@ from core.models import ObjectType from core.signals import clear_events from extras.choices import CustomFieldUIEditableChoices from extras.models import CustomField, ExportTemplate -from netbox.jobs import AsyncViewJob from netbox.object_actions import AddObject, BulkDelete, BulkEdit, BulkExport, BulkImport from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortRequest, AbortTransaction, PermissionsViolation from utilities.forms import BulkRenameForm, ConfirmationForm, restrict_form_fields from utilities.forms.bulk_import import BulkImportForm from utilities.htmx import htmx_partial +from utilities.jobs import AsyncJobData, is_background_request, process_request_as_job from utilities.permissions import get_permission_for_model from utilities.query import reapply_model_ordering -from utilities.request import copy_safe_request, safe_for_redirect +from utilities.request import safe_for_redirect from utilities.tables import get_table_configs from utilities.views import GetReturnURLMixin, get_viewname from .base import BaseMultiObjectView @@ -504,25 +504,11 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView): redirect_url = reverse(get_viewname(model, action='list')) new_objects = [] - # Defer the request to a background job? - if form.cleaned_data['background_job'] and not getattr(request, '_background', False): - - # Create a serializable copy of the original request - request_copy = copy_safe_request(request) - request_copy._background = True - - # Enqueue a job to perform the work in the background - job = AsyncViewJob.enqueue( - user=request.user, - view_cls=self.__class__, - request=request_copy, - ) - msg = _("Background job enqueued: {job}").format(job=job.pk) - logger.info(msg) - messages.info(request, msg) - - # Redirect to the model's list view - return redirect(redirect_url) + # If indicated, defer this request to a background job & redirect the user + if form.cleaned_data['background_job']: + if job := process_request_as_job(self.__class__, request): + messages.info(request, _("Background job enqueued: {job}").format(job=job.pk)) + return redirect(redirect_url) try: # Iterate through data and bind each record to a new model form instance. @@ -542,15 +528,20 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView): clear_events.send(sender=self) # If this request was executed via a background job, return the raw data for logging - if getattr(request, '_background', False): - result = [ - _('Created {object}').format(object=str(obj)) - for obj in new_objects - ] - return result, form.errors + if is_background_request(request): + return AsyncJobData( + log=[ + _('Created {object}').format(object=str(obj)) + for obj in new_objects + ], + errors=form.errors + ) if new_objects: - msg = f"Imported {len(new_objects)} {model._meta.verbose_name_plural}" + msg = _("Imported {count} {object_type}").format( + count=len(new_objects), + object_type=model._meta.verbose_name_plural + ) logger.info(msg) messages.success(request, msg) return redirect(f"{redirect_url}?modified_by_request={request.id}") diff --git a/netbox/utilities/jobs.py b/netbox/utilities/jobs.py new file mode 100644 index 000000000..d7ace484c --- /dev/null +++ b/netbox/utilities/jobs.py @@ -0,0 +1,45 @@ +from dataclasses import dataclass +from typing import List + +from netbox.jobs import AsyncViewJob +from utilities.request import copy_safe_request + +__all__ = ( + 'AsyncJobData', + 'is_background_request', + 'process_request_as_job', +) + + +@dataclass +class AsyncJobData: + log: List[str] + errors: List[str] + + +def is_background_request(request): + """ + Return True if the request is being processed as a background job. + """ + return getattr(request, '_background', False) + + +def process_request_as_job(view, request): + """ + Process a request using a view as a background job. + """ + + # Check that the request that is not already being processed as a background job (would be a loop) + if is_background_request(request): + return + + # Create a serializable copy of the original request + request_copy = copy_safe_request(request) + request_copy._background = True + + # Enqueue a job to perform the work in the background + return AsyncViewJob.enqueue( + user=request.user, + view_cls=view, + request=request_copy, + ) From 99b23d0df13a3ef57ec4901cb80f295fa0949ca3 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 2 Jul 2025 15:52:35 -0400 Subject: [PATCH 3/5] UI notification should link to enqueued job --- netbox/netbox/views/generic/bulk_views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/netbox/netbox/views/generic/bulk_views.py b/netbox/netbox/views/generic/bulk_views.py index 23577e7fb..eb4c2060e 100644 --- a/netbox/netbox/views/generic/bulk_views.py +++ b/netbox/netbox/views/generic/bulk_views.py @@ -507,7 +507,11 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView): # If indicated, defer this request to a background job & redirect the user if form.cleaned_data['background_job']: if job := process_request_as_job(self.__class__, request): - messages.info(request, _("Background job enqueued: {job}").format(job=job.pk)) + msg = _('Created background job {job}').format( + url=job.get_absolute_url(), + job=job + ) + messages.info(request, mark_safe(msg)) return redirect(redirect_url) try: From 3f7d24b596806f19daa4ce8dd9efa427fd24dd01 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 3 Jul 2025 10:31:27 -0400 Subject: [PATCH 4/5] Use an informative name for the job --- netbox/core/models/jobs.py | 2 +- netbox/netbox/views/generic/bulk_views.py | 8 ++++++-- netbox/utilities/jobs.py | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/netbox/core/models/jobs.py b/netbox/core/models/jobs.py index 779e767b6..b36cc16f5 100644 --- a/netbox/core/models/jobs.py +++ b/netbox/core/models/jobs.py @@ -116,7 +116,7 @@ class Job(models.Model): verbose_name_plural = _('jobs') def __str__(self): - return str(self.job_id) + return self.name def get_absolute_url(self): # TODO: Employ dynamic registration diff --git a/netbox/netbox/views/generic/bulk_views.py b/netbox/netbox/views/generic/bulk_views.py index eb4c2060e..67c8d6c56 100644 --- a/netbox/netbox/views/generic/bulk_views.py +++ b/netbox/netbox/views/generic/bulk_views.py @@ -506,8 +506,12 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView): # If indicated, defer this request to a background job & redirect the user if form.cleaned_data['background_job']: - if job := process_request_as_job(self.__class__, request): - msg = _('Created background job {job}').format( + job_name = _('Bulk import {count} {object_type}').format( + count=len(form.cleaned_data['data']), + object_type=model._meta.verbose_name_plural, + ) + if job := process_request_as_job(self.__class__, request, name=job_name): + msg = _('Created background job {job.pk}: {job.name}').format( url=job.get_absolute_url(), job=job ) diff --git a/netbox/utilities/jobs.py b/netbox/utilities/jobs.py index d7ace484c..800d54949 100644 --- a/netbox/utilities/jobs.py +++ b/netbox/utilities/jobs.py @@ -24,7 +24,7 @@ def is_background_request(request): return getattr(request, '_background', False) -def process_request_as_job(view, request): +def process_request_as_job(view, request, name=None): """ Process a request using a view as a background job. """ @@ -39,6 +39,7 @@ def process_request_as_job(view, request): # Enqueue a job to perform the work in the background return AsyncViewJob.enqueue( + name=name, user=request.user, view_cls=view, request=request_copy, From fccccd0f0cd840155e6e404687adf6bd24a0b305 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 3 Jul 2025 14:25:08 -0400 Subject: [PATCH 5/5] Disable background jobs for file uploads --- netbox/templates/generic/bulk_import.html | 1 - 1 file changed, 1 deletion(-) diff --git a/netbox/templates/generic/bulk_import.html b/netbox/templates/generic/bulk_import.html index 8e890969c..b743f8b15 100644 --- a/netbox/templates/generic/bulk_import.html +++ b/netbox/templates/generic/bulk_import.html @@ -73,7 +73,6 @@ Context: {% render_field form.upload_file %} {% render_field form.format %} {% render_field form.csv_delimiter %} - {% render_field form.background_job %}
{% if return_url %}