Closes #19891: Bulk operation jobs (#19897)

* Add background_job toggle to BulkEditForm

* Account for bug fix in v4.3.4

* Enable background jobs for bulk edit & bulk delete

* Move background_job field to a mixin

* Cosmetic improvements

* Misc cleanup

* Fix BackgroundJobMixin

netbox/netbox/forms/base.py

@@ -8,7 +8,7 @@ from django.utils.translation import gettext_lazy as _
 from core.models import ObjectType
 from extras.choices import *
 from extras.models import CustomField, Tag
-from utilities.forms import CSVModelForm
+from utilities.forms import BulkEditForm, CSVModelForm
 from utilities.forms.fields import CSVModelMultipleChoiceField, DynamicModelMultipleChoiceField
 from utilities.forms.mixins import CheckLastUpdatedMixin
 from .mixins import CustomFieldsMixin, SavedFiltersMixin, TagsMixin
@@ -100,7 +100,7 @@ class NetBoxModelImportForm(CSVModelForm, NetBoxModelForm):
         return customfield.to_form_field(for_csv_import=True)
 
 
-class NetBoxModelBulkEditForm(CustomFieldsMixin, forms.Form):
+class NetBoxModelBulkEditForm(CustomFieldsMixin, BulkEditForm):
     """
     Base form for modifying multiple NetBox objects (of the same type) in bulk via the UI. Adds support for custom
     fields and adding/removing tags.
@@ -108,9 +108,8 @@ class NetBoxModelBulkEditForm(CustomFieldsMixin, forms.Form):
     Attributes:
         fieldsets: An iterable of two-tuples which define a heading and field set to display per section of
             the rendered form (optional). If not defined, the all fields will be rendered as a single section.
-        nullable_fields: A list of field names indicating which fields support being set to null/empty
     """
-    nullable_fields = ()
+    fieldsets = None
 
     pk = forms.ModelMultipleChoiceField(
         queryset=None,  # Set from self.model on init

netbox/netbox/jobs.py

@@ -212,6 +212,5 @@ class AsyncViewJob(JobRunner):
         )
         notification.save()
 
-        # TODO: Waiting on fix for bug #19806
+        if data.errors:
-        # if errors:
+            raise JobFailed()
-        #     raise JobFailed()

netbox/netbox/views/generic/bulk_views.py

@@ -27,6 +27,7 @@ from utilities.exceptions import AbortRequest, AbortTransaction, PermissionsViol
 from utilities.export import TableExport
 from utilities.forms import BulkRenameForm, ConfirmationForm, restrict_form_fields
 from utilities.forms.bulk_import import BulkImportForm
+from utilities.forms.mixins import BackgroundJobMixin
 from utilities.htmx import htmx_partial
 from utilities.jobs import AsyncJobData, is_background_request, process_request_as_job
 from utilities.permissions import get_permission_for_model
@@ -513,12 +514,7 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
                     count=len(form.cleaned_data['data']),
                     object_type=model._meta.verbose_name_plural,
                 )
-                if job := process_request_as_job(self.__class__, request, name=job_name):
+                if process_request_as_job(self.__class__, request, name=job_name):
-                    msg = _('Created background job {job.pk}: <a href="{url}">{job.name}</a>').format(
-                        url=job.get_absolute_url(),
-                        job=job
-                    )
-                    messages.info(request, mark_safe(msg))
                     return redirect(redirect_url)
 
             try:
@@ -712,6 +708,16 @@ class BulkEditView(GetReturnURLMixin, BaseMultiObjectView):
         if '_apply' in request.POST:
             if form.is_valid():
                 logger.debug("Form validation was successful")
+
+                # If indicated, defer this request to a background job & redirect the user
+                if form.cleaned_data['background_job']:
+                    job_name = _('Bulk edit {count} {object_type}').format(
+                        count=len(form.cleaned_data['pk']),
+                        object_type=model._meta.verbose_name_plural,
+                    )
+                    if process_request_as_job(self.__class__, request, name=job_name):
+                        return redirect(self.get_return_url(request))
+
                 try:
                     with transaction.atomic(using=router.db_for_write(model)):
                         updated_objects = self._update_objects(form, request)
@@ -721,6 +727,16 @@ class BulkEditView(GetReturnURLMixin, BaseMultiObjectView):
                         if object_count != len(updated_objects):
                             raise PermissionsViolation
 
+                    # If this request was executed via a background job, return the raw data for logging
+                    if is_background_request(request):
+                        return AsyncJobData(
+                            log=[
+                                _('Updated {object}').format(object=str(obj))
+                                for obj in updated_objects
+                            ],
+                            errors=form.errors
+                        )
+
                     if updated_objects:
                         msg = f'Updated {len(updated_objects)} {model._meta.verbose_name_plural}'
                         logger.info(msg)
@@ -876,7 +892,7 @@ class BulkDeleteView(GetReturnURLMixin, BaseMultiObjectView):
         """
         Provide a standard bulk delete form if none has been specified for the view
         """
-        class BulkDeleteForm(ConfirmationForm):
+        class BulkDeleteForm(BackgroundJobMixin, ConfirmationForm):
             pk = ModelMultipleChoiceField(queryset=self.queryset, widget=MultipleHiddenInput)
 
         return BulkDeleteForm
@@ -908,6 +924,15 @@ class BulkDeleteView(GetReturnURLMixin, BaseMultiObjectView):
             if form.is_valid():
                 logger.debug("Form validation was successful")
 
+                # If indicated, defer this request to a background job & redirect the user
+                if form.cleaned_data['background_job']:
+                    job_name = _('Bulk delete {count} {object_type}').format(
+                        count=len(form.cleaned_data['pk']),
+                        object_type=model._meta.verbose_name_plural,
+                    )
+                    if process_request_as_job(self.__class__, request, name=job_name):
+                        return redirect(self.get_return_url(request))
+
                 # Delete objects
                 queryset = self.queryset.filter(pk__in=pk_list)
                 deleted_count = queryset.count()
@@ -929,6 +954,16 @@ class BulkDeleteView(GetReturnURLMixin, BaseMultiObjectView):
                     messages.error(request, mark_safe(e.message))
                     return redirect(self.get_return_url(request))
 
+                # If this request was executed via a background job, return the raw data for logging
+                if is_background_request(request):
+                    return AsyncJobData(
+                        log=[
+                            _('Deleted {object}').format(object=str(obj))
+                            for obj in queryset
+                        ],
+                        errors=form.errors
+                    )
+
                 msg = _("Deleted {count} {object_type}").format(
                     count=deleted_count,
                     object_type=model._meta.verbose_name_plural

netbox/templates/generic/bulk_delete.html

@@ -1,7 +1,8 @@
 {% extends 'generic/_base.html' %}
+{% load form_helpers %}
 {% load helpers %}
-{% load render_table from django_tables2 %}
 {% load i18n %}
+{% load render_table from django_tables2 %}
 
 {% comment %}
 Blocks:
@@ -58,13 +59,23 @@ Context:
     <div class="row mt-3">
       <form action="" method="post">
         {% csrf_token %}
+
+        {# Form fields #}
         {% for field in form.hidden_fields %}
           {{ field }}
         {% endfor %}
+
+        {# Meta fields #}
+        <div class="bg-primary-subtle border border-primary rounded-1 pt-3 mb-3">
+          {% render_field form.background_job %}
+        </div>
+
+        {# Form buttons #}
         <div class="text-end">
           <a href="{{ return_url }}" class="btn btn-outline-secondary">{% trans "Cancel" %}</a>
           <button type="submit" name="_confirm" class="btn btn-danger">{% trans "Delete" %} {{ table.rows|length }} {{ model|meta:"verbose_name_plural" }}</button>
         </div>
+
       </form>
     </div>
   </div>

netbox/templates/generic/bulk_edit.html

@@ -1,8 +1,8 @@
 {% extends 'generic/_base.html' %}
-{% load helpers %}
 {% load form_helpers %}
-{% load render_table from django_tables2 %}
+{% load helpers %}
 {% load i18n %}
+{% load render_table from django_tables2 %}
 
 {% comment %}
 Blocks:
@@ -102,6 +102,11 @@ Context:
 
         {% endif %}
 
+        {# Meta fields #}
+        <div class="bg-primary-subtle border border-primary rounded-1 pt-3 mb-3">
+          {% render_field form.background_job %}
+        </div>
+
         <div class="btn-float-group-right">
           <a href="{{ return_url }}" class="btn btn-outline-secondary btn-float">{% trans "Cancel" %}</a>
           <button type="submit" name="_apply" class="btn btn-primary">{% trans "Apply" %}</button>

netbox/templates/generic/bulk_import.html

@@ -1,6 +1,6 @@
 {% extends 'generic/_base.html' %}
-{% load helpers %}
 {% load form_helpers %}
+{% load helpers %}
 {% load i18n %}
 
 {% comment %}
@@ -47,10 +47,17 @@ Context:
         <form action="" method="post" enctype="multipart/form-data" class="form">
           {% csrf_token %}
           <input type="hidden" name="import_method" value="direct" />
+
+          {# Form fields #}
           {% render_field form.data %}
           {% render_field form.format %}
           {% render_field form.csv_delimiter %}
+
+          {# Meta fields #}
+          <div class="bg-primary-subtle border border-primary rounded-1 pt-3 mb-3">
             {% render_field form.background_job %}
+          </div>
+
           <div class="form-group">
             <div class="col col-md-12 text-end">
               {% if return_url %}
@@ -70,9 +77,12 @@ Context:
       <form action="" method="post" enctype="multipart/form-data" class="form">
         {% csrf_token %}
         <input type="hidden" name="import_method" value="upload" />
+
+        {# Form fields #}
         {% render_field form.upload_file %}
         {% render_field form.format %}
         {% render_field form.csv_delimiter %}
+
         <div class="form-group">
           <div class="col col-md-12 text-end">
             {% if return_url %}
@@ -91,11 +101,18 @@ Context:
       <form action="" method="post" enctype="multipart/form-data" class="form">
         {% csrf_token %}
         <input type="hidden" name="import_method" value="datafile" />
+
+        {# Form fields #}
         {% render_field form.data_source %}
         {% render_field form.data_file %}
         {% render_field form.format %}
         {% render_field form.csv_delimiter %}
+
+        {# Meta fields #}
+        <div class="bg-primary-subtle border border-primary rounded-1 pt-3 mb-3">
           {% render_field form.background_job %}
+        </div>
+
         <div class="form-group">
           <div class="col col-md-12 text-end">
             {% if return_url %}

netbox/users/forms/bulk_edit.py

@@ -17,7 +17,7 @@ __all__ = (
 )
 
 
-class UserBulkEditForm(forms.Form):
+class UserBulkEditForm(BulkEditForm):
     pk = forms.ModelMultipleChoiceField(
         queryset=User.objects.all(),
         widget=forms.MultipleHiddenInput
@@ -55,7 +55,7 @@ class UserBulkEditForm(forms.Form):
     nullable_fields = ('first_name', 'last_name')
 
 
-class GroupBulkEditForm(forms.Form):
+class GroupBulkEditForm(BulkEditForm):
     pk = forms.ModelMultipleChoiceField(
         queryset=Group.objects.all(),
         widget=forms.MultipleHiddenInput
@@ -73,7 +73,7 @@ class GroupBulkEditForm(forms.Form):
     nullable_fields = ('description',)
 
 
-class ObjectPermissionBulkEditForm(forms.Form):
+class ObjectPermissionBulkEditForm(BulkEditForm):
     pk = forms.ModelMultipleChoiceField(
         queryset=ObjectPermission.objects.all(),
         widget=forms.MultipleHiddenInput

netbox/utilities/forms/bulk_import.py

@@ -9,10 +9,11 @@ from django.utils.translation import gettext as _
 from core.forms.mixins import SyncedDataMixin
 from netbox.choices import CSVDelimiterChoices, ImportFormatChoices, ImportMethodChoices
 from utilities.constants import CSV_DELIMITERS
+from utilities.forms.mixins import BackgroundJobMixin
 from utilities.forms.utils import parse_csv
 
 
-class BulkImportForm(SyncedDataMixin, forms.Form):
+class BulkImportForm(BackgroundJobMixin, SyncedDataMixin, forms.Form):
     import_method = forms.ChoiceField(
         choices=ImportMethodChoices,
         required=False
@@ -37,11 +38,6 @@ class BulkImportForm(SyncedDataMixin, forms.Form):
         help_text=_("The character which delimits CSV fields. Applies only to CSV format."),
         required=False
     )
-    background_job = forms.BooleanField(
-        label=_('Background job'),
-        help_text=_("Enqueue a background job to complete the bulk import/update."),
-        required=False,
-    )
 
     data_field = 'data'
 

netbox/utilities/forms/forms.py

@@ -3,6 +3,8 @@ import re
 from django import forms
 from django.utils.translation import gettext as _
 
+from utilities.forms.mixins import BackgroundJobMixin
+
 __all__ = (
     'BulkEditForm',
     'BulkRenameForm',
@@ -28,9 +30,12 @@ class ConfirmationForm(forms.Form):
     )
 
 
-class BulkEditForm(forms.Form):
+class BulkEditForm(BackgroundJobMixin, forms.Form):
     """
     Provides bulk edit support for objects.
+
+    Attributes:
+        nullable_fields: A list of field names indicating which fields support being set to null/empty
     """
     nullable_fields = ()
 

netbox/utilities/forms/mixins.py

@@ -6,11 +6,20 @@ from django.core.validators import MaxValueValidator, MinValueValidator
 from django.utils.translation import gettext_lazy as _
 
 __all__ = (
+    'BackgroundJobMixin',
     'CheckLastUpdatedMixin',
     'DistanceValidationMixin',
 )
 
 
+class BackgroundJobMixin(forms.Form):
+    background_job = forms.BooleanField(
+        label=_('Background job'),
+        help_text=_("Execute this task via a background job"),
+        required=False,
+    )
+
+
 class CheckLastUpdatedMixin(forms.Form):
     """
     Checks whether the object being saved has been updated since the form was initialized. If so, validation fails.

netbox/utilities/jobs.py

@@ -1,6 +1,10 @@
 from dataclasses import dataclass
 from typing import List
 
+from django.contrib import messages
+from django.utils.safestring import mark_safe
+from django.utils.translation import gettext_lazy as _
+
 from netbox.jobs import AsyncViewJob
 from utilities.request import copy_safe_request
 
@@ -38,9 +42,19 @@ def process_request_as_job(view, request, name=None):
     request_copy._background = True
 
     # Enqueue a job to perform the work in the background
-    return AsyncViewJob.enqueue(
+    job = AsyncViewJob.enqueue(
         name=name,
         user=request.user,
         view_cls=view,
         request=request_copy,
     )
+
+    # Record a message on the original request indicating deferral to a background job
+    msg = _('Created background job {id}: <a href="{url}">{name}</a>').format(
+        id=job.pk,
+        url=job.get_absolute_url(),
+        name=job.name
+    )
+    messages.info(request, mark_safe(msg))
+
+    return job