adds rq retry options #12327

docs/configuration/miscellaneous.md

@@ -204,3 +204,31 @@ This parameter defines the URL of the repository that will be checked for new Ne
 Default: `300`
 
 The maximum execution time of a background task (such as running a custom script), in seconds.
+
+---
+
+## RQ_RETRY_MAX
+
+!!! note
+    This parameter was added in NetBox v3.5.
+
+Default: `1`
+
+The maximum number of times a background task will be retried before being marked as failed.
+
+---
+
+## RQ_RETRY_INTERVAL
+
+!!! note
+    This parameter was added in NetBox v3.5.
+
+Default: `[]`
+
+The number of seconds to wait before retrying a failed background task. This is a list of integers which will be used 
+as an interval. For example, `[60, 300, 3600]` will retry the task after 1 minute, 5 minutes, and 1 hour.
+
+!!! warning
+    If you use the interval, you should run your workers with `--with-scheduler` argument.
+
+---

netbox/core/models/jobs.py

@@ -9,6 +9,7 @@ from django.db import models
 from django.urls import reverse
 from django.utils import timezone
 from django.utils.translation import gettext as _
+from rq import Retry
 
 from core.choices import JobStatusChoices
 from extras.constants import EVENT_JOB_END, EVENT_JOB_START
@@ -219,5 +220,6 @@ class Job(models.Model):
                 event=event,
                 data=self.data,
                 timestamp=str(timezone.now()),
-                username=self.user.username
+                username=self.user.username,
+                retry=Retry(max=get_config().RQ_RETRY_MAX, interval=get_config().RQ_RETRY_INTERVAL)
             )

netbox/extras/webhooks.py

@@ -4,6 +4,7 @@ import hmac
 from django.contrib.contenttypes.models import ContentType
 from django.utils import timezone
 from django_rq import get_queue
+from rq import Retry
 
 from netbox.config import get_config
 from netbox.constants import RQ_QUEUE_DEFAULT
@@ -116,5 +117,6 @@ def flush_webhooks(queue):
                 snapshots=data['snapshots'],
                 timestamp=str(timezone.now()),
                 username=data['username'],
-                request_id=data['request_id']
+                request_id=data['request_id'],
+                retry=Retry(max=get_config().RQ_RETRY_MAX, interval=get_config().RQ_RETRY_INTERVAL)
             )

netbox/netbox/settings.py

@@ -140,6 +140,8 @@ REMOTE_AUTH_STAFF_USERS = getattr(configuration, 'REMOTE_AUTH_STAFF_USERS', [])
 REMOTE_AUTH_GROUP_SEPARATOR = getattr(configuration, 'REMOTE_AUTH_GROUP_SEPARATOR', '|')
 REPORTS_ROOT = getattr(configuration, 'REPORTS_ROOT', os.path.join(BASE_DIR, 'reports')).rstrip('/')
 RQ_DEFAULT_TIMEOUT = getattr(configuration, 'RQ_DEFAULT_TIMEOUT', 300)
+RQ_RETRY_MAX = getattr(configuration, 'RQ_RETRY_MAX', 1)
+RQ_RETRY_INTERVAL = getattr(configuration, 'RQ_RETRY_INTERVAL', [])
 SCRIPTS_ROOT = getattr(configuration, 'SCRIPTS_ROOT', os.path.join(BASE_DIR, 'scripts')).rstrip('/')
 SEARCH_BACKEND = getattr(configuration, 'SEARCH_BACKEND', 'netbox.search.backends.CachedValueSearchBackend')
 SECURE_SSL_REDIRECT = getattr(configuration, 'SECURE_SSL_REDIRECT', False)