NeoAnd/netbox
1
0
Fork 0
mirror of https://github.com/netbox-community/netbox.git synced 2025-07-22 20:12:00 -06:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

Closes #10945: Enable recurring execution of scheduled reports & scripts (#11096)

Browse Source 
* Add interval to JobResult

* Accept a recurrence interval when executing scripts & reports

* Cleaned up jobs list display

* Schedule next job only if a reference start time can be determined

* Improve validation for scheduled jobs
This commit is contained in:
Jeremy Stretch 2022-12-08 18:17:13 -05:00 committed by GitHub
parent 62b0f034e7
commit 4297c65f87
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 180 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
netbox/dcim/tables/sites.py
View File

@ -99,9 +99,9 @@ class SiteTable(TenancyColumnsMixin, ContactsColumnMixin, NetBoxTable):
class Meta(NetBoxTable.Meta):
model = Site
fields = (
'pk', 'id', 'name', 'slug', 'status', 'facility', 'region', 'group', 'tenant', 'tenant_group', 'asns', 'asn_count',
'time_zone', 'description', 'physical_address', 'shipping_address', 'latitude', 'longitude', 'comments',
'contacts', 'tags', 'created', 'last_updated', 'actions',
'pk', 'id', 'name', 'slug', 'status', 'facility', 'region', 'group', 'tenant', 'tenant_group', 'asns',
'asn_count', 'time_zone', 'description', 'physical_address', 'shipping_address', 'latitude', 'longitude',
'comments', 'contacts', 'tags', 'created', 'last_updated', 'actions',
)
default_columns = ('pk', 'name', 'status', 'facility', 'region', 'group', 'tenant', 'description')

6
netbox/extras/api/serializers.py
View File

@ -385,8 +385,8 @@ class JobResultSerializer(BaseModelSerializer):
class Meta:
model = JobResult
fields = [
'id', 'url', 'display', 'status', 'created', 'scheduled', 'started', 'completed', 'name', 'obj_type',
'user', 'data', 'job_id',
'id', 'url', 'display', 'status', 'created', 'scheduled', 'interval', 'started', 'completed', 'name',
'obj_type', 'user', 'data', 'job_id',
]
@ -414,6 +414,7 @@ class ReportDetailSerializer(ReportSerializer):
class ReportInputSerializer(serializers.Serializer):
schedule_at = serializers.DateTimeField(required=False, allow_null=True)
interval = serializers.IntegerField(required=False, allow_null=True)
#
@ -448,6 +449,7 @@ class ScriptInputSerializer(serializers.Serializer):
data = serializers.JSONField()
commit = serializers.BooleanField()
schedule_at = serializers.DateTimeField(required=False, allow_null=True)
interval = serializers.IntegerField(required=False, allow_null=True)
class ScriptLogMessageSerializer(serializers.Serializer):

31
netbox/extras/api/views.py
View File

@ -1,5 +1,4 @@
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q
from django.http import Http404
from django_rq.queues import get_connection
from rest_framework import status
@ -246,16 +245,14 @@ class ReportViewSet(ViewSet):
input_serializer = serializers.ReportInputSerializer(data=request.data)
if input_serializer.is_valid():
schedule_at = input_serializer.validated_data.get('schedule_at')
report_content_type = ContentType.objects.get(app_label='extras', model='report')
job_result = JobResult.enqueue_job(
run_report,
report.full_name,
report_content_type,
request.user,
name=report.full_name,
obj_type=ContentType.objects.get_for_model(Report),
user=request.user,
job_timeout=report.job_timeout,
schedule_at=schedule_at,
schedule_at=input_serializer.validated_data.get('schedule_at'),
interval=input_serializer.validated_data.get('interval')
)
report.result = job_result
@ -329,21 +326,17 @@ class ScriptViewSet(ViewSet):
raise RQWorkerNotRunningException()
if input_serializer.is_valid():
data = input_serializer.data['data']
commit = input_serializer.data['commit']
schedule_at = input_serializer.validated_data.get('schedule_at')
script_content_type = ContentType.objects.get(app_label='extras', model='script')
job_result = JobResult.enqueue_job(
run_script,
script.full_name,
script_content_type,
request.user,
data=data,
name=script.full_name,
obj_type=ContentType.objects.get_for_model(Script),
user=request.user,
data=input_serializer.data['data'],
request=copy_safe_request(request),
commit=commit,
commit=input_serializer.data['commit'],
job_timeout=script.job_timeout,
schedule_at=schedule_at,
schedule_at=input_serializer.validated_data.get('schedule_at'),
interval=input_serializer.validated_data.get('interval')
)
script.result = job_result
serializer = serializers.ScriptDetailSerializer(script, context={'request': request})

12
netbox/extras/choices.py
View File

@ -148,12 +148,12 @@ class JobResultStatusChoices(ChoiceSet):
STATUS_FAILED = 'failed'
CHOICES = (
(STATUS_PENDING, 'Pending'),
(STATUS_SCHEDULED, 'Scheduled'),
(STATUS_RUNNING, 'Running'),
(STATUS_COMPLETED, 'Completed'),
(STATUS_ERRORED, 'Errored'),
(STATUS_FAILED, 'Failed'),
(STATUS_PENDING, 'Pending', 'cyan'),
(STATUS_SCHEDULED, 'Scheduled', 'gray'),
(STATUS_RUNNING, 'Running', 'blue'),
(STATUS_COMPLETED, 'Completed', 'green'),
(STATUS_ERRORED, 'Errored', 'red'),
(STATUS_FAILED, 'Failed', 'red'),
)
TERMINAL_STATE_CHOICES = (

4
netbox/extras/filtersets.py
View File

@ -17,10 +17,10 @@ __all__ = (
'ConfigContextFilterSet',
'ContentTypeFilterSet',
'CustomFieldFilterSet',
'JobResultFilterSet',
'CustomLinkFilterSet',
'ExportTemplateFilterSet',
'ImageAttachmentFilterSet',
'JobResultFilterSet',
'JournalEntryFilterSet',
'LocalConfigContextFilterSet',
'ObjectChangeFilterSet',
@ -537,7 +537,7 @@ class JobResultFilterSet(BaseFilterSet):
class Meta:
model = JobResult
fields = ('id', 'status', 'user', 'obj_type', 'name')
fields = ('id', 'interval', 'status', 'user', 'obj_type', 'name')
def search(self, queryset, name, value):
if not value.strip():

14
netbox/extras/forms/reports.py
View File

@ -1,4 +1,5 @@
from django import forms
from django.utils import timezone
from django.utils.translation import gettext as _
from utilities.forms import BootstrapMixin, DateTimePicker
@ -15,3 +16,16 @@ class ReportForm(BootstrapMixin, forms.Form):
label=_("Schedule at"),
help_text=_("Schedule execution of report to a set time"),
)
interval = forms.IntegerField(
required=False,
min_value=1,
label=_("Recurs every"),
help_text=_("Interval at which this report is re-run (in minutes)")
)
def clean_schedule_at(self):
scheduled_time = self.cleaned_data['schedule_at']
if scheduled_time and scheduled_time < timezone.now():
raise forms.ValidationError(_('Scheduled time must be in the future.'))
return scheduled_time

22
netbox/extras/forms/scripts.py
View File

@ -1,4 +1,5 @@
from django import forms
from django.utils import timezone
from django.utils.translation import gettext as _
from utilities.forms import BootstrapMixin, DateTimePicker
@ -21,19 +22,36 @@ class ScriptForm(BootstrapMixin, forms.Form):
label=_("Schedule at"),
help_text=_("Schedule execution of script to a set time"),
)
_interval = forms.IntegerField(
required=False,
min_value=1,
label=_("Recurs every"),
help_text=_("Interval at which this script is re-run (in minutes)")
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Move _commit and _schedule_at to the end of the form
schedule_at = self.fields.pop('_schedule_at')
interval = self.fields.pop('_interval')
commit = self.fields.pop('_commit')
self.fields['_schedule_at'] = schedule_at
self.fields['_interval'] = interval
self.fields['_commit'] = commit
def clean__schedule_at(self):
scheduled_time = self.cleaned_data['_schedule_at']
if scheduled_time and scheduled_time < timezone.now():
raise forms.ValidationError({
'_schedule_at': _('Scheduled time must be in the future.')
})
return scheduled_time
@property
def requires_input(self):
"""
A boolean indicating whether the form requires user input (ignore the _commit and _schedule_at fields).
A boolean indicating whether the form requires user input (ignore the built-in fields).
"""
return bool(len(self.fields) > 2)
return bool(len(self.fields) > 3)

6
netbox/extras/migrations/0079_scheduled_jobs.py
View File

@ -1,3 +1,4 @@
import django.core.validators
from django.db import migrations, models
@ -13,6 +14,11 @@ class Migration(migrations.Migration):
name='scheduled',
field=models.DateTimeField(blank=True, null=True),
),
migrations.AddField(
model_name='jobresult',
name='interval',
field=models.PositiveIntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(1)]),
),
migrations.AddField(
model_name='jobresult',
name='started',

44
netbox/extras/models/models.py
View File

@ -7,7 +7,7 @@ from django.contrib.auth.models import User
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core.cache import cache
from django.core.validators import ValidationError
from django.core.validators import MinValueValidator, ValidationError
from django.db import models
from django.http import HttpResponse, QueryDict
from django.urls import reverse
@ -587,6 +587,14 @@ class JobResult(models.Model):
null=True,
blank=True
)
interval = models.PositiveIntegerField(
blank=True,
null=True,
validators=(
MinValueValidator(1),
),
help_text=_("Recurrence interval (in minutes)")
)
started = models.DateTimeField(
null=True,
blank=True
@ -635,6 +643,9 @@ class JobResult(models.Model):
def get_absolute_url(self):
return reverse(f'extras:{self.obj_type.name}_result', args=[self.pk])
def get_status_color(self):
return JobResultStatusChoices.colors.get(self.status)
@property
def duration(self):
if not self.completed:
@ -664,33 +675,32 @@ class JobResult(models.Model):
self.completed = timezone.now()
@classmethod
def enqueue_job(cls, func, name, obj_type, user, schedule_at=None, *args, **kwargs):
def enqueue_job(cls, func, name, obj_type, user, schedule_at=None, interval=None, *args, **kwargs):
"""
Create a JobResult instance and enqueue a job using the given callable
func: The callable object to be enqueued for execution
name: Name for the JobResult instance
obj_type: ContentType to link to the JobResult instance obj_type
user: User object to link to the JobResult instance
schedule_at: Schedule the job to be executed at the passed date and time
args: additional args passed to the callable
kwargs: additional kargs passed to the callable
Args:
func: The callable object to be enqueued for execution
name: Name for the JobResult instance
obj_type: ContentType to link to the JobResult instance obj_type
user: User object to link to the JobResult instance
schedule_at: Schedule the job to be executed at the passed date and time
interval: Recurrence interval (in minutes)
"""
job_result: JobResult = cls.objects.create(
rq_queue_name = get_config().QUEUE_MAPPINGS.get(obj_type.name, RQ_QUEUE_DEFAULT)
queue = django_rq.get_queue(rq_queue_name)
status = JobResultStatusChoices.STATUS_SCHEDULED if schedule_at else JobResultStatusChoices.STATUS_PENDING
job_result: JobResult = JobResult.objects.create(
name=name,
status=status,
obj_type=obj_type,
scheduled=schedule_at,
interval=interval,
user=user,
job_id=uuid.uuid4()
)
rq_queue_name = get_config().QUEUE_MAPPINGS.get(obj_type.name, RQ_QUEUE_DEFAULT)
queue = django_rq.get_queue(rq_queue_name)
if schedule_at:
job_result.status = JobResultStatusChoices.STATUS_SCHEDULED
job_result.scheduled = schedule_at
job_result.save()
queue.enqueue_at(schedule_at, func, job_id=str(job_result.job_id), job_result=job_result, **kwargs)
else:
queue.enqueue(func, job_id=str(job_result.job_id), job_result=job_result, **kwargs)

19
netbox/extras/reports.py
View File

@ -1,8 +1,8 @@
import importlib
import inspect
import logging
import pkgutil
import traceback
from datetime import timedelta
from django.conf import settings
from django.utils import timezone
@ -11,7 +11,6 @@ from django_rq import job
from .choices import JobResultStatusChoices, LogLevelChoices
from .models import JobResult
logger = logging.getLogger(__name__)
@ -85,10 +84,24 @@ def run_report(job_result, *args, **kwargs):
try:
job_result.start()
report.run(job_result)
except Exception as e:
except Exception:
job_result.set_status(JobResultStatusChoices.STATUS_ERRORED)
job_result.save()
logging.error(f"Error during execution of report {job_result.name}")
finally:
# Schedule the next job if an interval has been set
start_time = job_result.scheduled or job_result.started
if start_time and job_result.interval:
new_scheduled_time = start_time + timedelta(minutes=job_result.interval)
JobResult.enqueue_job(
run_report,
name=job_result.name,
obj_type=job_result.obj_type,
user=job_result.user,
job_timeout=report.job_timeout,
schedule_at=new_scheduled_time,
interval=job_result.interval
)
class Report(object):

20
netbox/extras/scripts.py
View File

@ -4,8 +4,9 @@ import logging
import os
import pkgutil
import sys
import traceback
import threading
import traceback
from datetime import timedelta
import yaml
from django import forms
@ -16,6 +17,7 @@ from django.utils.functional import classproperty
from extras.api.serializers import ScriptOutputSerializer
from extras.choices import JobResultStatusChoices, LogLevelChoices
from extras.models import JobResult
from extras.signals import clear_webhooks
from ipam.formfields import IPAddressFormField, IPNetworkFormField
from ipam.validators import MaxPrefixLengthValidator, MinPrefixLengthValidator, prefix_validator
@ -491,6 +493,22 @@ def run_script(data, request, commit=True, *args, **kwargs):
else:
_run_script()
# Schedule the next job if an interval has been set
if job_result.interval:
new_scheduled_time = job_result.scheduled + timedelta(minutes=job_result.interval)
JobResult.enqueue_job(
run_script,
name=job_result.name,
obj_type=job_result.obj_type,
user=job_result.user,
schedule_at=new_scheduled_time,
interval=job_result.interval,
job_timeout=script.job_timeout,
data=data,
request=request,
commit=commit
)
def get_scripts(use_names=False):
"""

19
netbox/extras/tables/tables.py
View File

@ -1,5 +1,6 @@
import django_tables2 as tables
from django.conf import settings
from django.utils.translation import gettext as _
from extras.models import *
from netbox.tables import NetBoxTable, columns
@ -8,9 +9,9 @@ from .template_code import *
__all__ = (
'ConfigContextTable',
'CustomFieldTable',
'JobResultTable',
'CustomLinkTable',
'ExportTemplateTable',
'JobResultTable',
'JournalEntryTable',
'ObjectChangeTable',
'SavedFilterTable',
@ -41,7 +42,15 @@ class JobResultTable(NetBoxTable):
name = tables.Column(
linkify=True
)
obj_type = columns.ContentTypeColumn(
verbose_name=_('Type')
)
status = columns.ChoiceFieldColumn()
created = columns.DateTimeColumn()
scheduled = columns.DateTimeColumn()
interval = columns.DurationColumn()
started = columns.DateTimeColumn()
completed = columns.DateTimeColumn()
actions = columns.ActionsColumn(
actions=('delete',)
)
@ -49,10 +58,12 @@ class JobResultTable(NetBoxTable):
class Meta(NetBoxTable.Meta):
model = JobResult
fields = (
'pk', 'id', 'name', 'obj_type', 'status', 'created', 'scheduled', 'started', 'completed', 'user', 'job_id',
'pk', 'id', 'obj_type', 'name', 'status', 'created', 'scheduled', 'interval', 'started', 'completed',
'user', 'job_id',
)
default_columns = (
'pk', 'id', 'name', 'obj_type', 'status', 'created', 'scheduled', 'started', 'completed', 'user',
'pk', 'id', 'obj_type', 'name', 'status', 'created', 'scheduled', 'interval', 'started', 'completed',
'user',
)

32
netbox/extras/views.py
View File

@ -676,7 +676,6 @@ class ReportView(ContentTypePermissionRequiredMixin, View):
form = ReportForm(request.POST)
if form.is_valid():
schedule_at = form.cleaned_data.get("schedule_at")
# Allow execution only if RQ worker process is running
if not Worker.count(get_connection('default')):
@ -686,14 +685,14 @@ class ReportView(ContentTypePermissionRequiredMixin, View):
})
# Run the Report. A new JobResult is created.
report_content_type = ContentType.objects.get(app_label='extras', model='report')
job_result = JobResult.enqueue_job(
run_report,
report.full_name,
report_content_type,
request.user,
job_timeout=report.job_timeout,
schedule_at=schedule_at,
name=report.full_name,
obj_type=ContentType.objects.get_for_model(Report),
user=request.user,
schedule_at=form.cleaned_data.get('schedule_at'),
interval=form.cleaned_data.get('interval'),
job_timeout=report.job_timeout
)
return redirect('extras:report_result', job_result_pk=job_result.pk)
@ -787,9 +786,8 @@ class ScriptView(ContentTypePermissionRequiredMixin, GetScriptMixin, View):
form = script.as_form(initial=normalize_querydict(request.GET))
# Look for a pending JobResult (use the latest one by creation timestamp)
script_content_type = ContentType.objects.get(app_label='extras', model='script')
script.result = JobResult.objects.filter(
obj_type=script_content_type,
obj_type=ContentType.objects.get_for_model(Script),
name=script.full_name,
).exclude(
status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES
@ -815,21 +813,17 @@ class ScriptView(ContentTypePermissionRequiredMixin, GetScriptMixin, View):
messages.error(request, "Unable to run script: RQ worker process not running.")
elif form.is_valid():
commit = form.cleaned_data.pop('_commit')
schedule_at = form.cleaned_data.pop("_schedule_at")
script_content_type = ContentType.objects.get(app_label='extras', model='script')
job_result = JobResult.enqueue_job(
run_script,
script.full_name,
script_content_type,
request.user,
name=script.full_name,
obj_type=ContentType.objects.get_for_model(Script),
user=request.user,
schedule_at=form.cleaned_data.pop('_schedule_at'),
interval=form.cleaned_data.pop('_interval'),
data=form.cleaned_data,
request=copy_safe_request(request),
commit=commit,
job_timeout=script.job_timeout,
schedule_at=schedule_at,
commit=form.cleaned_data.pop('_commit')
)
return redirect('extras:script_result', job_result_pk=job_result.pk)

2
netbox/netbox/navigation/menu.py
View File

@ -299,7 +299,7 @@ OTHER_MENU = Menu(
),
MenuItem(
link='extras:jobresult_list',
link_text=_('Job Results'),
link_text=_('Jobs'),
permissions=['extras.view_jobresult'],
),
),

19
netbox/netbox/tables/columns.py
View File

@ -28,6 +28,7 @@ __all__ = (
'ContentTypesColumn',
'CustomFieldColumn',
'CustomLinkColumn',
'DurationColumn',
'LinkedCountColumn',
'MarkdownColumn',
'ManyToManyColumn',
@ -77,6 +78,24 @@ class DateTimeColumn(tables.DateTimeColumn):
return cls(**kwargs)
class DurationColumn(tables.Column):
"""
Express a duration of time (in minutes) in a human-friendly format. Example: 437 minutes becomes "7h 17m"
"""
def render(self, value):
ret = ''
if days := value // 1440:
ret += f'{days}d '
if hours := value % 1440 // 60:
ret += f'{hours}h '
if minutes := value % 60:
ret += f'{minutes}m'
return ret.strip()
def value(self, value):
return value
class ManyToManyColumn(tables.ManyToManyColumn):
"""
Overrides django-tables2's stock ManyToManyColumn to ensure that value() returns only plaintext data.

3
netbox/templates/extras/htmx/report_result.html
View File

@ -1,10 +1,11 @@
{% load humanize %}
{% load helpers %}
<p>
{% if result.started %}
Started: <strong>{{ result.started|annotated_date }}</strong>
{% elif result.scheduled %}
Scheduled for: <strong>{{ result.scheduled|annotated_date }}</strong>
Scheduled for: <strong>{{ result.scheduled|annotated_date }}</strong> ({{ result.scheduled|naturaltime }})
{% else %}
Created: <strong>{{ result.created|annotated_date }}</strong>
{% endif %}

2
netbox/templates/extras/htmx/script_result.html
View File

@ -5,7 +5,7 @@
{% if result.started %}
Started: <strong>{{ result.started|annotated_date }}</strong>
{% elif result.scheduled %}
Scheduled for: <strong>{{ result.scheduled|annotated_date }}</strong>
Scheduled for: <strong>{{ result.scheduled|annotated_date }}</strong> ({{ result.scheduled|naturaltime }})
{% else %}
Created: <strong>{{ result.created|annotated_date }}</strong>
{% endif %}