diff --git a/netbox/core/constants.py b/netbox/core/constants.py new file mode 100644 index 000000000..3c3382dcc --- /dev/null +++ b/netbox/core/constants.py @@ -0,0 +1,26 @@ +from dataclasses import dataclass + +from django.utils.translation import gettext_lazy as _ +from rq.job import JobStatus + +__all__ = ( + 'RQ_TASK_STATUSES', +) + + +@dataclass +class Status: + label: str + color: str + + +RQ_TASK_STATUSES = { + JobStatus.QUEUED: Status(_('Queued'), 'cyan'), + JobStatus.FINISHED: Status(_('Finished'), 'green'), + JobStatus.FAILED: Status(_('Failed'), 'red'), + JobStatus.STARTED: Status(_('Started'), 'blue'), + JobStatus.DEFERRED: Status(_('Deferred'), 'gray'), + JobStatus.SCHEDULED: Status(_('Scheduled'), 'purple'), + JobStatus.STOPPED: Status(_('Stopped'), 'orange'), + JobStatus.CANCELED: Status(_('Cancelled'), 'yellow'), +} diff --git a/netbox/core/tables/__init__.py b/netbox/core/tables/__init__.py index 29dc7d85e..8f219afa4 100644 --- a/netbox/core/tables/__init__.py +++ b/netbox/core/tables/__init__.py @@ -1,4 +1,5 @@ from .config import * from .data import * from .jobs import * +from .tasks import * from .plugins import * diff --git a/netbox/core/tables/columns.py b/netbox/core/tables/columns.py index 93f1e3901..f3d985bc3 100644 --- a/netbox/core/tables/columns.py +++ b/netbox/core/tables/columns.py @@ -1,9 +1,12 @@ import django_tables2 as tables +from django.utils.safestring import mark_safe +from core.constants import RQ_TASK_STATUSES from netbox.registry import registry __all__ = ( 'BackendTypeColumn', + 'RQJobStatusColumn', ) @@ -18,3 +21,16 @@ class BackendTypeColumn(tables.Column): def value(self, value): return value + + +class RQJobStatusColumn(tables.Column): + """ + Render a colored label for the status of an RQ job. + """ + def render(self, value): + status = RQ_TASK_STATUSES.get(value) + return mark_safe(f'{status.label}') + + def value(self, value): + status = RQ_TASK_STATUSES.get(value) + return status.label diff --git a/netbox/core/tables/tasks.py b/netbox/core/tables/tasks.py new file mode 100644 index 000000000..531ec6375 --- /dev/null +++ b/netbox/core/tables/tasks.py @@ -0,0 +1,134 @@ +import django_tables2 as tables +from django.utils.translation import gettext_lazy as _ +from django_tables2.utils import A + +from core.tables.columns import RQJobStatusColumn +from netbox.tables import BaseTable + + +class BackgroundQueueTable(BaseTable): + name = tables.Column( + verbose_name=_("Name") + ) + jobs = tables.Column( + linkify=("core:background_task_list", [A("index"), "queued"]), + verbose_name=_("Queued") + ) + oldest_job_timestamp = tables.Column( + verbose_name=_("Oldest Task") + ) + started_jobs = tables.Column( + linkify=("core:background_task_list", [A("index"), "started"]), + verbose_name=_("Active") + ) + deferred_jobs = tables.Column( + linkify=("core:background_task_list", [A("index"), "deferred"]), + verbose_name=_("Deferred") + ) + finished_jobs = tables.Column( + linkify=("core:background_task_list", [A("index"), "finished"]), + verbose_name=_("Finished") + ) + failed_jobs = tables.Column( + linkify=("core:background_task_list", [A("index"), "failed"]), + verbose_name=_("Failed") + ) + scheduled_jobs = tables.Column( + linkify=("core:background_task_list", [A("index"), "scheduled"]), + verbose_name=_("Scheduled") + ) + workers = tables.Column( + linkify=("core:worker_list", [A("index")]), + verbose_name=_("Workers") + ) + host = tables.Column( + accessor="connection_kwargs__host", + verbose_name=_("Host") + ) + port = tables.Column( + accessor="connection_kwargs__port", + verbose_name=_("Port") + ) + db = tables.Column( + accessor="connection_kwargs__db", + verbose_name=_("DB") + ) + pid = tables.Column( + accessor="scheduler__pid", + verbose_name=_("Scheduler PID") + ) + + class Meta(BaseTable.Meta): + empty_text = _('No queues found') + fields = ( + 'name', 'jobs', 'oldest_job_timestamp', 'started_jobs', 'deferred_jobs', 'finished_jobs', 'failed_jobs', + 'scheduled_jobs', 'workers', 'host', 'port', 'db', 'pid', + ) + default_columns = ( + 'name', 'jobs', 'started_jobs', 'deferred_jobs', 'finished_jobs', 'failed_jobs', 'scheduled_jobs', + 'workers', + ) + + +class BackgroundTaskTable(BaseTable): + id = tables.Column( + linkify=("core:background_task", [A("id")]), + verbose_name=_("ID") + ) + created_at = tables.DateTimeColumn( + verbose_name=_("Created") + ) + enqueued_at = tables.DateTimeColumn( + verbose_name=_("Enqueued") + ) + ended_at = tables.DateTimeColumn( + verbose_name=_("Ended") + ) + status = RQJobStatusColumn( + verbose_name=_("Status"), + accessor='get_status' + ) + callable = tables.Column( + empty_values=(), + verbose_name=_("Callable") + ) + + class Meta(BaseTable.Meta): + empty_text = _('No tasks found') + fields = ( + 'id', 'created_at', 'enqueued_at', 'ended_at', 'status', 'callable', + ) + default_columns = ( + 'id', 'created_at', 'enqueued_at', 'ended_at', 'status', 'callable', + ) + + def render_callable(self, value, record): + try: + return record.func_name + except Exception as e: + return repr(e) + + +class WorkerTable(BaseTable): + name = tables.Column( + linkify=("core:worker", [A("name")]), + verbose_name=_("Name") + ) + state = tables.Column( + verbose_name=_("State") + ) + birth_date = tables.DateTimeColumn( + verbose_name=_("Birth") + ) + pid = tables.Column( + verbose_name=_("PID") + ) + + class Meta(BaseTable.Meta): + empty_text = _('No workers found') + fields = ( + 'name', 'state', 'birth_date', 'pid', + ) + default_columns = ( + 'name', 'state', 'birth_date', 'pid', + ) diff --git a/netbox/core/tests/test_views.py b/netbox/core/tests/test_views.py index 16d07f376..b7a951a0f 100644 --- a/netbox/core/tests/test_views.py +++ b/netbox/core/tests/test_views.py @@ -1,6 +1,16 @@ -from django.utils import timezone +import logging +import uuid +from datetime import datetime -from utilities.testing import ViewTestCases, create_tags +from django.urls import reverse +from django.utils import timezone +from django_rq import get_queue +from django_rq.settings import QUEUES_MAP +from django_rq.workers import get_worker +from rq.job import Job as RQ_Job, JobStatus +from rq.registry import DeferredJobRegistry, FailedJobRegistry, FinishedJobRegistry, StartedJobRegistry + +from utilities.testing import TestCase, ViewTestCases, create_tags from ..models import * @@ -87,3 +97,211 @@ class DataFileTestCase( ), ) DataFile.objects.bulk_create(data_files) + + +class BackgroundTaskTestCase(TestCase): + user_permissions = () + + # Dummy worker functions + @staticmethod + def dummy_job_default(): + return "Job finished" + + @staticmethod + def dummy_job_high(): + return "Job finished" + + @staticmethod + def dummy_job_failing(): + raise Exception("Job failed") + + def setUp(self): + super().setUp() + self.user.is_staff = True + self.user.is_active = True + self.user.save() + + # Clear all queues prior to running each test + get_queue('default').connection.flushall() + get_queue('high').connection.flushall() + get_queue('low').connection.flushall() + + def test_background_queue_list(self): + url = reverse('core:background_queue_list') + + # Attempt to load view without permission + self.user.is_staff = False + self.user.save() + response = self.client.get(url) + self.assertEqual(response.status_code, 403) + + # Load view with permission + self.user.is_staff = True + self.user.save() + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertIn('default', str(response.content)) + self.assertIn('high', str(response.content)) + self.assertIn('low', str(response.content)) + + def test_background_tasks_list_default(self): + queue = get_queue('default') + queue.enqueue(self.dummy_job_default) + queue_index = QUEUES_MAP['default'] + + response = self.client.get(reverse('core:background_task_list', args=[queue_index, 'queued'])) + self.assertEqual(response.status_code, 200) + self.assertIn('BackgroundTaskTestCase.dummy_job_default', str(response.content)) + + def test_background_tasks_list_high(self): + queue = get_queue('high') + queue.enqueue(self.dummy_job_high) + queue_index = QUEUES_MAP['high'] + + response = self.client.get(reverse('core:background_task_list', args=[queue_index, 'queued'])) + self.assertEqual(response.status_code, 200) + self.assertIn('BackgroundTaskTestCase.dummy_job_high', str(response.content)) + + def test_background_tasks_list_finished(self): + queue = get_queue('default') + job = queue.enqueue(self.dummy_job_default) + queue_index = QUEUES_MAP['default'] + + registry = FinishedJobRegistry(queue.name, queue.connection) + registry.add(job, 2) + response = self.client.get(reverse('core:background_task_list', args=[queue_index, 'finished'])) + self.assertEqual(response.status_code, 200) + self.assertIn('BackgroundTaskTestCase.dummy_job_default', str(response.content)) + + def test_background_tasks_list_failed(self): + queue = get_queue('default') + job = queue.enqueue(self.dummy_job_default) + queue_index = QUEUES_MAP['default'] + + registry = FailedJobRegistry(queue.name, queue.connection) + registry.add(job, 2) + response = self.client.get(reverse('core:background_task_list', args=[queue_index, 'failed'])) + self.assertEqual(response.status_code, 200) + self.assertIn('BackgroundTaskTestCase.dummy_job_default', str(response.content)) + + def test_background_tasks_scheduled(self): + queue = get_queue('default') + queue.enqueue_at(datetime.now(), self.dummy_job_default) + queue_index = QUEUES_MAP['default'] + + response = self.client.get(reverse('core:background_task_list', args=[queue_index, 'scheduled'])) + self.assertEqual(response.status_code, 200) + self.assertIn('BackgroundTaskTestCase.dummy_job_default', str(response.content)) + + def test_background_tasks_list_deferred(self): + queue = get_queue('default') + job = queue.enqueue(self.dummy_job_default) + queue_index = QUEUES_MAP['default'] + + registry = DeferredJobRegistry(queue.name, queue.connection) + registry.add(job, 2) + response = self.client.get(reverse('core:background_task_list', args=[queue_index, 'deferred'])) + self.assertEqual(response.status_code, 200) + self.assertIn('BackgroundTaskTestCase.dummy_job_default', str(response.content)) + + def test_background_task(self): + queue = get_queue('default') + job = queue.enqueue(self.dummy_job_default) + + response = self.client.get(reverse('core:background_task', args=[job.id])) + self.assertEqual(response.status_code, 200) + self.assertIn('Background Tasks', str(response.content)) + self.assertIn(str(job.id), str(response.content)) + self.assertIn('Callable', str(response.content)) + self.assertIn('Meta', str(response.content)) + self.assertIn('Keyword Arguments', str(response.content)) + + def test_background_task_delete(self): + queue = get_queue('default') + job = queue.enqueue(self.dummy_job_default) + + response = self.client.post(reverse('core:background_task_delete', args=[job.id]), {'confirm': True}) + self.assertEqual(response.status_code, 302) + self.assertFalse(RQ_Job.exists(job.id, connection=queue.connection)) + self.assertNotIn(job.id, queue.job_ids) + + def test_background_task_requeue(self): + queue = get_queue('default') + + # Enqueue & run a job that will fail + job = queue.enqueue(self.dummy_job_failing) + worker = get_worker('default') + worker.work(burst=True) + self.assertTrue(job.is_failed) + + # Re-enqueue the failed job and check that its status has been reset + response = self.client.get(reverse('core:background_task_requeue', args=[job.id])) + self.assertEqual(response.status_code, 302) + self.assertFalse(job.is_failed) + + def test_background_task_enqueue(self): + queue = get_queue('default') + + # Enqueue some jobs that each depends on its predecessor + job = previous_job = None + for _ in range(0, 3): + job = queue.enqueue(self.dummy_job_default, depends_on=previous_job) + previous_job = job + + # Check that the last job to be enqueued has a status of deferred + self.assertIsNotNone(job) + self.assertEqual(job.get_status(), JobStatus.DEFERRED) + self.assertIsNone(job.enqueued_at) + + # Force-enqueue the deferred job + response = self.client.get(reverse('core:background_task_enqueue', args=[job.id])) + self.assertEqual(response.status_code, 302) + + # Check that job's status is updated correctly + job = queue.fetch_job(job.id) + self.assertEqual(job.get_status(), JobStatus.QUEUED) + self.assertIsNotNone(job.enqueued_at) + + def test_background_task_stop(self): + queue = get_queue('default') + + worker = get_worker('default') + job = queue.enqueue(self.dummy_job_default) + worker.prepare_job_execution(job) + + self.assertEqual(job.get_status(), JobStatus.STARTED) + + # Stop those jobs using the view + started_job_registry = StartedJobRegistry(queue.name, connection=queue.connection) + self.assertEqual(len(started_job_registry), 1) + response = self.client.get(reverse('core:background_task_stop', args=[job.id])) + self.assertEqual(response.status_code, 302) + worker.monitor_work_horse(job, queue) # Sets the job as Failed and removes from Started + self.assertEqual(len(started_job_registry), 0) + + canceled_job_registry = FailedJobRegistry(queue.name, connection=queue.connection) + self.assertEqual(len(canceled_job_registry), 1) + self.assertIn(job.id, canceled_job_registry) + + def test_worker_list(self): + worker1 = get_worker('default', name=uuid.uuid4().hex) + worker1.register_birth() + + worker2 = get_worker('high') + worker2.register_birth() + + queue_index = QUEUES_MAP['default'] + response = self.client.get(reverse('core:worker_list', args=[queue_index])) + self.assertEqual(response.status_code, 200) + self.assertIn(str(worker1.name), str(response.content)) + self.assertNotIn(str(worker2.name), str(response.content)) + + def test_worker(self): + worker1 = get_worker('default', name=uuid.uuid4().hex) + worker1.register_birth() + + response = self.client.get(reverse('core:worker', args=[worker1.name])) + self.assertEqual(response.status_code, 200) + self.assertIn(str(worker1.name), str(response.content)) + self.assertIn('Birth', str(response.content)) + self.assertIn('Total working time', str(response.content)) diff --git a/netbox/core/urls.py b/netbox/core/urls.py index 3bb5cd24c..bac2eed37 100644 --- a/netbox/core/urls.py +++ b/netbox/core/urls.py @@ -25,6 +25,17 @@ urlpatterns = ( path('jobs//', views.JobView.as_view(), name='job'), path('jobs//delete/', views.JobDeleteView.as_view(), name='job_delete'), + # Background Tasks + path('background-queues/', views.BackgroundQueueListView.as_view(), name='background_queue_list'), + path('background-queues///', views.BackgroundTaskListView.as_view(), name='background_task_list'), + path('background-tasks//', views.BackgroundTaskView.as_view(), name='background_task'), + path('background-tasks//delete/', views.BackgroundTaskDeleteView.as_view(), name='background_task_delete'), + path('background-tasks//requeue/', views.BackgroundTaskRequeueView.as_view(), name='background_task_requeue'), + path('background-tasks//enqueue/', views.BackgroundTaskEnqueueView.as_view(), name='background_task_enqueue'), + path('background-tasks//stop/', views.BackgroundTaskStopView.as_view(), name='background_task_stop'), + path('background-workers//', views.WorkerListView.as_view(), name='worker_list'), + path('background-workers//', views.WorkerView.as_view(), name='worker'), + # Config revisions path('config-revisions/', views.ConfigRevisionListView.as_view(), name='configrevision_list'), path('config-revisions/add/', views.ConfigRevisionEditView.as_view(), name='configrevision_add'), diff --git a/netbox/core/views.py b/netbox/core/views.py index f81957927..5662b126e 100644 --- a/netbox/core/views.py +++ b/netbox/core/views.py @@ -3,13 +3,28 @@ from django.conf import settings from django.contrib import messages from django.contrib.auth.mixins import UserPassesTestMixin from django.core.cache import cache -from django.http import HttpResponseForbidden +from django.http import HttpResponseForbidden, Http404 from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ from django.views.generic import View +from django_rq.queues import get_queue_by_index, get_redis_connection +from django_rq.settings import QUEUES_MAP, QUEUES_LIST +from django_rq.utils import get_jobs, get_statistics, stop_jobs +from rq import requeue_job +from rq.exceptions import NoSuchJobError +from rq.job import Job as RQ_Job, JobStatus as RQJobStatus +from rq.registry import ( + DeferredJobRegistry, FailedJobRegistry, FinishedJobRegistry, ScheduledJobRegistry, StartedJobRegistry, +) +from rq.worker import Worker +from rq.worker_registration import clean_worker_registry from netbox.config import get_config, PARAMS from netbox.views import generic from netbox.views.generic.base import BaseObjectView +from netbox.views.generic.mixins import TableMixin +from utilities.forms import ConfirmationForm from utilities.utils import count_related from utilities.views import ContentTypePermissionRequiredMixin, register_model_view from . import filtersets, forms, tables @@ -237,6 +252,276 @@ class ConfigRevisionRestoreView(ContentTypePermissionRequiredMixin, View): return redirect(candidate_config.get_absolute_url()) +# +# Background Tasks (RQ) +# + +class BaseRQView(UserPassesTestMixin, View): + + def test_func(self): + return self.request.user.is_staff + + +class BackgroundQueueListView(TableMixin, BaseRQView): + table = tables.BackgroundQueueTable + + def get(self, request): + data = get_statistics(run_maintenance_tasks=True)["queues"] + table = self.get_table(data, request, bulk_actions=False) + + return render(request, 'core/rq_queue_list.html', { + 'table': table, + }) + + +class BackgroundTaskListView(TableMixin, BaseRQView): + table = tables.BackgroundTaskTable + + def get_table_data(self, request, queue, status): + jobs = [] + + # Call get_jobs() to returned queued tasks + if status == RQJobStatus.QUEUED: + return queue.get_jobs() + + # For other statuses, determine the registry to list (or raise a 404 for invalid statuses) + try: + registry_cls = { + RQJobStatus.STARTED: StartedJobRegistry, + RQJobStatus.DEFERRED: DeferredJobRegistry, + RQJobStatus.FINISHED: FinishedJobRegistry, + RQJobStatus.FAILED: FailedJobRegistry, + RQJobStatus.SCHEDULED: ScheduledJobRegistry, + }[status] + except KeyError: + raise Http404 + registry = registry_cls(queue.name, queue.connection) + + job_ids = registry.get_job_ids() + if status != RQJobStatus.DEFERRED: + jobs = get_jobs(queue, job_ids, registry) + else: + # Deferred jobs require special handling + for job_id in job_ids: + try: + jobs.append(RQ_Job.fetch(job_id, connection=queue.connection, serializer=queue.serializer)) + except NoSuchJobError: + pass + + if jobs and status == RQJobStatus.SCHEDULED: + for job in jobs: + job.scheduled_at = registry.get_scheduled_time(job) + + return jobs + + def get(self, request, queue_index, status): + queue = get_queue_by_index(queue_index) + data = self.get_table_data(request, queue, status) + table = self.get_table(data, request, False) + + # If this is an HTMX request, return only the rendered table HTML + if request.htmx: + return render(request, 'htmx/table.html', { + 'table': table, + }) + + return render(request, 'core/rq_task_list.html', { + 'table': table, + 'queue': queue, + 'status': status, + }) + + +class BackgroundTaskView(BaseRQView): + + def get(self, request, job_id): + # all the RQ queues should use the same connection + config = QUEUES_LIST[0] + try: + job = RQ_Job.fetch(job_id, connection=get_redis_connection(config['connection_config']),) + except NoSuchJobError: + raise Http404(_("Job {job_id} not found").format(job_id=job_id)) + + queue_index = QUEUES_MAP[job.origin] + queue = get_queue_by_index(queue_index) + + try: + exc_info = job._exc_info + except AttributeError: + exc_info = None + + return render(request, 'core/rq_task.html', { + 'queue': queue, + 'job': job, + 'queue_index': queue_index, + 'dependency_id': job._dependency_id, + 'exc_info': exc_info, + }) + + +class BackgroundTaskDeleteView(BaseRQView): + + def get(self, request, job_id): + if not request.htmx: + return redirect(reverse('core:background_queue_list')) + + form = ConfirmationForm(initial=request.GET) + + return render(request, 'htmx/delete_form.html', { + 'object_type': 'background task', + 'object': job_id, + 'form': form, + 'form_url': reverse('core:background_task_delete', kwargs={'job_id': job_id}) + }) + + def post(self, request, job_id): + form = ConfirmationForm(request.POST) + + if form.is_valid(): + # all the RQ queues should use the same connection + config = QUEUES_LIST[0] + try: + job = RQ_Job.fetch(job_id, connection=get_redis_connection(config['connection_config']),) + except NoSuchJobError: + raise Http404(_("Job {job_id} not found").format(job_id=job_id)) + + queue_index = QUEUES_MAP[job.origin] + queue = get_queue_by_index(queue_index) + + # Remove job id from queue and delete the actual job + queue.connection.lrem(queue.key, 0, job.id) + job.delete() + messages.success(request, f'Deleted job {job_id}') + else: + messages.error(request, f'Error deleting job: {form.errors[0]}') + + return redirect(reverse('core:background_queue_list')) + + +class BackgroundTaskRequeueView(BaseRQView): + + def get(self, request, job_id): + # all the RQ queues should use the same connection + config = QUEUES_LIST[0] + try: + job = RQ_Job.fetch(job_id, connection=get_redis_connection(config['connection_config']),) + except NoSuchJobError: + raise Http404(_("Job {job_id} not found").format(job_id=job_id)) + + queue_index = QUEUES_MAP[job.origin] + queue = get_queue_by_index(queue_index) + + requeue_job(job_id, connection=queue.connection, serializer=queue.serializer) + messages.success(request, f'You have successfully requeued: {job_id}') + return redirect(reverse('core:background_task', args=[job_id])) + + +class BackgroundTaskEnqueueView(BaseRQView): + + def get(self, request, job_id): + # all the RQ queues should use the same connection + config = QUEUES_LIST[0] + try: + job = RQ_Job.fetch(job_id, connection=get_redis_connection(config['connection_config']),) + except NoSuchJobError: + raise Http404(_("Job {job_id} not found").format(job_id=job_id)) + + queue_index = QUEUES_MAP[job.origin] + queue = get_queue_by_index(queue_index) + + try: + # _enqueue_job is new in RQ 1.14, this is used to enqueue + # job regardless of its dependencies + queue._enqueue_job(job) + except AttributeError: + queue.enqueue_job(job) + + # Remove job from correct registry if needed + if job.get_status() == RQJobStatus.DEFERRED: + registry = DeferredJobRegistry(queue.name, queue.connection) + registry.remove(job) + elif job.get_status() == RQJobStatus.FINISHED: + registry = FinishedJobRegistry(queue.name, queue.connection) + registry.remove(job) + elif job.get_status() == RQJobStatus.SCHEDULED: + registry = ScheduledJobRegistry(queue.name, queue.connection) + registry.remove(job) + + messages.success(request, f'You have successfully enqueued: {job_id}') + return redirect(reverse('core:background_task', args=[job_id])) + + +class BackgroundTaskStopView(BaseRQView): + + def get(self, request, job_id): + # all the RQ queues should use the same connection + config = QUEUES_LIST[0] + try: + job = RQ_Job.fetch(job_id, connection=get_redis_connection(config['connection_config']),) + except NoSuchJobError: + raise Http404(_("Job {job_id} not found").format(job_id=job_id)) + + queue_index = QUEUES_MAP[job.origin] + queue = get_queue_by_index(queue_index) + + stopped, _ = stop_jobs(queue, job_id) + if len(stopped) == 1: + messages.success(request, f'You have successfully stopped {job_id}') + else: + messages.error(request, f'Failed to stop {job_id}') + + return redirect(reverse('core:background_task', args=[job_id])) + + +class WorkerListView(TableMixin, BaseRQView): + table = tables.WorkerTable + + def get_table_data(self, request, queue): + clean_worker_registry(queue) + all_workers = Worker.all(queue.connection) + workers = [worker for worker in all_workers if queue.name in worker.queue_names()] + return workers + + def get(self, request, queue_index): + queue = get_queue_by_index(queue_index) + data = self.get_table_data(request, queue) + + table = self.get_table(data, request, False) + + # If this is an HTMX request, return only the rendered table HTML + if request.htmx: + if request.htmx.target != 'object_list': + table.embedded = True + # Hide selection checkboxes + if 'pk' in table.base_columns: + table.columns.hide('pk') + return render(request, 'htmx/table.html', { + 'table': table, + 'queue': queue, + }) + + return render(request, 'core/rq_worker_list.html', { + 'table': table, + 'queue': queue, + }) + + +class WorkerView(BaseRQView): + + def get(self, request, key): + # all the RQ queues should use the same connection + config = QUEUES_LIST[0] + worker = Worker.find_by_key('rq:worker:' + key, connection=get_redis_connection(config['connection_config'])) + # Convert microseconds to milliseconds + worker.total_working_time = worker.total_working_time / 1000 + + return render(request, 'core/rq_worker.html', { + 'worker': worker, + 'job': worker.get_current_job(), + 'total_working_time': worker.total_working_time * 1000, + }) + + # # Plugins # diff --git a/netbox/netbox/navigation/menu.py b/netbox/netbox/navigation/menu.py index e63947424..8ea133528 100644 --- a/netbox/netbox/navigation/menu.py +++ b/netbox/netbox/navigation/menu.py @@ -451,13 +451,18 @@ ADMIN_MENU = Menu( ), ), MenuGroup( - label=_('Plugins'), + label=_('System'), items=( MenuItem( link='core:plugin_list', link_text=_('Plugins'), staff_only=True ), + MenuItem( + link='core:background_queue_list', + link_text=_('Background Tasks'), + staff_only=True + ), ), ), ), diff --git a/netbox/netbox/urls.py b/netbox/netbox/urls.py index 7f37f01f1..38044a613 100644 --- a/netbox/netbox/urls.py +++ b/netbox/netbox/urls.py @@ -72,7 +72,6 @@ _patterns = [ path('api/plugins/', include((plugin_api_patterns, 'plugins-api'))), # Admin - path('admin/background-tasks/', include('django_rq.urls')), path('admin/', admin_site.urls), ] diff --git a/netbox/templates/admin/index.html b/netbox/templates/admin/index.html deleted file mode 100644 index b96a872b0..000000000 --- a/netbox/templates/admin/index.html +++ /dev/null @@ -1,25 +0,0 @@ -{% extends "admin/index.html" %} -{% load i18n %} - -{% block content_title %}{% endblock %} - -{% block sidebar %} - {{ block.super }} -
- - - - - - - - - - -
{% trans "System" %}
- {% trans "Background Tasks" %} -
- {% trans "Installed plugins" %} -
-
-{% endblock %} diff --git a/netbox/templates/core/rq_queue_list.html b/netbox/templates/core/rq_queue_list.html new file mode 100644 index 000000000..44203ee25 --- /dev/null +++ b/netbox/templates/core/rq_queue_list.html @@ -0,0 +1,34 @@ +{% extends 'generic/object_list.html' %} +{% load i18n %} +{% load render_table from django_tables2 %} + +{% block title %}{% trans "Background Queues" %}{% endblock %} + +{% block controls %}{% endblock %} + +{% block tabs %} + +{% endblock tabs %} + +{% block content %} +
+
+ {# Table configuration button #} +
+ +
+
+
+ +
+ {% render_table table %} +
+{% endblock content %} diff --git a/netbox/templates/core/rq_task.html b/netbox/templates/core/rq_task.html new file mode 100644 index 000000000..68829e258 --- /dev/null +++ b/netbox/templates/core/rq_task.html @@ -0,0 +1,117 @@ +{% extends 'generic/object.html' %} +{% load i18n %} +{% load buttons %} +{% load helpers %} +{% load render_table from django_tables2 %} + +{% block breadcrumbs %} + + +{% endblock breadcrumbs %} + +{% block title %}{% trans "Job" %} {{ job.id }}{% endblock %} + +{% block subtitle %} +
+ {% trans "Created" %} {{ job.created_at|annotated_date }} +
+{% endblock subtitle %} + +{% block object_identifier %}{% endblock %} + +{% block controls %} +
+ {% url 'core:background_task_delete' job_id=job.id as delete_url %} + {% include "buttons/delete.html" with url=delete_url %} + + {% if job.is_started %} + + {% trans "Stop" %} + + {% endif %} + {% if job.is_failed %} + + {% trans "Requeue" %} + + {% endif %} + {% if not job.is_queued and not job.is_failed %} + + {% trans "Enqueue" %} + + {% endif %} + +
+{% endblock controls %} + +{% block tabs %} + +{% endblock tabs %} + +{% block content %} +
+
+
+
{% trans "Job" %}
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {# TODO: Render as formatted JSON #} + + + {% if dependency_id %} + + + + + {% endif %} + {% if exc_info %} + + + + + {% endif %} +
{% trans "Queue" %}{{ job.origin|placeholder }}
{% trans "Timeout" %}{{ job.timeout|placeholder }}
{% trans "Result TTL" %}{{ job.result_ttl|placeholder }}
{% trans "Created" %}{{ job.created_at|annotated_date }}
{% trans "Queued" %}{{ job.enqueued_at|annotated_date }}
{% trans "Status" %}{{ job.get_status|placeholder }}
{% trans "Callable" %}{{ object.get_type_display|placeholder }}
{% trans "Meta" %}{{ job.meta|placeholder }}
{% trans "Arguments" %}{{ jobs.args|placeholder }}
{% trans "Keyword Arguments" %}{{ job.kwargs }}
{% trans "Depends on" %}{{ dependency_id }}
{% trans "Exception" %}
{% if job.exc_info %}{{ job.exc_info|linebreaks }}{% endif %}
+
+
+
+{% endblock content %} diff --git a/netbox/templates/core/rq_task_list.html b/netbox/templates/core/rq_task_list.html new file mode 100644 index 000000000..228b67f52 --- /dev/null +++ b/netbox/templates/core/rq_task_list.html @@ -0,0 +1,104 @@ +{% extends 'generic/object_list.html' %} +{% load buttons %} +{% load helpers %} +{% load i18n %} +{% load render_table from django_tables2 %} + +{% block page-header %} +
+
+ {# Breadcrumbs #} + +
+
+
+

{% trans 'Background Tasks' %}

+
+
+
+{% endblock page-header %} + +{% block title %}{{ status|capfirst }} {% trans "tasks in " %}{{ queue.name }}{% endblock %} + +{% block tabs %} + +{% endblock tabs %} + +{% block content %} + + {# Object list tab #} +
+ + {# Object table controls #} + {% include 'inc/table_controls_htmx.html' with table_modal="ObjectTable_config" %} + +
+ {% csrf_token %} + {# "Select all" form #} + {% if table.paginator.num_pages > 1 %} +
+
+
+
+ {% if 'bulk_edit' in actions %} + {% bulk_edit_button model query_params=request.GET %} + {% endif %} + {% if 'bulk_delete' in actions %} + {% bulk_delete_button model query_params=request.GET %} + {% endif %} +
+
+ + +
+
+
+
+ {% endif %} + +
+ {% csrf_token %} + + + {# Objects table #} +
+
+ {% include 'htmx/table.html' %} +
+
+ {# /Objects table #} + + {# Form buttons #} +
+ {% block bulk_buttons %} + {% if 'bulk_edit' in actions %} + {% bulk_edit_button model query_params=request.GET %} + {% endif %} + {% if 'bulk_delete' in actions %} + {% bulk_delete_button model query_params=request.GET %} + {% endif %} + {% endblock %} +
+ {# /Form buttons #} + +
+
+
+ {# /Object list tab #} + +{% endblock content %} diff --git a/netbox/templates/core/rq_worker.html b/netbox/templates/core/rq_worker.html new file mode 100644 index 000000000..d44f00b66 --- /dev/null +++ b/netbox/templates/core/rq_worker.html @@ -0,0 +1,82 @@ +{% extends 'generic/object.html' %} +{% load i18n %} +{% load helpers %} +{% load render_table from django_tables2 %} + +{% block breadcrumbs %} + +{% endblock breadcrumbs %} + +{% block title %}{% trans "Worker Info" %} {{ job.id }}{% endblock %} + +{% block subtitle %} +
+ {% trans "Created" %} {{ worker.birth_date|annotated_date }} +
+{% endblock subtitle %} + +{% block object_identifier %}{% endblock %} + +{% block controls %} +
+
+ {% block extra_controls %}{% endblock %} +
+
+{% endblock controls %} + +{% block tabs %} + +{% endblock tabs %} + +{% block content %} +
+
+
+
{% trans "Worker" %}
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
{% trans "Name" %}{{ worker.name|placeholder }}
{% trans "State" %}{{ worker.get_state|bettertitle|placeholder }}
{% trans "Birth" %}{{ worker.birth_date|annotated_date }}
{% trans "Queues" %}{{ worker.queue_names|join:", " }}
{% trans "PID" %}{{ worker.pid|placeholder }}
{% trans "Curent Job" %}{{ job.func_name|placeholder }}
{% trans "Successful job count" %}{{ worker.successful_job_count|placeholder }}
{% trans "Failed job count" %}{{ worker.failed_job_count }}
{% trans "Total working time" %}{{ total_working_time }} {% trans "seconds" %}
+
+
+
+{% endblock content %} diff --git a/netbox/templates/core/rq_worker_list.html b/netbox/templates/core/rq_worker_list.html new file mode 100644 index 000000000..02f49f9e9 --- /dev/null +++ b/netbox/templates/core/rq_worker_list.html @@ -0,0 +1,58 @@ +{% extends 'generic/object_list.html' %} +{% load helpers %} +{% load i18n %} +{% load render_table from django_tables2 %} + +{% block page-header %} +
+
+ {# Breadcrumbs #} + +
+
+
+

{% trans 'Background Workers' %}

+
+
+
+{% endblock page-header %} + +{% block title %}{{ status|capfirst }} {% trans "Workers in " %}{{ queue.name }}{% endblock %} + +{% block controls %}{% endblock %} + +{% block tabs %} + +{% endblock tabs %} + +{% block content %} +
+
+ {# Table configuration button #} +
+ +
+
+
+ +
+ {% render_table table %} +
+{% endblock content %} + +{% block modals %} + {% table_config_form table table_name="ObjectTable" %} +{% endblock modals %}