7848 Add RQ API (#17938)

* 7848 Add Background Tasks (RQ) to API * 7848 Tasks * 7848 cleanup * 7848 add worker support * 7848 switch to APIView * 7848 Task detail view * 7848 Task enqueue, requeue, stop * 7848 Task enqueue, requeue, stop * 7848 Task enqueue, requeue, stop * 7848 tests * 7848 tests * 7848 OpenAPI doc generation * 7848 OpenAPI doc generation * 7848 review changes * 7848 viewset * 7848 viewset * 7848 fix tests * 7848 more viewsets * 7848 fix docstring * 7848 review comments * 7848 review comments - get all tasks * 7848 queue detail view * 7848 cleanup * 7848 cleanup * 7848 cleanup * 7848 cleanup * Rename viewsets for consistency w/serializers * Misc cleanup * 7848 review changes * 7848 review changes * 7848 add test * 7848 queue detail view * 7848 fix tests * 7848 fix the spectacular test failure * 7848 fix the spectacular test failure * Misc cleanup --------- Co-authored-by: Jeremy Stretch <jstretch@netboxlabs.com>
2026-01-14 07:42:18 -06:00 · 2024-11-26 07:01:06 -08:00
parent 9b4b56febc
commit a24576f126
9 changed files with 612 additions and 99 deletions
--- a/netbox/core/api/schema.py
+++ b/netbox/core/api/schema.py
@@ -158,6 +158,9 @@ class NetBoxAutoSchema(AutoSchema):
        fields = {} if hasattr(serializer, 'child') else serializer.fields
        remove_fields = []

+        # If you get a failure here for "AttributeError: 'cached_property' object has no attribute 'items'"
+        # it is probably because you are using a viewsets.ViewSet for the API View and are defining a
+        # serializer_class. You will also need to define a get_serializer() method like for GenericAPIView.
        for child_name, child in fields.items():
            # read_only fields don't need to be in writable (write only) serializers
            if 'read_only' in dir(child) and child.read_only:
--- a/netbox/core/api/serializers.py
+++ b/netbox/core/api/serializers.py
@@ -1,3 +1,4 @@
 from .serializers_.change_logging import *
 from .serializers_.data import *
 from .serializers_.jobs import *
+from .serializers_.tasks import *
--- a/netbox/core/api/serializers_/tasks.py
+++ b/netbox/core/api/serializers_/tasks.py
@@ -0,0 +1,87 @@
+from rest_framework import serializers
+from rest_framework.reverse import reverse
+
+__all__ = (
+    'BackgroundTaskSerializer',
+    'BackgroundQueueSerializer',
+    'BackgroundWorkerSerializer',
+)
+
+
+class BackgroundTaskSerializer(serializers.Serializer):
+    id = serializers.CharField()
+    url = serializers.HyperlinkedIdentityField(
+        view_name='core-api:rqtask-detail',
+        lookup_field='id',
+        lookup_url_kwarg='pk'
+    )
+    description = serializers.CharField()
+    origin = serializers.CharField()
+    func_name = serializers.CharField()
+    args = serializers.ListField(child=serializers.CharField())
+    kwargs = serializers.DictField()
+    result = serializers.CharField()
+    timeout = serializers.IntegerField()
+    result_ttl = serializers.IntegerField()
+    created_at = serializers.DateTimeField()
+    enqueued_at = serializers.DateTimeField()
+    started_at = serializers.DateTimeField()
+    ended_at = serializers.DateTimeField()
+    worker_name = serializers.CharField()
+    position = serializers.SerializerMethodField()
+    status = serializers.SerializerMethodField()
+    meta = serializers.DictField()
+    last_heartbeat = serializers.CharField()
+
+    is_finished = serializers.BooleanField()
+    is_queued = serializers.BooleanField()
+    is_failed = serializers.BooleanField()
+    is_started = serializers.BooleanField()
+    is_deferred = serializers.BooleanField()
+    is_canceled = serializers.BooleanField()
+    is_scheduled = serializers.BooleanField()
+    is_stopped = serializers.BooleanField()
+
+    def get_position(self, obj) -> int:
+        return obj.get_position()
+
+    def get_status(self, obj) -> str:
+        return obj.get_status()
+
+
+class BackgroundQueueSerializer(serializers.Serializer):
+    name = serializers.CharField()
+    url = serializers.SerializerMethodField()
+    jobs = serializers.IntegerField()
+    oldest_job_timestamp = serializers.CharField()
+    index = serializers.IntegerField()
+    scheduler_pid = serializers.CharField()
+    workers = serializers.IntegerField()
+    finished_jobs = serializers.IntegerField()
+    started_jobs = serializers.IntegerField()
+    deferred_jobs = serializers.IntegerField()
+    failed_jobs = serializers.IntegerField()
+    scheduled_jobs = serializers.IntegerField()
+
+    def get_url(self, obj):
+        return reverse('core-api:rqqueue-detail', args=[obj['name']], request=self.context.get("request"))
+
+
+class BackgroundWorkerSerializer(serializers.Serializer):
+    name = serializers.CharField()
+    url = serializers.HyperlinkedIdentityField(
+        view_name='core-api:rqworker-detail',
+        lookup_field='name'
+    )
+    state = serializers.SerializerMethodField()
+    birth_date = serializers.DateTimeField()
+    queue_names = serializers.ListField(
+        child=serializers.CharField()
+    )
+    pid = serializers.CharField()
+    successful_job_count = serializers.IntegerField()
+    failed_job_count = serializers.IntegerField()
+    total_working_time = serializers.IntegerField()
+
+    def get_state(self, obj):
+        return obj.get_state()
--- a/netbox/core/api/urls.py
+++ b/netbox/core/api/urls.py
@@ -1,6 +1,7 @@
 from netbox.api.routers import NetBoxRouter
 from . import views

+app_name = 'core-api'

 router = NetBoxRouter()
 router.APIRootView = views.CoreRootView
@@ -9,6 +10,8 @@ router.register('data-sources', views.DataSourceViewSet)
 router.register('data-files', views.DataFileViewSet)
 router.register('jobs', views.JobViewSet)
 router.register('object-changes', views.ObjectChangeViewSet)
+router.register('background-queues', views.BackgroundQueueViewSet, basename='rqqueue')
+router.register('background-workers', views.BackgroundWorkerViewSet, basename='rqworker')
+router.register('background-tasks', views.BackgroundTaskViewSet, basename='rqtask')

-app_name = 'core-api'
 urlpatterns = router.urls
--- a/netbox/core/api/views.py
+++ b/netbox/core/api/views.py
@@ -1,5 +1,8 @@
+from django.http import Http404, HttpResponse
 from django.shortcuts import get_object_or_404
 from django.utils.translation import gettext_lazy as _
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import extend_schema
 from rest_framework.decorators import action
 from rest_framework.exceptions import PermissionDenied
 from rest_framework.response import Response
@@ -10,8 +13,17 @@ from core import filtersets
 from core.choices import DataSourceStatusChoices
 from core.jobs import SyncDataSourceJob
 from core.models import *
+from core.utils import delete_rq_job, enqueue_rq_job, get_rq_jobs, requeue_rq_job, stop_rq_job
+from django_rq.queues import get_redis_connection
+from django_rq.utils import get_statistics
+from django_rq.settings import QUEUES_LIST
 from netbox.api.metadata import ContentTypeMetadata
+from netbox.api.pagination import LimitOffsetListPagination
 from netbox.api.viewsets import NetBoxModelViewSet, NetBoxReadOnlyModelViewSet
+from rest_framework import viewsets
+from rest_framework.permissions import IsAdminUser
+from rq.job import Job as RQ_Job
+from rq.worker import Worker
 from . import serializers


@@ -71,3 +83,152 @@ class ObjectChangeViewSet(ReadOnlyModelViewSet):
    queryset = ObjectChange.objects.valid_models()
    serializer_class = serializers.ObjectChangeSerializer
    filterset_class = filtersets.ObjectChangeFilterSet
+
+
+class BaseRQViewSet(viewsets.ViewSet):
+    """
+    Base class for RQ view sets. Provides a list() method. Subclasses must implement get_data().
+    """
+    permission_classes = [IsAdminUser]
+    serializer_class = None
+
+    def get_data(self):
+        raise NotImplementedError()
+
+    @extend_schema(responses={200: OpenApiTypes.OBJECT})
+    def list(self, request):
+        data = self.get_data()
+        paginator = LimitOffsetListPagination()
+        data = paginator.paginate_list(data, request)
+
+        serializer = self.serializer_class(data, many=True, context={'request': request})
+        return paginator.get_paginated_response(serializer.data)
+
+    def get_serializer(self, *args, **kwargs):
+        """
+        Return the serializer instance that should be used for validating and
+        deserializing input, and for serializing output.
+        """
+        serializer_class = self.get_serializer_class()
+        kwargs['context'] = self.get_serializer_context()
+        return serializer_class(*args, **kwargs)
+
+
+class BackgroundQueueViewSet(BaseRQViewSet):
+    """
+    Retrieve a list of RQ Queues.
+    Note: Queue names are not URL safe so not returning a detail view.
+    """
+    serializer_class = serializers.BackgroundQueueSerializer
+    lookup_field = 'name'
+    lookup_value_regex = r'[\w.@+-]+'
+
+    def get_view_name(self):
+        return "Background Queues"
+
+    def get_data(self):
+        return get_statistics(run_maintenance_tasks=True)["queues"]
+
+    @extend_schema(responses={200: OpenApiTypes.OBJECT})
+    def retrieve(self, request, name):
+        data = self.get_data()
+        if not data:
+            raise Http404
+
+        for queue in data:
+            if queue['name'] == name:
+                serializer = self.serializer_class(queue, context={'request': request})
+                return Response(serializer.data)
+
+        raise Http404
+
+
+class BackgroundWorkerViewSet(BaseRQViewSet):
+    """
+    Retrieve a list of RQ Workers.
+    """
+    serializer_class = serializers.BackgroundWorkerSerializer
+    lookup_field = 'name'
+
+    def get_view_name(self):
+        return "Background Workers"
+
+    def get_data(self):
+        config = QUEUES_LIST[0]
+        return Worker.all(get_redis_connection(config['connection_config']))
+
+    def retrieve(self, request, name):
+        # all the RQ queues should use the same connection
+        config = QUEUES_LIST[0]
+        workers = Worker.all(get_redis_connection(config['connection_config']))
+        worker = next((item for item in workers if item.name == name), None)
+        if not worker:
+            raise Http404
+
+        serializer = serializers.BackgroundWorkerSerializer(worker, context={'request': request})
+        return Response(serializer.data)
+
+
+class BackgroundTaskViewSet(BaseRQViewSet):
+    """
+    Retrieve a list of RQ Tasks.
+    """
+    serializer_class = serializers.BackgroundTaskSerializer
+
+    def get_view_name(self):
+        return "Background Tasks"
+
+    def get_data(self):
+        return get_rq_jobs()
+
+    def get_task_from_id(self, task_id):
+        config = QUEUES_LIST[0]
+        task = RQ_Job.fetch(task_id, connection=get_redis_connection(config['connection_config']))
+        if not task:
+            raise Http404
+
+        return task
+
+    @extend_schema(responses={200: OpenApiTypes.OBJECT})
+    def retrieve(self, request, pk):
+        """
+        Retrieve the details of the specified RQ Task.
+        """
+        task = self.get_task_from_id(pk)
+        serializer = self.serializer_class(task, context={'request': request})
+        return Response(serializer.data)
+
+    @action(methods=["POST"], detail=True)
+    def delete(self, request, pk):
+        """
+        Delete the specified RQ Task.
+        """
+        delete_rq_job(pk)
+        return HttpResponse(status=200)
+
+    @action(methods=["POST"], detail=True)
+    def requeue(self, request, pk):
+        """
+        Requeues the specified RQ Task.
+        """
+        requeue_rq_job(pk)
+        return HttpResponse(status=200)
+
+    @action(methods=["POST"], detail=True)
+    def enqueue(self, request, pk):
+        """
+        Enqueues the specified RQ Task.
+        """
+        enqueue_rq_job(pk)
+        return HttpResponse(status=200)
+
+    @action(methods=["POST"], detail=True)
+    def stop(self, request, pk):
+        """
+        Stops the specified RQ Task.
+        """
+        stopped_jobs = stop_rq_job(pk)
+        if len(stopped_jobs) == 1:
+            return HttpResponse(status=200)
+        else:
+            return HttpResponse(status=204)