Closes #9599: Add cursor pagination mode

docs/integrations/rest-api.md

@@ -273,7 +273,11 @@ When retrieving devices and virtual machines via the REST API, each will include
 
 ## Pagination
 
-API responses which contain a list of many objects will be paginated for efficiency. The root JSON object returned by a list endpoint contains the following attributes:
+API responses which contain a list of many objects will be paginated for efficiency. NetBox supports two pagination modes: Limit Offset Pagination (default) and Cursor Pagination. The two modes can be toggled with the `pagination_mode` parameter.
+
+### Limit Offset Pagination
+
+The root JSON object returned by a list endpoint contains the following attributes:
 
 * `count`: The total number of all objects matching the query
 * `next`: A hyperlink to the next page of results (if applicable)
@@ -325,6 +329,23 @@ The response will return devices 1 through 100. The URL provided in the `next` a
 }
 ```
 
+### Cursor Pagination
+
+Cursor Pagination presents a cursor indicator and it allows the client to retrieve the next or the previous page with respect to the cursor. The returned list is sorted in reverse order of the "created" field. In this mode, the client can get a consistent view of the items even when items are being deleted by other clients during the process. More information about proper use of cursor pagination can be found in the [Djaongo REST framework documentation](https://www.django-rest-framework.org/api-guide/pagination/#cursorpagination).
+
+Here is an example of a cursor pagination request:
+```
+GET http://netbox/api/dcim/devices/?pagination_mode=cursor
+```
+The JSON object returned contains `next`, `previous` and `results`. Please refer to **Limit Offset Pagination** for meanings of these fields.
+
+Similarly, the default page size is determined by the [`PAGINATE_COUNT`](../configuration/dynamic-settings.md#paginate_count) configuration parameter. To retrieve a hundred devices at a time, you would make a request for:
+```
+GET http://netbox/api/dcim/devices/?pagination_mode=cursor&limit=100
+```
+
+### Maximum Page Size
+
 The maximum number of objects that can be returned is limited by the [`MAX_PAGE_SIZE`](../configuration/miscellaneous.md#max_page_size) configuration parameter, which is 1000 by default. Setting this to `0` or `None` will remove the maximum limit. An API consumer can then pass `?limit=0` to retrieve _all_ matching objects with a single request.
 
 !!! warning

netbox/dcim/api/views.py

@@ -1,3 +1,4 @@
+from functools import partial
 import socket
 
 from django.http import Http404, HttpResponse, HttpResponseForbidden
@@ -20,7 +21,7 @@ from ipam.models import Prefix, VLAN
 from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired
 from netbox.api.exceptions import ServiceUnavailable
 from netbox.api.metadata import ContentTypeMetadata
-from netbox.api.pagination import StripCountAnnotationsPaginator
+from netbox.api.pagination import StripCountAnnotationsPaginator, TwoModePagination
 from netbox.api.viewsets import NetBoxModelViewSet
 from netbox.config import get_config
 from netbox.constants import NESTED_SERIALIZER_PREFIX
@@ -397,7 +398,7 @@ class DeviceViewSet(ConfigContextQuerySetMixin, NetBoxModelViewSet):
         'virtual_chassis__master', 'primary_ip4__nat_outside', 'primary_ip6__nat_outside', 'tags',
     )
     filterset_class = filtersets.DeviceFilterSet
-    pagination_class = StripCountAnnotationsPaginator
+    pagination_class = partial(TwoModePagination, StripCountAnnotationsPaginator)
 
     def get_serializer_class(self):
         """

netbox/netbox/api/pagination.py

@@ -1,7 +1,11 @@
 from django.db.models import QuerySet
-from rest_framework.pagination import LimitOffsetPagination
+from django.utils.encoding import force_str
+from django.utils.translation import gettext_lazy as _
+from rest_framework.pagination import LimitOffsetPagination, CursorPagination, BasePagination, _reverse_ordering
+from rest_framework.response import Response
 
 from netbox.config import get_config
+from rest_framework.compat import coreapi, coreschema
 
 
 class OptionalLimitOffsetPagination(LimitOffsetPagination):
@@ -83,3 +87,167 @@ class StripCountAnnotationsPaginator(OptionalLimitOffsetPagination):
         cloned_queryset.query.annotations.clear()
 
         return cloned_queryset.count()
+
+
+class CursorPaginationWithNoLimit(CursorPagination):
+    """
+    Allow setting limit=0 to disable pagination for a request. The limit can only be disabled if
+    MAX_PAGE_SIZE has been set to 0 or None.
+    """
+    page_size_query_param = 'limit'
+
+    def __init__(self) -> None:
+        self.default_page_size = get_config().PAGINATE_COUNT
+
+    def paginate_queryset(self, queryset, request, view=None):
+        """
+        Reuses the implementation from CursorPagination with minor modification to handle limit=0.
+        """
+        self.page_size = self.get_page_size(request)
+
+        self.base_url = request.build_absolute_uri()
+        self.ordering = self.get_ordering(request, queryset, view)
+
+        self.cursor = self.decode_cursor(request)
+        if self.cursor is None:
+            (offset, reverse, current_position) = (0, False, None)
+        else:
+            (offset, reverse, current_position) = self.cursor
+
+        # Cursor pagination always enforces an ordering.
+        if reverse:
+            queryset = queryset.order_by(*_reverse_ordering(self.ordering))
+        else:
+            queryset = queryset.order_by(*self.ordering)
+
+        # If we have a cursor with a fixed position then filter by that.
+        if current_position is not None:
+            order = self.ordering[0]
+            is_reversed = order.startswith('-')
+            order_attr = order.lstrip('-')
+
+            # Test for: (cursor reversed) XOR (queryset reversed)
+            if self.cursor.reverse != is_reversed:
+                kwargs = {order_attr + '__lt': current_position}
+            else:
+                kwargs = {order_attr + '__gt': current_position}
+
+            queryset = queryset.filter(**kwargs)
+
+        if self.page_size:
+            # If we have an offset cursor then offset the entire page by that amount.
+            # We also always fetch an extra item in order to determine if there is a
+            # page following on from this one.
+            results = list(queryset[offset:offset + self.page_size + 1])
+            self.page = list(results[:self.page_size])
+        else:
+            self.page = results = list(queryset[offset:])
+
+        # Determine the position of the final item following the page.
+        if len(results) > len(self.page):
+            has_following_position = True
+            following_position = self._get_position_from_instance(results[-1], self.ordering)
+        else:
+            has_following_position = False
+            following_position = None
+
+        if reverse:
+            # If we have a reverse queryset, then the query ordering was in reverse
+            # so we need to reverse the items again before returning them to the user.
+            self.page = list(reversed(self.page))
+
+            # Determine next and previous positions for reverse cursors.
+            self.has_next = (current_position is not None) or (offset > 0)
+            self.has_previous = has_following_position
+            if self.has_next:
+                self.next_position = current_position
+            if self.has_previous:
+                self.previous_position = following_position
+        else:
+            # Determine next and previous positions for forward cursors.
+            self.has_next = has_following_position
+            self.has_previous = (current_position is not None) or (offset > 0)
+            if self.has_next:
+                self.next_position = following_position
+            if self.has_previous:
+                self.previous_position = current_position
+
+        # Display page controls in the browsable API if there is more
+        # than one page.
+        if (self.has_previous or self.has_next) and self.template is not None:
+            self.display_page_controls = True
+
+        return self.page
+
+    def get_page_size(self, request):
+        if self.page_size_query_param:
+            try:
+                page_size = int(request.query_params[self.page_size_query_param])
+                if page_size < 0:
+                    raise ValueError()
+                # Enforce maximum page size, if defined
+                MAX_PAGE_SIZE = get_config().MAX_PAGE_SIZE
+                if MAX_PAGE_SIZE:
+                    return MAX_PAGE_SIZE if page_size == 0 else min(page_size, MAX_PAGE_SIZE)
+                return page_size
+            except (KeyError, ValueError):
+                pass
+
+        return self.default_page_size
+
+
+class TwoModePagination(BasePagination):
+    """
+    Pagination that allows user to toggle between LimitOffsetPagination and CursorPagination. The default
+    is LimitOffsetPagination.
+    """
+    pagination_mode_param = 'pagination_mode'
+    pagination_mode_description = _(
+        'Mode selector for LimitOffsetPagination (default) and CursorPagination.\n'
+        '`offset` and `limit` are used for LimitOffsetPagination.\n'
+        '`cursor` and `limit` are used for CursorPagination.')
+    limit_offset_key = 'limit_offset'
+    cursor_pagination_key = 'cursor'
+
+    def __init__(
+        self,
+        limit_offset_paginator=OptionalLimitOffsetPagination,
+        cursor_paginator=CursorPaginationWithNoLimit,
+    ) -> None:
+        self._limit_offset_pagination = limit_offset_paginator()
+        self._cursor_pagination = cursor_paginator()
+
+        self._chosen_pagination = self._limit_offset_pagination
+
+    def paginate_queryset(self, queryset, request: Response, view=None):
+        mode = request.query_params.get(self.pagination_mode_param)
+        if mode == self.cursor_pagination_key:
+            self._chosen_pagination = self._cursor_pagination
+        return self._chosen_pagination.paginate_queryset(queryset, request, view)
+
+    def get_paginated_response(self, data):
+        return self._chosen_pagination.get_paginated_response(data)
+
+    def get_paginated_response_schema(self, schema):
+        return self._chosen_pagination.get_paginated_response_schema(schema)
+
+    def to_html(self):  # pragma: no cover
+        return self._chosen_pagination.to_html()
+
+    def get_schema_fields(self, view):
+        mode_field = coreapi.Field(
+            name=self.pagination_mode_param,
+            required=False,
+            location='query',
+            schema=coreschema.Enum(
+                title='Pagination Mode',
+                description=force_str(self.pagination_mode_description),
+                enum=[self.limit_offset_key, self.cursor_pagination_key],
+            )
+        )
+        return [mode_field] \
+            + self._limit_offset_pagination.get_schema_fields(view) \
+            + self._cursor_pagination.get_schema_fields(view)[:1]  # "limit" is shared between the two modes
+
+    def get_schema_operation_parameters(self, view):
+        return self._chosen_pagination.get_schema_operation_parameters(view)

netbox/netbox/settings.py

@@ -532,7 +532,7 @@ REST_FRAMEWORK = {
         'rest_framework.filters.OrderingFilter',
     ),
     'DEFAULT_METADATA_CLASS': 'netbox.api.metadata.BulkOperationMetadata',
-    'DEFAULT_PAGINATION_CLASS': 'netbox.api.pagination.OptionalLimitOffsetPagination',
+    'DEFAULT_PAGINATION_CLASS': 'netbox.api.pagination.TwoModePagination',
     'DEFAULT_PERMISSION_CLASSES': (
         'netbox.api.authentication.TokenPermissions',
     ),

netbox/utilities/tests/test_api.py

@@ -1,3 +1,4 @@
+from typing import Final
 import urllib.parse
 
 from django.contrib.contenttypes.models import ContentType
@@ -123,7 +124,7 @@ class WritableNestedSerializerTest(APITestCase):
         self.assertEqual(VLAN.objects.count(), 0)
 
 
-class APIPaginationTestCase(APITestCase):
+class APILimitOffsetPaginationTestCase(APITestCase):
     user_permissions = ('dcim.view_site',)
 
     @classmethod
@@ -234,6 +235,107 @@ class APIOrderingTestCase(APITestCase):
         )
 
 
+class APICursorPaginationTestCase(APITestCase):
+    user_permissions = ('dcim.view_site',)
+
+    @classmethod
+    def setUpTestData(cls):
+        cls.url = reverse('dcim-api:site-list') + "?pagination_mode=cursor"
+        cls.initial_record_count = 100
+
+        # Create a large number of Sites for testing
+        Site.objects.bulk_create([
+            Site(name=f'Site {i}', slug=f'site-{i}') for i in range(1, 1 + cls.initial_record_count)
+        ])
+
+    def test_default_page_size(self):
+        response = self.client.get(self.url, format='json', **self.header)
+        page_size = get_config().PAGINATE_COUNT
+        self.assertLess(page_size, self.initial_record_count, "Default page size not sufficient for data set")
+
+        self.assertHttpStatus(response, status.HTTP_200_OK)
+        self.assertIn('cursor=', response.data['next'])
+        self.assertIn('pagination_mode=cursor', response.data['next'])
+        self.assertIsNone(response.data['previous'])
+        self.assertEqual(len(response.data['results']), page_size)
+
+    def test_custom_page_size(self):
+        page_size: Final[int] = 10
+        response = self.client.get(f'{self.url}&limit={page_size}', format='json', **self.header)
+
+        self.assertHttpStatus(response, status.HTTP_200_OK)
+        self.assertIn('cursor=', response.data['next'])
+        self.assertIn('pagination_mode=cursor', response.data['next'])
+        self.assertIsNone(response.data['previous'])
+        self.assertEqual(len(response.data['results']), page_size)
+
+    @override_settings(MAX_PAGE_SIZE=20)
+    def test_max_page_size(self):
+        response = self.client.get(f'{self.url}&limit=0', format='json', **self.header)
+
+        self.assertHttpStatus(response, status.HTTP_200_OK)
+        self.assertIn('cursor=', response.data['next'])
+        self.assertIn('pagination_mode=cursor', response.data['next'])
+        self.assertIsNone(response.data['previous'])
+        self.assertEqual(len(response.data['results']), 20)
+
+    @override_settings(MAX_PAGE_SIZE=0)
+    def test_max_page_size_disabled(self):
+        response = self.client.get(f'{self.url}&limit=0', format='json', **self.header)
+
+        self.assertHttpStatus(response, status.HTTP_200_OK)
+        self.assertIsNone(response.data['next'])
+        self.assertIsNone(response.data['previous'])
+        self.assertEqual(len(response.data['results']), self.initial_record_count)
+
+    def test_next_and_previous(self):
+        page_size: Final[int] = 10
+        page_1_response = self.client.get(f'{self.url}&limit={page_size}', format='json', **self.header)
+        page_2_response = self.client.get(page_1_response.data['next'], format='json', **self.header)
+        prev_response = self.client.get(page_2_response.data['previous'], format='json', **self.header)
+
+        self.assertHttpStatus(page_2_response, status.HTTP_200_OK)
+        self.assertEqual(len(page_2_response.data['results']), page_size)
+        self.assertIsNotNone(page_2_response.data['next'])
+        self.assertListEqual(page_1_response.data['results'], prev_response.data['results'])
+
+    def test_invalid_cursor(self):
+        response = self.client.get(f'{self.url}&cursor=invalid', format='json', **self.header)
+
+        self.assertHttpStatus(response, status.HTTP_404_NOT_FOUND)
+        self.assertEqual(response.data['detail'], 'Invalid cursor')
+
+    def test_ignore_invalid_page_size(self):
+        response = self.client.get(f'{self.url}&limit=-1', format='json', **self.header)
+
+        self.assertHttpStatus(response, status.HTTP_200_OK)
+        self.assertEqual(len(response.data['results']), get_config().PAGINATE_COUNT)
+
+    @override_settings(MAX_PAGE_SIZE=55)
+    def test_delete_in_between(self):
+        """
+        CursorPagination should get all 101 objects even if one of the objects is deleted from page 1 during
+        the process.
+        """
+        # create an extra record
+        site_to_delete = Site.objects.create(name=f'Site - to delete', slug=f'site-to-delete')
+        try:
+            page_1_response = self.client.get(f'{self.url}&limit=55', format='json', **self.header)
+            site_to_delete.delete()
+        except Exception as e:
+            site_to_delete.delete()
+            raise e
+
+        page_2_response = self.client.get(page_1_response.data['next'], format='json', **self.header)
+
+        self.assertHttpStatus(page_2_response, status.HTTP_200_OK)
+        self.assertEqual(
+            len(page_1_response.data['results']) + len(page_2_response.data['results']),
+            self.initial_record_count + 1,
+        )
+        self.assertIsNone(page_2_response.data['next'])
+
+
 class APIDocsTestCase(TestCase):
 
     def setUp(self):