Merge pull request #6717 from netbox-community/6713-release-checking

Closes #6713: Move release checking to the housekeeping routine

docs/configuration/optional-settings.md

@@ -478,19 +478,11 @@ When remote user authentication is in use, this is the name of the HTTP header w
 
 ---
 
-## RELEASE_CHECK_TIMEOUT
-
-Default: 86,400 (24 hours)
-
-The number of seconds to retain the latest version that is fetched from the GitHub API before automatically invalidating it and fetching it from the API again. This must be set to at least one hour (3600 seconds).
-
----
-
 ## RELEASE_CHECK_URL
 
 Default: None (disabled)
 
-This parameter defines the URL of the repository that will be checked periodically for new NetBox releases. When a new release is detected, a message will be displayed to administrative users on the home page. This can be set to the official repository (`'https://api.github.com/repos/netbox-community/netbox/releases'`) or a custom fork. Set this to `None` to disable automatic update checks.
+This parameter defines the URL of the repository that will be checked for new NetBox releases. When a new release is detected, a message will be displayed to administrative users on the home page. This can be set to the official repository (`'https://api.github.com/repos/netbox-community/netbox/releases'`) or a custom fork. Set this to `None` to disable automatic update checks.
 
 !!! note
     The URL provided **must** be compatible with the [GitHub REST API](https://docs.github.com/en/rest).

docs/release-notes/version-3.0.md

@@ -68,10 +68,12 @@ CustomValidator can also be subclassed to enforce more complex logic by overridi
 * [#6068](https://github.com/netbox-community/netbox/issues/6068) - Drop support for legacy static CSV export
 * [#6338](https://github.com/netbox-community/netbox/issues/6338) - Decimal fields are no longer coerced to strings in REST API
 * [#6639](https://github.com/netbox-community/netbox/issues/6639) - Drop support for queryset caching (django-cacheops)
+* [#6713](https://github.com/netbox-community/netbox/issues/6713) - Checking for new releases is now done as part of the housekeeping routine
 
 ### Configuration Changes
 
 * The `CACHE_TIMEOUT` configuration parameter has been removed.
+* The `RELEASE_CHECK_TIMEOUT` configuration parameter has been removed.
 
 ### REST API Changes
 

netbox/extras/management/commands/housekeeping.py

@@ -1,10 +1,13 @@
 from datetime import timedelta
 from importlib import import_module
 
+import requests
 from django.conf import settings
+from django.core.cache import cache
 from django.core.management.base import BaseCommand
 from django.db import DEFAULT_DB_ALIAS
 from django.utils import timezone
+from packaging import version
 
 from extras.models import ObjectChange
 
@@ -48,4 +51,37 @@ class Command(BaseCommand):
                 f"\tSkipping: No retention period specified (CHANGELOG_RETENTION = {settings.CHANGELOG_RETENTION})"
             )
 
+        # Check for new releases (if enabled)
+        self.stdout.write("[*] Checking for latest release")
+        if settings.RELEASE_CHECK_URL:
+            headers = {
+                'Accept': 'application/vnd.github.v3+json',
+            }
+
+            try:
+                self.stdout.write(f"\tFetching {settings.RELEASE_CHECK_URL}")
+                response = requests.get(
+                    url=settings.RELEASE_CHECK_URL,
+                    headers=headers,
+                    proxies=settings.HTTP_PROXIES
+                )
+                response.raise_for_status()
+
+                releases = []
+                for release in response.json():
+                    if 'tag_name' not in release or release.get('devrelease') or release.get('prerelease'):
+                        continue
+                    releases.append((version.parse(release['tag_name']), release.get('html_url')))
+                latest_release = max(releases)
+                self.stdout.write(f"\tFound {len(response.json())} releases; {len(releases)} usable")
+                self.stdout.write(f"\tLatest release: {latest_release[0]}")
+
+                # Cache the most recent release
+                cache.set('latest_release', latest_release, None)
+
+            except requests.exceptions.RequestException as exc:
+                self.stdout.write(f"\tRequest error: {exc}")
+        else:
+            self.stdout.write(f"\tSkipping: RELEASE_CHECK_URL not set")
+
         self.stdout.write("Finished.", self.style.SUCCESS)

netbox/netbox/configuration.example.py

@@ -241,9 +241,6 @@ REMOTE_AUTH_AUTO_CREATE_USER = True
 REMOTE_AUTH_DEFAULT_GROUPS = []
 REMOTE_AUTH_DEFAULT_PERMISSIONS = {}
 
-# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour.
-RELEASE_CHECK_TIMEOUT = 24 * 3600
-
 # This repository is used to check whether there is a new release of NetBox available. Set to None to disable the
 # version check or use the URL below to check for release in the official NetBox repository.
 RELEASE_CHECK_URL = None

netbox/netbox/releases.py

@@ -1,32 +0,0 @@
-import logging
-
-from django.conf import settings
-from django.core.cache import cache
-from django_rq import get_queue
-
-from utilities.background_tasks import get_releases
-
-logger = logging.getLogger('netbox.releases')
-
-
-def get_latest_release(pre_releases=False):
-    if settings.RELEASE_CHECK_URL:
-        logger.debug("Checking for most recent release")
-        latest_release = cache.get('latest_release')
-        if latest_release:
-            logger.debug(f"Found cached release: {latest_release}")
-            return latest_release
-        else:
-            # Check for an existing job. This can happen if the RQ worker process is not running.
-            queue = get_queue('check_releases')
-            if queue.jobs:
-                logger.warning("Job to check for new releases is already queued; skipping")
-            else:
-                # Get the releases in the background worker, it will fill the cache
-                logger.info("Initiating background task to retrieve updated releases list")
-                get_releases.delay(pre_releases=pre_releases)
-
-    else:
-        logger.debug("Skipping release check; RELEASE_CHECK_URL not defined")
-
-    return 'unknown', None

netbox/netbox/settings.py

@@ -47,7 +47,13 @@ except ModuleNotFoundError as e:
 
 # Warn on removed config parameters
 if hasattr(configuration, 'CACHE_TIMEOUT'):
-    warnings.warn("The CACHE_TIMEOUT configuration parameter was removed in v3.0.0 and no longer has any effect.")
+    warnings.warn(
+        "The CACHE_TIMEOUT configuration parameter was removed in v3.0.0 and no longer has any effect."
+    )
+if hasattr(configuration, 'RELEASE_CHECK_TIMEOUT'):
+    warnings.warn(
+        "The RELEASE_CHECK_TIMEOUT configuration parameter was removed in v3.0.0 and no longer has any effect."
+    )
 
 # Enforce required configuration parameters
 for parameter in ['ALLOWED_HOSTS', 'DATABASE', 'SECRET_KEY', 'REDIS']:
@@ -114,7 +120,6 @@ REMOTE_AUTH_DEFAULT_PERMISSIONS = getattr(configuration, 'REMOTE_AUTH_DEFAULT_PE
 REMOTE_AUTH_ENABLED = getattr(configuration, 'REMOTE_AUTH_ENABLED', False)
 REMOTE_AUTH_HEADER = getattr(configuration, 'REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER')
 RELEASE_CHECK_URL = getattr(configuration, 'RELEASE_CHECK_URL', None)
-RELEASE_CHECK_TIMEOUT = getattr(configuration, 'RELEASE_CHECK_TIMEOUT', 24 * 3600)
 REPORTS_ROOT = getattr(configuration, 'REPORTS_ROOT', os.path.join(BASE_DIR, 'reports')).rstrip('/')
 RQ_DEFAULT_TIMEOUT = getattr(configuration, 'RQ_DEFAULT_TIMEOUT', 300)
 SCRIPTS_ROOT = getattr(configuration, 'SCRIPTS_ROOT', os.path.join(BASE_DIR, 'scripts')).rstrip('/')
@@ -141,10 +146,6 @@ if RELEASE_CHECK_URL:
     except ValidationError as err:
         raise ImproperlyConfigured(str(err))
 
-# Enforce a minimum cache timeout for update checks
-if RELEASE_CHECK_TIMEOUT < 3600:
-    raise ImproperlyConfigured("RELEASE_CHECK_TIMEOUT has to be at least 3600 seconds (1 hour)")
-
 
 #
 # Database
@@ -545,8 +546,7 @@ else:
     }
 
 RQ_QUEUES = {
-    'default': RQ_PARAMS,  # Webhooks
+    'default': RQ_PARAMS,
-    'check_releases': RQ_PARAMS,
 }
 
 

netbox/netbox/tests/test_releases.py

@@ -1,138 +0,0 @@
-from io import BytesIO
-from logging import ERROR
-from unittest.mock import Mock, patch
-
-import requests
-from django.conf import settings
-from django.core.cache import cache
-from django.test import SimpleTestCase, override_settings
-from packaging.version import Version
-from requests import Response
-
-from utilities.background_tasks import get_releases
-
-
-def successful_github_response(url, *_args, **_kwargs):
-    r = Response()
-    r.url = url
-    r.status_code = 200
-    r.reason = 'OK'
-    r.headers = {
-        'Content-Type': 'application/json; charset=utf-8',
-    }
-    r.raw = BytesIO(b'''[
-        {
-            "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.7.8",
-            "tag_name": "v2.7.8",
-            "prerelease": false
-        },
-        {
-            "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.6-beta1",
-            "tag_name": "v2.6-beta1",
-            "prerelease": true
-        },
-        {
-            "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.5.9",
-            "tag_name": "v2.5.9",
-            "prerelease": false
-        }
-    ]
-    ''')
-    return r
-
-
-def unsuccessful_github_response(url, *_args, **_kwargs):
-    r = Response()
-    r.url = url
-    r.status_code = 404
-    r.reason = 'Not Found'
-    r.headers = {
-        'Content-Type': 'application/json; charset=utf-8',
-    }
-    r.raw = BytesIO(b'''{
-        "message": "Not Found",
-        "documentation_url": "https://developer.github.com/v3/repos/releases/#list-releases-for-a-repository"
-    }
-    ''')
-    return r
-
-
-@override_settings(RELEASE_CHECK_URL='https://localhost/unittest/releases', RELEASE_CHECK_TIMEOUT=160876)
-class GetReleasesTestCase(SimpleTestCase):
-    @patch.object(requests, 'get')
-    @patch.object(cache, 'set')
-    def test_pre_releases(self, dummy_cache_set: Mock, dummy_request_get: Mock):
-        dummy_request_get.side_effect = successful_github_response
-
-        releases = get_releases(pre_releases=True)
-
-        # Check result
-        self.assertListEqual(releases, [
-            (Version('2.7.8'), 'https://github.com/netbox-community/netbox/releases/tag/v2.7.8'),
-            (Version('2.6b1'), 'https://github.com/netbox-community/netbox/releases/tag/v2.6-beta1'),
-            (Version('2.5.9'), 'https://github.com/netbox-community/netbox/releases/tag/v2.5.9')
-        ])
-
-        # Check if correct request is made
-        dummy_request_get.assert_called_once_with(
-            'https://localhost/unittest/releases',
-            headers={'Accept': 'application/vnd.github.v3+json'},
-            proxies=settings.HTTP_PROXIES
-        )
-
-        # Check if result is put in cache
-        dummy_cache_set.assert_called_once_with(
-            'latest_release',
-            max(releases),
-            160876
-        )
-
-    @patch.object(requests, 'get')
-    @patch.object(cache, 'set')
-    def test_no_pre_releases(self, dummy_cache_set: Mock, dummy_request_get: Mock):
-        dummy_request_get.side_effect = successful_github_response
-
-        releases = get_releases(pre_releases=False)
-
-        # Check result
-        self.assertListEqual(releases, [
-            (Version('2.7.8'), 'https://github.com/netbox-community/netbox/releases/tag/v2.7.8'),
-            (Version('2.5.9'), 'https://github.com/netbox-community/netbox/releases/tag/v2.5.9')
-        ])
-
-        # Check if correct request is made
-        dummy_request_get.assert_called_once_with(
-            'https://localhost/unittest/releases',
-            headers={'Accept': 'application/vnd.github.v3+json'},
-            proxies=settings.HTTP_PROXIES
-        )
-
-        # Check if result is put in cache
-        dummy_cache_set.assert_called_once_with(
-            'latest_release',
-            max(releases),
-            160876
-        )
-
-    @patch.object(requests, 'get')
-    def test_failed_request(self, dummy_request_get: Mock):
-        dummy_request_get.side_effect = unsuccessful_github_response
-
-        with self.assertLogs(level=ERROR) as cm:
-            releases = get_releases()
-
-        # Check log entry
-        self.assertEqual(len(cm.output), 1)
-        log_output = cm.output[0]
-        last_log_line = log_output.split('\n')[-1]
-        self.assertRegex(last_log_line, '404 .* Not Found')
-
-        # Check result
-        self.assertListEqual(releases, [])
-
-        # Check if correct request is made
-        dummy_request_get.assert_called_once_with(
-            'https://localhost/unittest/releases',
-            headers={'Accept': 'application/vnd.github.v3+json'},
-            proxies=settings.HTTP_PROXIES
-        )

netbox/netbox/views/__init__.py

@@ -3,6 +3,7 @@ import sys
 
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
+from django.core.cache import cache
 from django.db.models import F
 from django.http import HttpResponseServerError
 from django.shortcuts import redirect, render
@@ -23,7 +24,6 @@ from extras.models import ObjectChange, JobResult
 from ipam.models import Aggregate, IPAddress, Prefix, VLAN, VRF
 from netbox.constants import SEARCH_MAX_RESULTS, SEARCH_TYPES
 from netbox.forms import SearchForm
-from netbox.releases import get_latest_release
 from tenancy.models import Tenant
 from virtualization.models import Cluster, VirtualMachine
 
@@ -119,10 +119,10 @@ class HomeView(View):
         # Check whether a new release is available. (Only for staff/superusers.)
         new_release = None
         if request.user.is_staff or request.user.is_superuser:
-            latest_release, release_url = get_latest_release()
+            latest_release = cache.get('latest_release')
-            if isinstance(latest_release, version.Version):
+            if latest_release:
-                current_version = version.parse(settings.VERSION)
+                release_version, release_url = latest_release
-                if latest_release > current_version:
+                if release_version > version.parse(settings.VERSION):
                     new_release = {
                         'version': str(latest_release),
                         'url': release_url,

netbox/utilities/background_tasks.py

@@ -1,42 +0,0 @@
-import logging
-
-import requests
-from django.conf import settings
-from django.core.cache import cache
-from django_rq import job
-from packaging import version
-
-# Get an instance of a logger
-logger = logging.getLogger('netbox.releases')
-
-
-@job('check_releases')
-def get_releases(pre_releases=False):
-    url = settings.RELEASE_CHECK_URL
-    headers = {
-        'Accept': 'application/vnd.github.v3+json',
-    }
-    releases = []
-
-    try:
-        logger.info(f"Fetching new releases from {url}")
-        response = requests.get(url, headers=headers, proxies=settings.HTTP_PROXIES)
-        response.raise_for_status()
-        total_releases = len(response.json())
-
-        for release in response.json():
-            if 'tag_name' not in release:
-                continue
-            if not pre_releases and (release.get('devrelease') or release.get('prerelease')):
-                continue
-            releases.append((version.parse(release['tag_name']), release.get('html_url')))
-        logger.debug(f"Found {total_releases} releases; {len(releases)} usable")
-
-    except requests.exceptions.RequestException as exc:
-        logger.exception(f"Error while fetching latest release from {url}: {exc}")
-        return []
-
-    # Cache the most recent release
-    cache.set('latest_release', max(releases), settings.RELEASE_CHECK_TIMEOUT)
-
-    return releases