From 8850add0a867621cbcd2b635c02ea4510615f117 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 7 Jul 2021 21:37:35 -0400 Subject: [PATCH 1/5] Add release check to the housekeeping task --- .../management/commands/housekeeping.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/netbox/extras/management/commands/housekeeping.py b/netbox/extras/management/commands/housekeeping.py index 53dd74b96..4dbfa6725 100644 --- a/netbox/extras/management/commands/housekeeping.py +++ b/netbox/extras/management/commands/housekeeping.py @@ -1,10 +1,13 @@ from datetime import timedelta from importlib import import_module +import requests from django.conf import settings +from django.core.cache import cache from django.core.management.base import BaseCommand from django.db import DEFAULT_DB_ALIAS from django.utils import timezone +from packaging import version from extras.models import ObjectChange @@ -48,4 +51,37 @@ class Command(BaseCommand): f"\tSkipping: No retention period specified (CHANGELOG_RETENTION = {settings.CHANGELOG_RETENTION})" ) + # Check for new releases (if enabled) + self.stdout.write("[*] Checking for latest release") + if settings.RELEASE_CHECK_URL: + headers = { + 'Accept': 'application/vnd.github.v3+json', + } + + try: + self.stdout.write(f"\tFetching {settings.RELEASE_CHECK_URL}") + response = requests.get( + url=settings.RELEASE_CHECK_URL, + headers=headers, + proxies=settings.HTTP_PROXIES + ) + response.raise_for_status() + + releases = [] + for release in response.json(): + if 'tag_name' not in release or release.get('devrelease') or release.get('prerelease'): + continue + releases.append((version.parse(release['tag_name']), release.get('html_url'))) + latest_release = max(releases) + self.stdout.write(f"\tFound {len(response.json())} releases; {len(releases)} usable") + self.stdout.write(f"\tLatest release: {latest_release[0]}") + + # Cache the most recent release + cache.set('latest_release', latest_release, None) + + except requests.exceptions.RequestException as exc: + self.stdout.write(f"\tRequest error: {exc}") + else: + self.stdout.write(f"\tSkipping: RELEASE_CHECK_URL not set") + self.stdout.write("Finished.", self.style.SUCCESS) From af58204cd7ab6e4fb2226e87c9f1d758b10c062c Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 7 Jul 2021 21:50:48 -0400 Subject: [PATCH 2/5] Remove release checkng background task --- netbox/netbox/releases.py | 32 ------- netbox/netbox/tests/test_releases.py | 138 --------------------------- netbox/netbox/views/__init__.py | 10 +- netbox/utilities/background_tasks.py | 42 -------- 4 files changed, 5 insertions(+), 217 deletions(-) delete mode 100644 netbox/netbox/releases.py delete mode 100644 netbox/netbox/tests/test_releases.py delete mode 100644 netbox/utilities/background_tasks.py diff --git a/netbox/netbox/releases.py b/netbox/netbox/releases.py deleted file mode 100644 index 57aa37e54..000000000 --- a/netbox/netbox/releases.py +++ /dev/null @@ -1,32 +0,0 @@ -import logging - -from django.conf import settings -from django.core.cache import cache -from django_rq import get_queue - -from utilities.background_tasks import get_releases - -logger = logging.getLogger('netbox.releases') - - -def get_latest_release(pre_releases=False): - if settings.RELEASE_CHECK_URL: - logger.debug("Checking for most recent release") - latest_release = cache.get('latest_release') - if latest_release: - logger.debug(f"Found cached release: {latest_release}") - return latest_release - else: - # Check for an existing job. This can happen if the RQ worker process is not running. - queue = get_queue('check_releases') - if queue.jobs: - logger.warning("Job to check for new releases is already queued; skipping") - else: - # Get the releases in the background worker, it will fill the cache - logger.info("Initiating background task to retrieve updated releases list") - get_releases.delay(pre_releases=pre_releases) - - else: - logger.debug("Skipping release check; RELEASE_CHECK_URL not defined") - - return 'unknown', None diff --git a/netbox/netbox/tests/test_releases.py b/netbox/netbox/tests/test_releases.py deleted file mode 100644 index 5af25754c..000000000 --- a/netbox/netbox/tests/test_releases.py +++ /dev/null @@ -1,138 +0,0 @@ -from io import BytesIO -from logging import ERROR -from unittest.mock import Mock, patch - -import requests -from django.conf import settings -from django.core.cache import cache -from django.test import SimpleTestCase, override_settings -from packaging.version import Version -from requests import Response - -from utilities.background_tasks import get_releases - - -def successful_github_response(url, *_args, **_kwargs): - r = Response() - r.url = url - r.status_code = 200 - r.reason = 'OK' - r.headers = { - 'Content-Type': 'application/json; charset=utf-8', - } - r.raw = BytesIO(b'''[ - { - "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.7.8", - "tag_name": "v2.7.8", - "prerelease": false - }, - { - "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.6-beta1", - "tag_name": "v2.6-beta1", - "prerelease": true - }, - { - "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.5.9", - "tag_name": "v2.5.9", - "prerelease": false - } - ] - ''') - return r - - -def unsuccessful_github_response(url, *_args, **_kwargs): - r = Response() - r.url = url - r.status_code = 404 - r.reason = 'Not Found' - r.headers = { - 'Content-Type': 'application/json; charset=utf-8', - } - r.raw = BytesIO(b'''{ - "message": "Not Found", - "documentation_url": "https://developer.github.com/v3/repos/releases/#list-releases-for-a-repository" - } - ''') - return r - - -@override_settings(RELEASE_CHECK_URL='https://localhost/unittest/releases', RELEASE_CHECK_TIMEOUT=160876) -class GetReleasesTestCase(SimpleTestCase): - @patch.object(requests, 'get') - @patch.object(cache, 'set') - def test_pre_releases(self, dummy_cache_set: Mock, dummy_request_get: Mock): - dummy_request_get.side_effect = successful_github_response - - releases = get_releases(pre_releases=True) - - # Check result - self.assertListEqual(releases, [ - (Version('2.7.8'), 'https://github.com/netbox-community/netbox/releases/tag/v2.7.8'), - (Version('2.6b1'), 'https://github.com/netbox-community/netbox/releases/tag/v2.6-beta1'), - (Version('2.5.9'), 'https://github.com/netbox-community/netbox/releases/tag/v2.5.9') - ]) - - # Check if correct request is made - dummy_request_get.assert_called_once_with( - 'https://localhost/unittest/releases', - headers={'Accept': 'application/vnd.github.v3+json'}, - proxies=settings.HTTP_PROXIES - ) - - # Check if result is put in cache - dummy_cache_set.assert_called_once_with( - 'latest_release', - max(releases), - 160876 - ) - - @patch.object(requests, 'get') - @patch.object(cache, 'set') - def test_no_pre_releases(self, dummy_cache_set: Mock, dummy_request_get: Mock): - dummy_request_get.side_effect = successful_github_response - - releases = get_releases(pre_releases=False) - - # Check result - self.assertListEqual(releases, [ - (Version('2.7.8'), 'https://github.com/netbox-community/netbox/releases/tag/v2.7.8'), - (Version('2.5.9'), 'https://github.com/netbox-community/netbox/releases/tag/v2.5.9') - ]) - - # Check if correct request is made - dummy_request_get.assert_called_once_with( - 'https://localhost/unittest/releases', - headers={'Accept': 'application/vnd.github.v3+json'}, - proxies=settings.HTTP_PROXIES - ) - - # Check if result is put in cache - dummy_cache_set.assert_called_once_with( - 'latest_release', - max(releases), - 160876 - ) - - @patch.object(requests, 'get') - def test_failed_request(self, dummy_request_get: Mock): - dummy_request_get.side_effect = unsuccessful_github_response - - with self.assertLogs(level=ERROR) as cm: - releases = get_releases() - - # Check log entry - self.assertEqual(len(cm.output), 1) - log_output = cm.output[0] - last_log_line = log_output.split('\n')[-1] - self.assertRegex(last_log_line, '404 .* Not Found') - - # Check result - self.assertListEqual(releases, []) - - # Check if correct request is made - dummy_request_get.assert_called_once_with( - 'https://localhost/unittest/releases', - headers={'Accept': 'application/vnd.github.v3+json'}, - proxies=settings.HTTP_PROXIES - ) diff --git a/netbox/netbox/views/__init__.py b/netbox/netbox/views/__init__.py index d54c79e10..ccd383c1f 100644 --- a/netbox/netbox/views/__init__.py +++ b/netbox/netbox/views/__init__.py @@ -3,6 +3,7 @@ import sys from django.conf import settings from django.contrib.contenttypes.models import ContentType +from django.core.cache import cache from django.db.models import F from django.http import HttpResponseServerError from django.shortcuts import redirect, render @@ -23,7 +24,6 @@ from extras.models import ObjectChange, JobResult from ipam.models import Aggregate, IPAddress, Prefix, VLAN, VRF from netbox.constants import SEARCH_MAX_RESULTS, SEARCH_TYPES from netbox.forms import SearchForm -from netbox.releases import get_latest_release from tenancy.models import Tenant from virtualization.models import Cluster, VirtualMachine @@ -119,10 +119,10 @@ class HomeView(View): # Check whether a new release is available. (Only for staff/superusers.) new_release = None if request.user.is_staff or request.user.is_superuser: - latest_release, release_url = get_latest_release() - if isinstance(latest_release, version.Version): - current_version = version.parse(settings.VERSION) - if latest_release > current_version: + latest_release = cache.get('latest_release') + if latest_release: + release_version, release_url = latest_release + if release_version > version.parse(settings.VERSION): new_release = { 'version': str(latest_release), 'url': release_url, diff --git a/netbox/utilities/background_tasks.py b/netbox/utilities/background_tasks.py deleted file mode 100644 index e0af6ebef..000000000 --- a/netbox/utilities/background_tasks.py +++ /dev/null @@ -1,42 +0,0 @@ -import logging - -import requests -from django.conf import settings -from django.core.cache import cache -from django_rq import job -from packaging import version - -# Get an instance of a logger -logger = logging.getLogger('netbox.releases') - - -@job('check_releases') -def get_releases(pre_releases=False): - url = settings.RELEASE_CHECK_URL - headers = { - 'Accept': 'application/vnd.github.v3+json', - } - releases = [] - - try: - logger.info(f"Fetching new releases from {url}") - response = requests.get(url, headers=headers, proxies=settings.HTTP_PROXIES) - response.raise_for_status() - total_releases = len(response.json()) - - for release in response.json(): - if 'tag_name' not in release: - continue - if not pre_releases and (release.get('devrelease') or release.get('prerelease')): - continue - releases.append((version.parse(release['tag_name']), release.get('html_url'))) - logger.debug(f"Found {total_releases} releases; {len(releases)} usable") - - except requests.exceptions.RequestException as exc: - logger.exception(f"Error while fetching latest release from {url}: {exc}") - return [] - - # Cache the most recent release - cache.set('latest_release', max(releases), settings.RELEASE_CHECK_TIMEOUT) - - return releases From 16529372f00a18883887da9e4a29db66e153f266 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 7 Jul 2021 21:55:07 -0400 Subject: [PATCH 3/5] Remove the RELEASE_CHECK_TIMEOUT parameter --- docs/configuration/optional-settings.md | 10 +--------- netbox/netbox/configuration.example.py | 3 --- netbox/netbox/settings.py | 13 +++++++------ 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/docs/configuration/optional-settings.md b/docs/configuration/optional-settings.md index 8a9924bca..46c08bdc1 100644 --- a/docs/configuration/optional-settings.md +++ b/docs/configuration/optional-settings.md @@ -478,19 +478,11 @@ When remote user authentication is in use, this is the name of the HTTP header w --- -## RELEASE_CHECK_TIMEOUT - -Default: 86,400 (24 hours) - -The number of seconds to retain the latest version that is fetched from the GitHub API before automatically invalidating it and fetching it from the API again. This must be set to at least one hour (3600 seconds). - ---- - ## RELEASE_CHECK_URL Default: None (disabled) -This parameter defines the URL of the repository that will be checked periodically for new NetBox releases. When a new release is detected, a message will be displayed to administrative users on the home page. This can be set to the official repository (`'https://api.github.com/repos/netbox-community/netbox/releases'`) or a custom fork. Set this to `None` to disable automatic update checks. +This parameter defines the URL of the repository that will be checked for new NetBox releases. When a new release is detected, a message will be displayed to administrative users on the home page. This can be set to the official repository (`'https://api.github.com/repos/netbox-community/netbox/releases'`) or a custom fork. Set this to `None` to disable automatic update checks. !!! note The URL provided **must** be compatible with the [GitHub REST API](https://docs.github.com/en/rest). diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index 9491db27d..ebbb05891 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -241,9 +241,6 @@ REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] REMOTE_AUTH_DEFAULT_PERMISSIONS = {} -# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour. -RELEASE_CHECK_TIMEOUT = 24 * 3600 - # This repository is used to check whether there is a new release of NetBox available. Set to None to disable the # version check or use the URL below to check for release in the official NetBox repository. RELEASE_CHECK_URL = None diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 2ba1a047a..1ec20af24 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -47,7 +47,13 @@ except ModuleNotFoundError as e: # Warn on removed config parameters if hasattr(configuration, 'CACHE_TIMEOUT'): - warnings.warn("The CACHE_TIMEOUT configuration parameter was removed in v3.0.0 and no longer has any effect.") + warnings.warn( + "The CACHE_TIMEOUT configuration parameter was removed in v3.0.0 and no longer has any effect." + ) +if hasattr(configuration, 'RELEASE_CHECK_TIMEOUT'): + warnings.warn( + "The RELEASE_CHECK_TIMEOUT configuration parameter was removed in v3.0.0 and no longer has any effect." + ) # Enforce required configuration parameters for parameter in ['ALLOWED_HOSTS', 'DATABASE', 'SECRET_KEY', 'REDIS']: @@ -114,7 +120,6 @@ REMOTE_AUTH_DEFAULT_PERMISSIONS = getattr(configuration, 'REMOTE_AUTH_DEFAULT_PE REMOTE_AUTH_ENABLED = getattr(configuration, 'REMOTE_AUTH_ENABLED', False) REMOTE_AUTH_HEADER = getattr(configuration, 'REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER') RELEASE_CHECK_URL = getattr(configuration, 'RELEASE_CHECK_URL', None) -RELEASE_CHECK_TIMEOUT = getattr(configuration, 'RELEASE_CHECK_TIMEOUT', 24 * 3600) REPORTS_ROOT = getattr(configuration, 'REPORTS_ROOT', os.path.join(BASE_DIR, 'reports')).rstrip('/') RQ_DEFAULT_TIMEOUT = getattr(configuration, 'RQ_DEFAULT_TIMEOUT', 300) SCRIPTS_ROOT = getattr(configuration, 'SCRIPTS_ROOT', os.path.join(BASE_DIR, 'scripts')).rstrip('/') @@ -141,10 +146,6 @@ if RELEASE_CHECK_URL: except ValidationError as err: raise ImproperlyConfigured(str(err)) -# Enforce a minimum cache timeout for update checks -if RELEASE_CHECK_TIMEOUT < 3600: - raise ImproperlyConfigured("RELEASE_CHECK_TIMEOUT has to be at least 3600 seconds (1 hour)") - # # Database From 1f048a6d6101cdbc4c9cb973572809000418fabd Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 7 Jul 2021 21:58:54 -0400 Subject: [PATCH 4/5] Changelog for #6713 --- docs/release-notes/version-3.0.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/release-notes/version-3.0.md b/docs/release-notes/version-3.0.md index db9aa1b82..9ed9a9492 100644 --- a/docs/release-notes/version-3.0.md +++ b/docs/release-notes/version-3.0.md @@ -68,10 +68,12 @@ CustomValidator can also be subclassed to enforce more complex logic by overridi * [#6068](https://github.com/netbox-community/netbox/issues/6068) - Drop support for legacy static CSV export * [#6338](https://github.com/netbox-community/netbox/issues/6338) - Decimal fields are no longer coerced to strings in REST API * [#6639](https://github.com/netbox-community/netbox/issues/6639) - Drop support for queryset caching (django-cacheops) +* [#6713](https://github.com/netbox-community/netbox/issues/6713) - Checking for new releases is now done as part of the housekeeping routine ### Configuration Changes * The `CACHE_TIMEOUT` configuration parameter has been removed. +* The `RELEASE_CHECK_TIMEOUT` configuration parameter has been removed. ### REST API Changes From 918d739acb92bc34f1fab83cf4661d371881e81a Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 7 Jul 2021 22:10:10 -0400 Subject: [PATCH 5/5] Remove the check_releases queue --- netbox/netbox/settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 1ec20af24..09447a7a9 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -546,8 +546,7 @@ else: } RQ_QUEUES = { - 'default': RQ_PARAMS, # Webhooks - 'check_releases': RQ_PARAMS, + 'default': RQ_PARAMS, }