mirror of
https://github.com/netbox-community/netbox.git
synced 2025-07-19 05:21:55 -06:00
Merge pull request #6717 from netbox-community/6713-release-checking
Closes #6713: Move release checking to the housekeeping routine
This commit is contained in:
Jeremy Stretch
GitHub
commit
65aaab5f38
@ -478,19 +478,11 @@ When remote user authentication is in use, this is the name of the HTTP header w
|
||||
|
||||
---
|
||||
|
||||
## RELEASE_CHECK_TIMEOUT
|
||||
|
||||
Default: 86,400 (24 hours)
|
||||
|
||||
The number of seconds to retain the latest version that is fetched from the GitHub API before automatically invalidating it and fetching it from the API again. This must be set to at least one hour (3600 seconds).
|
||||
|
||||
---
|
||||
|
||||
## RELEASE_CHECK_URL
|
||||
|
||||
Default: None (disabled)
|
||||
|
||||
This parameter defines the URL of the repository that will be checked periodically for new NetBox releases. When a new release is detected, a message will be displayed to administrative users on the home page. This can be set to the official repository (`'https://api.github.com/repos/netbox-community/netbox/releases'`) or a custom fork. Set this to `None` to disable automatic update checks.
|
||||
This parameter defines the URL of the repository that will be checked for new NetBox releases. When a new release is detected, a message will be displayed to administrative users on the home page. This can be set to the official repository (`'https://api.github.com/repos/netbox-community/netbox/releases'`) or a custom fork. Set this to `None` to disable automatic update checks.
|
||||
|
||||
!!! note
|
||||
The URL provided **must** be compatible with the [GitHub REST API](https://docs.github.com/en/rest).
|
||||
|
||||
@ -68,10 +68,12 @@ CustomValidator can also be subclassed to enforce more complex logic by overridi
|
||||
* [#6068](https://github.com/netbox-community/netbox/issues/6068) - Drop support for legacy static CSV export
|
||||
* [#6338](https://github.com/netbox-community/netbox/issues/6338) - Decimal fields are no longer coerced to strings in REST API
|
||||
* [#6639](https://github.com/netbox-community/netbox/issues/6639) - Drop support for queryset caching (django-cacheops)
|
||||
* [#6713](https://github.com/netbox-community/netbox/issues/6713) - Checking for new releases is now done as part of the housekeeping routine
|
||||
|
||||
### Configuration Changes
|
||||
|
||||
* The `CACHE_TIMEOUT` configuration parameter has been removed.
|
||||
* The `RELEASE_CHECK_TIMEOUT` configuration parameter has been removed.
|
||||
|
||||
### REST API Changes
|
||||
|
||||
|
||||
@ -1,10 +1,13 @@
|
||||
from datetime import timedelta
|
||||
from importlib import import_module
|
||||
|
||||
import requests
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import DEFAULT_DB_ALIAS
|
||||
from django.utils import timezone
|
||||
from packaging import version
|
||||
|
||||
from extras.models import ObjectChange
|
||||
|
||||
@ -48,4 +51,37 @@ class Command(BaseCommand):
|
||||
f"\tSkipping: No retention period specified (CHANGELOG_RETENTION = {settings.CHANGELOG_RETENTION})"
|
||||
)
|
||||
|
||||
# Check for new releases (if enabled)
|
||||
self.stdout.write("[*] Checking for latest release")
|
||||
if settings.RELEASE_CHECK_URL:
|
||||
headers = {
|
||||
'Accept': 'application/vnd.github.v3+json',
|
||||
}
|
||||
|
||||
try:
|
||||
self.stdout.write(f"\tFetching {settings.RELEASE_CHECK_URL}")
|
||||
response = requests.get(
|
||||
url=settings.RELEASE_CHECK_URL,
|
||||
headers=headers,
|
||||
proxies=settings.HTTP_PROXIES
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
releases = []
|
||||
for release in response.json():
|
||||
if 'tag_name' not in release or release.get('devrelease') or release.get('prerelease'):
|
||||
continue
|
||||
releases.append((version.parse(release['tag_name']), release.get('html_url')))
|
||||
latest_release = max(releases)
|
||||
self.stdout.write(f"\tFound {len(response.json())} releases; {len(releases)} usable")
|
||||
self.stdout.write(f"\tLatest release: {latest_release[0]}")
|
||||
|
||||
# Cache the most recent release
|
||||
cache.set('latest_release', latest_release, None)
|
||||
|
||||
except requests.exceptions.RequestException as exc:
|
||||
self.stdout.write(f"\tRequest error: {exc}")
|
||||
else:
|
||||
self.stdout.write(f"\tSkipping: RELEASE_CHECK_URL not set")
|
||||
|
||||
self.stdout.write("Finished.", self.style.SUCCESS)
|
||||
|
||||
@ -241,9 +241,6 @@ REMOTE_AUTH_AUTO_CREATE_USER = True
|
||||
REMOTE_AUTH_DEFAULT_GROUPS = []
|
||||
REMOTE_AUTH_DEFAULT_PERMISSIONS = {}
|
||||
|
||||
# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour.
|
||||
RELEASE_CHECK_TIMEOUT = 24 * 3600
|
||||
|
||||
# This repository is used to check whether there is a new release of NetBox available. Set to None to disable the
|
||||
# version check or use the URL below to check for release in the official NetBox repository.
|
||||
RELEASE_CHECK_URL = None
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
import logging
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django_rq import get_queue
|
||||
|
||||
from utilities.background_tasks import get_releases
|
||||
|
||||
logger = logging.getLogger('netbox.releases')
|
||||
|
||||
|
||||
def get_latest_release(pre_releases=False):
|
||||
if settings.RELEASE_CHECK_URL:
|
||||
logger.debug("Checking for most recent release")
|
||||
latest_release = cache.get('latest_release')
|
||||
if latest_release:
|
||||
logger.debug(f"Found cached release: {latest_release}")
|
||||
return latest_release
|
||||
else:
|
||||
# Check for an existing job. This can happen if the RQ worker process is not running.
|
||||
queue = get_queue('check_releases')
|
||||
if queue.jobs:
|
||||
logger.warning("Job to check for new releases is already queued; skipping")
|
||||
else:
|
||||
# Get the releases in the background worker, it will fill the cache
|
||||
logger.info("Initiating background task to retrieve updated releases list")
|
||||
get_releases.delay(pre_releases=pre_releases)
|
||||
|
||||
else:
|
||||
logger.debug("Skipping release check; RELEASE_CHECK_URL not defined")
|
||||
|
||||
return 'unknown', None
|
||||
@ -47,7 +47,13 @@ except ModuleNotFoundError as e:
|
||||
|
||||
# Warn on removed config parameters
|
||||
if hasattr(configuration, 'CACHE_TIMEOUT'):
|
||||
warnings.warn("The CACHE_TIMEOUT configuration parameter was removed in v3.0.0 and no longer has any effect.")
|
||||
warnings.warn(
|
||||
"The CACHE_TIMEOUT configuration parameter was removed in v3.0.0 and no longer has any effect."
|
||||
)
|
||||
if hasattr(configuration, 'RELEASE_CHECK_TIMEOUT'):
|
||||
warnings.warn(
|
||||
"The RELEASE_CHECK_TIMEOUT configuration parameter was removed in v3.0.0 and no longer has any effect."
|
||||
)
|
||||
|
||||
# Enforce required configuration parameters
|
||||
for parameter in ['ALLOWED_HOSTS', 'DATABASE', 'SECRET_KEY', 'REDIS']:
|
||||
@ -114,7 +120,6 @@ REMOTE_AUTH_DEFAULT_PERMISSIONS = getattr(configuration, 'REMOTE_AUTH_DEFAULT_PE
|
||||
REMOTE_AUTH_ENABLED = getattr(configuration, 'REMOTE_AUTH_ENABLED', False)
|
||||
REMOTE_AUTH_HEADER = getattr(configuration, 'REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER')
|
||||
RELEASE_CHECK_URL = getattr(configuration, 'RELEASE_CHECK_URL', None)
|
||||
RELEASE_CHECK_TIMEOUT = getattr(configuration, 'RELEASE_CHECK_TIMEOUT', 24 * 3600)
|
||||
REPORTS_ROOT = getattr(configuration, 'REPORTS_ROOT', os.path.join(BASE_DIR, 'reports')).rstrip('/')
|
||||
RQ_DEFAULT_TIMEOUT = getattr(configuration, 'RQ_DEFAULT_TIMEOUT', 300)
|
||||
SCRIPTS_ROOT = getattr(configuration, 'SCRIPTS_ROOT', os.path.join(BASE_DIR, 'scripts')).rstrip('/')
|
||||
@ -141,10 +146,6 @@ if RELEASE_CHECK_URL:
|
||||
except ValidationError as err:
|
||||
raise ImproperlyConfigured(str(err))
|
||||
|
||||
# Enforce a minimum cache timeout for update checks
|
||||
if RELEASE_CHECK_TIMEOUT < 3600:
|
||||
raise ImproperlyConfigured("RELEASE_CHECK_TIMEOUT has to be at least 3600 seconds (1 hour)")
|
||||
|
||||
|
||||
#
|
||||
# Database
|
||||
@ -545,8 +546,7 @@ else:
|
||||
}
|
||||
|
||||
RQ_QUEUES = {
|
||||
'default': RQ_PARAMS, # Webhooks
|
||||
'check_releases': RQ_PARAMS,
|
||||
'default': RQ_PARAMS,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1,138 +0,0 @@
|
||||
from io import BytesIO
|
||||
from logging import ERROR
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import requests
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.test import SimpleTestCase, override_settings
|
||||
from packaging.version import Version
|
||||
from requests import Response
|
||||
|
||||
from utilities.background_tasks import get_releases
|
||||
|
||||
|
||||
def successful_github_response(url, *_args, **_kwargs):
|
||||
r = Response()
|
||||
r.url = url
|
||||
r.status_code = 200
|
||||
r.reason = 'OK'
|
||||
r.headers = {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
}
|
||||
r.raw = BytesIO(b'''[
|
||||
{
|
||||
"html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.7.8",
|
||||
"tag_name": "v2.7.8",
|
||||
"prerelease": false
|
||||
},
|
||||
{
|
||||
"html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.6-beta1",
|
||||
"tag_name": "v2.6-beta1",
|
||||
"prerelease": true
|
||||
},
|
||||
{
|
||||
"html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.5.9",
|
||||
"tag_name": "v2.5.9",
|
||||
"prerelease": false
|
||||
}
|
||||
]
|
||||
''')
|
||||
return r
|
||||
|
||||
|
||||
def unsuccessful_github_response(url, *_args, **_kwargs):
|
||||
r = Response()
|
||||
r.url = url
|
||||
r.status_code = 404
|
||||
r.reason = 'Not Found'
|
||||
r.headers = {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
}
|
||||
r.raw = BytesIO(b'''{
|
||||
"message": "Not Found",
|
||||
"documentation_url": "https://developer.github.com/v3/repos/releases/#list-releases-for-a-repository"
|
||||
}
|
||||
''')
|
||||
return r
|
||||
|
||||
|
||||
@override_settings(RELEASE_CHECK_URL='https://localhost/unittest/releases', RELEASE_CHECK_TIMEOUT=160876)
|
||||
class GetReleasesTestCase(SimpleTestCase):
|
||||
@patch.object(requests, 'get')
|
||||
@patch.object(cache, 'set')
|
||||
def test_pre_releases(self, dummy_cache_set: Mock, dummy_request_get: Mock):
|
||||
dummy_request_get.side_effect = successful_github_response
|
||||
|
||||
releases = get_releases(pre_releases=True)
|
||||
|
||||
# Check result
|
||||
self.assertListEqual(releases, [
|
||||
(Version('2.7.8'), 'https://github.com/netbox-community/netbox/releases/tag/v2.7.8'),
|
||||
(Version('2.6b1'), 'https://github.com/netbox-community/netbox/releases/tag/v2.6-beta1'),
|
||||
(Version('2.5.9'), 'https://github.com/netbox-community/netbox/releases/tag/v2.5.9')
|
||||
])
|
||||
|
||||
# Check if correct request is made
|
||||
dummy_request_get.assert_called_once_with(
|
||||
'https://localhost/unittest/releases',
|
||||
headers={'Accept': 'application/vnd.github.v3+json'},
|
||||
proxies=settings.HTTP_PROXIES
|
||||
)
|
||||
|
||||
# Check if result is put in cache
|
||||
dummy_cache_set.assert_called_once_with(
|
||||
'latest_release',
|
||||
max(releases),
|
||||
160876
|
||||
)
|
||||
|
||||
@patch.object(requests, 'get')
|
||||
@patch.object(cache, 'set')
|
||||
def test_no_pre_releases(self, dummy_cache_set: Mock, dummy_request_get: Mock):
|
||||
dummy_request_get.side_effect = successful_github_response
|
||||
|
||||
releases = get_releases(pre_releases=False)
|
||||
|
||||
# Check result
|
||||
self.assertListEqual(releases, [
|
||||
(Version('2.7.8'), 'https://github.com/netbox-community/netbox/releases/tag/v2.7.8'),
|
||||
(Version('2.5.9'), 'https://github.com/netbox-community/netbox/releases/tag/v2.5.9')
|
||||
])
|
||||
|
||||
# Check if correct request is made
|
||||
dummy_request_get.assert_called_once_with(
|
||||
'https://localhost/unittest/releases',
|
||||
headers={'Accept': 'application/vnd.github.v3+json'},
|
||||
proxies=settings.HTTP_PROXIES
|
||||
)
|
||||
|
||||
# Check if result is put in cache
|
||||
dummy_cache_set.assert_called_once_with(
|
||||
'latest_release',
|
||||
max(releases),
|
||||
160876
|
||||
)
|
||||
|
||||
@patch.object(requests, 'get')
|
||||
def test_failed_request(self, dummy_request_get: Mock):
|
||||
dummy_request_get.side_effect = unsuccessful_github_response
|
||||
|
||||
with self.assertLogs(level=ERROR) as cm:
|
||||
releases = get_releases()
|
||||
|
||||
# Check log entry
|
||||
self.assertEqual(len(cm.output), 1)
|
||||
log_output = cm.output[0]
|
||||
last_log_line = log_output.split('\n')[-1]
|
||||
self.assertRegex(last_log_line, '404 .* Not Found')
|
||||
|
||||
# Check result
|
||||
self.assertListEqual(releases, [])
|
||||
|
||||
# Check if correct request is made
|
||||
dummy_request_get.assert_called_once_with(
|
||||
'https://localhost/unittest/releases',
|
||||
headers={'Accept': 'application/vnd.github.v3+json'},
|
||||
proxies=settings.HTTP_PROXIES
|
||||
)
|
||||
@ -3,6 +3,7 @@ import sys
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.core.cache import cache
|
||||
from django.db.models import F
|
||||
from django.http import HttpResponseServerError
|
||||
from django.shortcuts import redirect, render
|
||||
@ -23,7 +24,6 @@ from extras.models import ObjectChange, JobResult
|
||||
from ipam.models import Aggregate, IPAddress, Prefix, VLAN, VRF
|
||||
from netbox.constants import SEARCH_MAX_RESULTS, SEARCH_TYPES
|
||||
from netbox.forms import SearchForm
|
||||
from netbox.releases import get_latest_release
|
||||
from tenancy.models import Tenant
|
||||
from virtualization.models import Cluster, VirtualMachine
|
||||
|
||||
@ -119,10 +119,10 @@ class HomeView(View):
|
||||
# Check whether a new release is available. (Only for staff/superusers.)
|
||||
new_release = None
|
||||
if request.user.is_staff or request.user.is_superuser:
|
||||
latest_release, release_url = get_latest_release()
|
||||
if isinstance(latest_release, version.Version):
|
||||
current_version = version.parse(settings.VERSION)
|
||||
if latest_release > current_version:
|
||||
latest_release = cache.get('latest_release')
|
||||
if latest_release:
|
||||
release_version, release_url = latest_release
|
||||
if release_version > version.parse(settings.VERSION):
|
||||
new_release = {
|
||||
'version': str(latest_release),
|
||||
'url': release_url,
|
||||
|
||||
@ -1,42 +0,0 @@
|
||||
import logging
|
||||
|
||||
import requests
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django_rq import job
|
||||
from packaging import version
|
||||
|
||||
# Get an instance of a logger
|
||||
logger = logging.getLogger('netbox.releases')
|
||||
|
||||
|
||||
@job('check_releases')
|
||||
def get_releases(pre_releases=False):
|
||||
url = settings.RELEASE_CHECK_URL
|
||||
headers = {
|
||||
'Accept': 'application/vnd.github.v3+json',
|
||||
}
|
||||
releases = []
|
||||
|
||||
try:
|
||||
logger.info(f"Fetching new releases from {url}")
|
||||
response = requests.get(url, headers=headers, proxies=settings.HTTP_PROXIES)
|
||||
response.raise_for_status()
|
||||
total_releases = len(response.json())
|
||||
|
||||
for release in response.json():
|
||||
if 'tag_name' not in release:
|
||||
continue
|
||||
if not pre_releases and (release.get('devrelease') or release.get('prerelease')):
|
||||
continue
|
||||
releases.append((version.parse(release['tag_name']), release.get('html_url')))
|
||||
logger.debug(f"Found {total_releases} releases; {len(releases)} usable")
|
||||
|
||||
except requests.exceptions.RequestException as exc:
|
||||
logger.exception(f"Error while fetching latest release from {url}: {exc}")
|
||||
return []
|
||||
|
||||
# Cache the most recent release
|
||||
cache.set('latest_release', max(releases), settings.RELEASE_CHECK_TIMEOUT)
|
||||
|
||||
return releases
|
||||
Loading…
Reference in New Issue
Block a user