diff --git a/docs/configuration/optional-settings.md b/docs/configuration/optional-settings.md index cbe01728c..f302e000d 100644 --- a/docs/configuration/optional-settings.md +++ b/docs/configuration/optional-settings.md @@ -157,6 +157,24 @@ Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce uni --- +## UPDATE_REPO_URL + +Default: None + +The releases of this repository are checked to detect new releases, which are shown on the home page of the web interface. You can change this to your own fork of the NetBox repository, or set it to `None` to disable the check. The URL provided **must** be compatible with the GitHub API. + +Use `'https://api.github.com/repos/netbox-community/netbox'` to check for release in the official NetBox repository. + +--- + +## UPDATE_CACHE_TIMEOUT + +Default: 86,400 (24 hours) + +The number of seconds to retain the latest version that is fetched from the GitHub API before automatically invalidating it and fetching it from the API again. This must be set to at least one hour (3600 seconds). + +--- + ## LOGGING By default, all messages of INFO severity or higher will be logged to the console. Additionally, if `DEBUG` is False and email access has been configured, ERROR and CRITICAL messages will be emailed to the users defined in `ADMINS`. diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index 7002def9b..3c7de6128 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -124,6 +124,14 @@ EXEMPT_VIEW_PERMISSIONS = [ # 'ipam.prefix', ] +# This repository is used to check whether there is a new release of NetBox available. Set to None to disable the +# version check or use the URL below to check for release in the official NetBox repository. +UPDATE_REPO_URL = None +# UPDATE_REPO_URL = 'https://api.github.com/repos/netbox-community/netbox' + +# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour. +UPDATE_CACHE_TIMEOUT = 24 * 3600 + # Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs: # https://docs.djangoproject.com/en/stable/topics/logging/ LOGGING = {} diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 7f4f44b1a..59ece861e 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -1,8 +1,10 @@ import logging import os import platform +import re import socket import warnings +from urllib.parse import urlsplit from django.contrib.messages import constants as messages from django.core.exceptions import ImproperlyConfigured @@ -78,6 +80,8 @@ DEVELOPER = getattr(configuration, 'DEVELOPER', False) EMAIL = getattr(configuration, 'EMAIL', {}) ENFORCE_GLOBAL_UNIQUE = getattr(configuration, 'ENFORCE_GLOBAL_UNIQUE', False) EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', []) +UPDATE_REPO_URL = getattr(configuration, 'UPDATE_REPO_URL', None) +UPDATE_CACHE_TIMEOUT = getattr(configuration, 'UPDATE_CACHE_TIMEOUT', 24 * 3600) LOGGING = getattr(configuration, 'LOGGING', {}) LOGIN_REQUIRED = getattr(configuration, 'LOGIN_REQUIRED', False) LOGIN_TIMEOUT = getattr(configuration, 'LOGIN_TIMEOUT', None) @@ -302,6 +306,31 @@ AUTHENTICATION_BACKENDS = [ 'utilities.auth_backends.ViewExemptModelBackend', ] +# GitHub repository for version check +if UPDATE_REPO_URL: + UPDATE_REPO_URL = UPDATE_REPO_URL.rstrip('/') + try: + scheme, netloc, path, query, fragment = urlsplit(UPDATE_REPO_URL) + except ValueError: + raise ImproperlyConfigured("UPDATE_REPO_URL must be a valid URL") + + if scheme not in ('http', 'https'): + raise ImproperlyConfigured("UPDATE_REPO_URL must be a valid http:// or https:// URL") + + if not re.fullmatch(r'/repos/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', path): + raise ImproperlyConfigured( + "GITHUB_REPOSITORY must contain the base URL of the GitHub API in a form like " + "'https://api.github.com/repos//'" + ) + + # Don't allow ? (query) and # (fragment) in the URL + if query or fragment: + raise ImproperlyConfigured("UPDATE_REPO_URL may not contain a ? (query) or # (fragment)") + +# Enforce a cache timeout of at least an hour to protect GitHub +if UPDATE_CACHE_TIMEOUT < 3600: + raise ImproperlyConfigured("UPDATE_CACHE_TIMEOUT has to be at least 3600 seconds (1 hour)") + # Internationalization LANGUAGE_CODE = 'en-us' USE_I18N = True diff --git a/netbox/netbox/tests/test_get_releases.py b/netbox/netbox/tests/test_get_releases.py new file mode 100644 index 000000000..1fc709b03 --- /dev/null +++ b/netbox/netbox/tests/test_get_releases.py @@ -0,0 +1,160 @@ +from io import BytesIO +from logging import ERROR +from unittest.mock import Mock, patch + +import requests +from cacheops import CacheMiss, RedisCache +from django.test import SimpleTestCase, override_settings +from packaging.version import Version +from requests import Response + +from utilities.background_tasks import get_releases + + +def successful_github_response(url, *_args, **_kwargs): + r = Response() + r.url = url + r.status_code = 200 + r.reason = 'OK' + r.headers = { + 'Content-Type': 'application/json; charset=utf-8', + } + r.raw = BytesIO(b'''[ + { + "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.7.8", + "tag_name": "v2.7.8", + "prerelease": false + }, + { + "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.6-beta1", + "tag_name": "v2.6-beta1", + "prerelease": true + }, + { + "html_url": "https://github.com/netbox-community/netbox/releases/tag/v2.5.9", + "tag_name": "v2.5.9", + "prerelease": false + } + ] + ''') + return r + + +def unsuccessful_github_response(url, *_args, **_kwargs): + r = Response() + r.url = url + r.status_code = 404 + r.reason = 'Not Found' + r.headers = { + 'Content-Type': 'application/json; charset=utf-8', + } + r.raw = BytesIO(b'''{ + "message": "Not Found", + "documentation_url": "https://developer.github.com/v3/repos/releases/#list-releases-for-a-repository" + } + ''') + return r + + +@override_settings(UPDATE_REPO_URL='https://localhost/unittest', UPDATE_CACHE_TIMEOUT=160876) +class GetReleasesTestCase(SimpleTestCase): + @patch.object(requests, 'get') + @patch.object(RedisCache, 'set') + @patch.object(RedisCache, 'get') + def test_pre_releases(self, dummy_cache_get: Mock, dummy_cache_set: Mock, dummy_request_get: Mock): + dummy_cache_get.side_effect = CacheMiss() + dummy_request_get.side_effect = successful_github_response + + releases = get_releases(pre_releases=True) + + # Check result + self.assertListEqual(releases, [ + (Version('2.7.8'), 'https://github.com/netbox-community/netbox/releases/tag/v2.7.8'), + (Version('2.6b1'), 'https://github.com/netbox-community/netbox/releases/tag/v2.6-beta1'), + (Version('2.5.9'), 'https://github.com/netbox-community/netbox/releases/tag/v2.5.9') + ]) + + # Check if correct request is made + dummy_request_get.assert_called_once() + dummy_request_get.assert_called_with('https://localhost/unittest/releases', + headers={ + 'Accept': 'application/vnd.github.v3+json' + }) + + # Check if result is put in cache + dummy_cache_set.assert_called_once() + dummy_cache_set.assert_called_with('netbox_releases', releases, 160876) + + @patch.object(requests, 'get') + @patch.object(RedisCache, 'set') + @patch.object(RedisCache, 'get') + def test_no_pre_releases(self, dummy_cache_get: Mock, dummy_cache_set: Mock, dummy_request_get: Mock): + dummy_cache_get.side_effect = CacheMiss() + dummy_request_get.side_effect = successful_github_response + + releases = get_releases(pre_releases=False) + + # Check result + self.assertListEqual(releases, [ + (Version('2.7.8'), 'https://github.com/netbox-community/netbox/releases/tag/v2.7.8'), + (Version('2.5.9'), 'https://github.com/netbox-community/netbox/releases/tag/v2.5.9') + ]) + + # Check if correct request is made + dummy_request_get.assert_called_once() + dummy_request_get.assert_called_with('https://localhost/unittest/releases', + headers={ + 'Accept': 'application/vnd.github.v3+json' + }) + + # Check if result is put in cache + dummy_cache_set.assert_called_once() + dummy_cache_set.assert_called_with('netbox_releases', releases, 160876) + + @patch.object(requests, 'get') + @patch.object(RedisCache, 'set') + @patch.object(RedisCache, 'get') + def test_failed_request(self, dummy_cache_get: Mock, dummy_cache_set: Mock, dummy_request_get: Mock): + dummy_cache_get.side_effect = CacheMiss() + dummy_request_get.side_effect = unsuccessful_github_response + + with self.assertLogs(level=ERROR) as cm: + releases = get_releases() + + # Check log entry + self.assertEqual(len(cm.output), 1) + log_output = cm.output[0] + last_log_line = log_output.split('\n')[-1] + self.assertRegex(last_log_line, '404 .* Not Found') + + # Check result + self.assertListEqual(releases, []) + + # Check if correct request is made + dummy_request_get.assert_called_once() + dummy_request_get.assert_called_with('https://localhost/unittest/releases', + headers={ + 'Accept': 'application/vnd.github.v3+json' + }) + + # Check if failure is put in cache + dummy_cache_set.assert_called_once() + dummy_cache_set.assert_called_with('netbox_releases_no_retry', 'https://localhost/unittest/releases', 900) + + @patch.object(requests, 'get') + @patch.object(RedisCache, 'set') + @patch.object(RedisCache, 'get') + def test_blocked_retry(self, dummy_cache_get: Mock, dummy_cache_set: Mock, dummy_request_get: Mock): + dummy_cache_get.return_value = 'https://localhost/unittest/releases' + dummy_request_get.side_effect = successful_github_response + + releases = get_releases() + + # Check result + self.assertListEqual(releases, []) + + # Check if request is NOT made + dummy_request_get.assert_not_called() + + # Check if cache is not updated + dummy_cache_set.assert_not_called() diff --git a/netbox/netbox/views.py b/netbox/netbox/views.py index 05bcea90d..7634893a7 100644 --- a/netbox/netbox/views.py +++ b/netbox/netbox/views.py @@ -1,8 +1,10 @@ from collections import OrderedDict -from django.db.models import Count, F +from django.conf import settings +from django.db.models import Count, F, OuterRef, Subquery from django.shortcuts import render from django.views.generic import View +from packaging import version from rest_framework.response import Response from rest_framework.reverse import reverse from rest_framework.views import APIView @@ -31,6 +33,7 @@ from secrets.tables import SecretTable from tenancy.filters import TenantFilterSet from tenancy.models import Tenant from tenancy.tables import TenantTable +from utilities.releases import get_latest_release from virtualization.filters import ClusterFilterSet, VirtualMachineFilterSet from virtualization.models import Cluster, VirtualMachine from virtualization.tables import ClusterTable, VirtualMachineDetailTable @@ -240,11 +243,25 @@ class HomeView(View): } + new_release = None + new_release_url = None + + if request.user.is_staff or request.user.is_superuser: + # Only check for new releases if the current user might be able to do anything about it + latest_release, github_url = get_latest_release() + if isinstance(latest_release, version.Version): + current_version = version.parse(settings.VERSION) + if latest_release > current_version: + new_release = str(latest_release) + new_release_url = github_url + return render(request, self.template_name, { 'search_form': SearchForm(), 'stats': stats, 'report_results': ReportResult.objects.order_by('-created')[:10], - 'changelog': ObjectChange.objects.prefetch_related('user', 'changed_object_type')[:15] + 'changelog': ObjectChange.objects.prefetch_related('user', 'changed_object_type')[:15], + 'new_release': new_release, + 'new_release_url': new_release_url, }) diff --git a/netbox/templates/home.html b/netbox/templates/home.html index 6977bba4c..b560d8894 100644 --- a/netbox/templates/home.html +++ b/netbox/templates/home.html @@ -1,6 +1,19 @@ {% extends '_base.html' %} {% load helpers %} +{% block header %} + {{ block.super }} + {% if new_release %} + + {% endif %} +{% endblock %} + + {% block content %} {% include 'search_form.html' %}
diff --git a/netbox/utilities/background_tasks.py b/netbox/utilities/background_tasks.py new file mode 100644 index 000000000..2accc5d03 --- /dev/null +++ b/netbox/utilities/background_tasks.py @@ -0,0 +1,51 @@ +import logging + +import requests +from cacheops.simple import cache, CacheMiss +from django.conf import settings +from django_rq import job +from packaging import version + +# Get an instance of a logger +logger = logging.getLogger(__name__) + + +@job +def get_releases(pre_releases=False): + url = '{}/releases'.format(settings.UPDATE_REPO_URL) + headers = { + 'Accept': 'application/vnd.github.v3+json', + } + + # Check whether this URL has failed and shouldn't be retried yet + try: + failed_url = cache.get('netbox_releases_no_retry') + if url == failed_url: + return [] + except CacheMiss: + pass + + releases = [] + + # noinspection PyBroadException + try: + response = requests.get(url, headers=headers) + response.raise_for_status() + + for release in response.json(): + if 'tag_name' not in release: + continue + + if not pre_releases and (release.get('devrelease') or release.get('prerelease')): + continue + + releases.append((version.parse(release['tag_name']), release.get('html_url'))) + except Exception: + # Don't retry this URL for 15 minutes + cache.set('netbox_releases_no_retry', url, 900) + + logger.exception("Error while fetching {}".format(url)) + return [] + + cache.set('netbox_releases', releases, settings.UPDATE_CACHE_TIMEOUT) + return releases diff --git a/netbox/utilities/releases.py b/netbox/utilities/releases.py new file mode 100644 index 000000000..1374957a2 --- /dev/null +++ b/netbox/utilities/releases.py @@ -0,0 +1,24 @@ +import logging + +from cacheops import CacheMiss, cache +from django.conf import settings + +from utilities.background_tasks import get_releases + +# Get an instance of a logger +logger = logging.getLogger(__name__) + + +def get_latest_release(pre_releases=False): + if settings.UPDATE_REPO_URL: + try: + releases = cache.get('netbox_releases') + if releases: + return max(releases) + except CacheMiss: + logger.debug("Starting background task to get releases") + + # Get the releases in the background worker, it will fill the cache + get_releases.delay(pre_releases=pre_releases) + + return 'unknown', None