From 4e65117e7c77c3a07b90c448b7af645e41f08c71 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 4 Mar 2025 08:24:54 -0500 Subject: [PATCH] Closes #18627: Proxy routing (#18681) * Introduce proxy routing * Misc cleanup * Document PROXY_ROUTERS parameter --- docs/configuration/system.md | 14 ++++- netbox/core/data_backends.py | 26 ++++----- netbox/core/jobs.py | 3 +- netbox/core/plugins.py | 6 +- netbox/extras/dashboard/widgets.py | 3 +- .../management/commands/housekeeping.py | 3 +- netbox/extras/webhooks.py | 8 ++- netbox/netbox/settings.py | 13 ++++- netbox/utilities/proxy.py | 55 +++++++++++++++++++ 9 files changed, 108 insertions(+), 23 deletions(-) create mode 100644 netbox/utilities/proxy.py diff --git a/docs/configuration/system.md b/docs/configuration/system.md index af3a6f5e6..81c1a6a94 100644 --- a/docs/configuration/system.md +++ b/docs/configuration/system.md @@ -64,7 +64,7 @@ Email is sent from NetBox only for critical events or if configured for [logging ## HTTP_PROXIES -Default: None +Default: Empty A dictionary of HTTP proxies to use for outbound requests originating from NetBox (e.g. when sending webhook requests). Proxies should be specified by schema (HTTP and HTTPS) as per the [Python requests library documentation](https://requests.readthedocs.io/en/latest/user/advanced/#proxies). For example: @@ -75,6 +75,8 @@ HTTP_PROXIES = { } ``` +If more flexibility is needed in determining which proxy to use for a given request, consider implementing one or more custom proxy routers via the [`PROXY_ROUTERS`](#proxy_routers) parameter. + --- ## INTERNAL_IPS @@ -160,6 +162,16 @@ The file path to the location where media files (such as image attachments) are --- +## PROXY_ROUTERS + +Default: `["utilities.proxy.DefaultProxyRouter"]` + +A list of Python classes responsible for determining which proxy server(s) to use for outbound HTTP requests. Each item in the list can be the class itself or the dotted path to the class. + +The `route()` method on each class must return a dictionary of candidate proxies arranged by protocol (e.g. `http` and/or `https`), or None if no viable proxy can be determined. The default class, `DefaultProxyRouter`, simply returns the content of [`HTTP_PROXIES`](#http_proxies). + +--- + ## REPORTS_ROOT Default: `$INSTALL_ROOT/netbox/reports/` diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index 770a3b258..9ba1d5dfd 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -7,13 +7,13 @@ from pathlib import Path from urllib.parse import urlparse from django import forms -from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.utils.translation import gettext as _ from netbox.data_backends import DataBackend from netbox.utils import register_data_backend from utilities.constants import HTTP_PROXY_SUPPORTED_SCHEMAS, HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS +from utilities.proxy import resolve_proxies from utilities.socks import ProxyPoolManager from .exceptions import SyncError @@ -70,18 +70,18 @@ class GitBackend(DataBackend): # Initialize backend config config = ConfigDict() - self.use_socks = False + self.socks_proxy = None # Apply HTTP proxy (if configured) - if settings.HTTP_PROXIES: - if proxy := settings.HTTP_PROXIES.get(self.url_scheme, None): - if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS: - raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}") + proxies = resolve_proxies(url=self.url, context={'client': self}) or {} + if proxy := proxies.get(self.url_scheme): + if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS: + raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}") - if self.url_scheme in ('http', 'https'): - config.set("http", "proxy", proxy) - if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS: - self.use_socks = True + if self.url_scheme in ('http', 'https'): + config.set("http", "proxy", proxy) + if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS: + self.socks_proxy = proxy return config @@ -98,8 +98,8 @@ class GitBackend(DataBackend): } # check if using socks for proxy - if so need to use custom pool_manager - if self.use_socks: - clone_args['pool_manager'] = ProxyPoolManager(settings.HTTP_PROXIES.get(self.url_scheme)) + if self.socks_proxy: + clone_args['pool_manager'] = ProxyPoolManager(self.socks_proxy) if self.url_scheme in ('http', 'https'): if self.params.get('username'): @@ -147,7 +147,7 @@ class S3Backend(DataBackend): # Initialize backend config return Boto3Config( - proxies=settings.HTTP_PROXIES, + proxies=resolve_proxies(url=self.url, context={'client': self}), ) @contextmanager diff --git a/netbox/core/jobs.py b/netbox/core/jobs.py index 891b1cbdb..b3dfaf1e7 100644 --- a/netbox/core/jobs.py +++ b/netbox/core/jobs.py @@ -5,6 +5,7 @@ import sys from django.conf import settings from netbox.jobs import JobRunner, system_job from netbox.search.backends import search_backend +from utilities.proxy import resolve_proxies from .choices import DataSourceStatusChoices, JobIntervalChoices from .exceptions import SyncError from .models import DataSource @@ -71,7 +72,7 @@ class SystemHousekeepingJob(JobRunner): url=settings.CENSUS_URL, params=census_data, timeout=3, - proxies=settings.HTTP_PROXIES + proxies=resolve_proxies(url=settings.CENSUS_URL) ) except requests.exceptions.RequestException: pass diff --git a/netbox/core/plugins.py b/netbox/core/plugins.py index e6d09711f..d31a699e4 100644 --- a/netbox/core/plugins.py +++ b/netbox/core/plugins.py @@ -11,6 +11,7 @@ from django.core.cache import cache from netbox.plugins import PluginConfig from netbox.registry import registry from utilities.datetime import datetime_from_timestamp +from utilities.proxy import resolve_proxies USER_AGENT_STRING = f'NetBox/{settings.RELEASE.version} {settings.RELEASE.edition}' CACHE_KEY_CATALOG_FEED = 'plugins-catalog-feed' @@ -120,10 +121,11 @@ def get_catalog_plugins(): def get_pages(): # TODO: pagination is currently broken in API payload = {'page': '1', 'per_page': '50'} + proxies = resolve_proxies(url=settings.PLUGIN_CATALOG_URL) first_page = session.get( settings.PLUGIN_CATALOG_URL, headers={'User-Agent': USER_AGENT_STRING}, - proxies=settings.HTTP_PROXIES, + proxies=proxies, timeout=3, params=payload ).json() @@ -135,7 +137,7 @@ def get_catalog_plugins(): next_page = session.get( settings.PLUGIN_CATALOG_URL, headers={'User-Agent': USER_AGENT_STRING}, - proxies=settings.HTTP_PROXIES, + proxies=proxies, timeout=3, params=payload ).json() diff --git a/netbox/extras/dashboard/widgets.py b/netbox/extras/dashboard/widgets.py index eeed5414f..c04f8f423 100644 --- a/netbox/extras/dashboard/widgets.py +++ b/netbox/extras/dashboard/widgets.py @@ -17,6 +17,7 @@ from core.models import ObjectType from extras.choices import BookmarkOrderingChoices from utilities.object_types import object_type_identifier, object_type_name from utilities.permissions import get_permission_for_model +from utilities.proxy import resolve_proxies from utilities.querydict import dict_to_querydict from utilities.templatetags.builtins.filters import render_markdown from utilities.views import get_viewname @@ -330,7 +331,7 @@ class RSSFeedWidget(DashboardWidget): response = requests.get( url=self.config['feed_url'], headers={'User-Agent': f'NetBox/{settings.RELEASE.version}'}, - proxies=settings.HTTP_PROXIES, + proxies=resolve_proxies(url=self.config['feed_url'], context={'client': self}), timeout=3 ) response.raise_for_status() diff --git a/netbox/extras/management/commands/housekeeping.py b/netbox/extras/management/commands/housekeeping.py index ade486fc0..ade20a118 100644 --- a/netbox/extras/management/commands/housekeeping.py +++ b/netbox/extras/management/commands/housekeeping.py @@ -11,6 +11,7 @@ from packaging import version from core.models import Job, ObjectChange from netbox.config import Config +from utilities.proxy import resolve_proxies class Command(BaseCommand): @@ -107,7 +108,7 @@ class Command(BaseCommand): response = requests.get( url=settings.RELEASE_CHECK_URL, headers=headers, - proxies=settings.HTTP_PROXIES + proxies=resolve_proxies(url=settings.RELEASE_CHECK_URL) ) response.raise_for_status() diff --git a/netbox/extras/webhooks.py b/netbox/extras/webhooks.py index 889c97ac2..368075217 100644 --- a/netbox/extras/webhooks.py +++ b/netbox/extras/webhooks.py @@ -3,10 +3,10 @@ import hmac import logging import requests -from django.conf import settings from django_rq import job from jinja2.exceptions import TemplateError +from utilities.proxy import resolve_proxies from .constants import WEBHOOK_EVENT_TYPES logger = logging.getLogger('netbox.webhooks') @@ -63,9 +63,10 @@ def send_webhook(event_rule, model_name, event_type, data, timestamp, username, raise e # Prepare the HTTP request + url = webhook.render_payload_url(context) params = { 'method': webhook.http_method, - 'url': webhook.render_payload_url(context), + 'url': url, 'headers': headers, 'data': body.encode('utf8'), } @@ -88,7 +89,8 @@ def send_webhook(event_rule, model_name, event_type, data, timestamp, username, session.verify = webhook.ssl_verification if webhook.ca_file_path: session.verify = webhook.ca_file_path - response = session.send(prepared_request, proxies=settings.HTTP_PROXIES) + proxies = resolve_proxies(url=url, context={'client': webhook}) + response = session.send(prepared_request, proxies=proxies) if 200 <= response.status_code <= 299: logger.info(f"Request succeeded; response status {response.status_code}") diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index a17bb7730..0ad46b21e 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -9,6 +9,7 @@ import warnings from django.contrib.messages import constants as messages from django.core.exceptions import ImproperlyConfigured, ValidationError from django.core.validators import URLValidator +from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ from netbox.config import PARAMS as CONFIG_PARAMS @@ -116,7 +117,7 @@ EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', []) FIELD_CHOICES = getattr(configuration, 'FIELD_CHOICES', {}) FILE_UPLOAD_MAX_MEMORY_SIZE = getattr(configuration, 'FILE_UPLOAD_MAX_MEMORY_SIZE', 2621440) GRAPHQL_MAX_ALIASES = getattr(configuration, 'GRAPHQL_MAX_ALIASES', 10) -HTTP_PROXIES = getattr(configuration, 'HTTP_PROXIES', None) +HTTP_PROXIES = getattr(configuration, 'HTTP_PROXIES', {}) INTERNAL_IPS = getattr(configuration, 'INTERNAL_IPS', ('127.0.0.1', '::1')) ISOLATED_DEPLOYMENT = getattr(configuration, 'ISOLATED_DEPLOYMENT', False) JINJA2_FILTERS = getattr(configuration, 'JINJA2_FILTERS', {}) @@ -131,6 +132,7 @@ MEDIA_ROOT = getattr(configuration, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media' METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False) PLUGINS = getattr(configuration, 'PLUGINS', []) PLUGINS_CONFIG = getattr(configuration, 'PLUGINS_CONFIG', {}) +PROXY_ROUTERS = getattr(configuration, 'PROXY_ROUTERS', ['utilities.proxy.DefaultProxyRouter']) QUEUE_MAPPINGS = getattr(configuration, 'QUEUE_MAPPINGS', {}) REDIS = getattr(configuration, 'REDIS') # Required RELEASE_CHECK_URL = getattr(configuration, 'RELEASE_CHECK_URL', None) @@ -201,6 +203,14 @@ if RELEASE_CHECK_URL: "RELEASE_CHECK_URL must be a valid URL. Example: https://api.github.com/repos/netbox-community/netbox" ) +# Validate configured proxy routers +for path in PROXY_ROUTERS: + if type(path) is str: + try: + import_string(path) + except ImportError: + raise ImproperlyConfigured(f"Invalid path in PROXY_ROUTERS: {path}") + # # Database @@ -577,6 +587,7 @@ if SENTRY_ENABLED: sample_rate=SENTRY_SAMPLE_RATE, traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE, send_default_pii=SENTRY_SEND_DEFAULT_PII, + # TODO: Support proxy routing http_proxy=HTTP_PROXIES.get('http') if HTTP_PROXIES else None, https_proxy=HTTP_PROXIES.get('https') if HTTP_PROXIES else None ) diff --git a/netbox/utilities/proxy.py b/netbox/utilities/proxy.py new file mode 100644 index 000000000..8c9e3d196 --- /dev/null +++ b/netbox/utilities/proxy.py @@ -0,0 +1,55 @@ +from django.conf import settings +from django.utils.module_loading import import_string +from urllib.parse import urlparse + +__all__ = ( + 'DefaultProxyRouter', + 'resolve_proxies', +) + + +class DefaultProxyRouter: + """ + Base class for a proxy router. + """ + @staticmethod + def _get_protocol_from_url(url): + """ + Determine the applicable protocol (e.g. HTTP or HTTPS) from the given URL. + """ + return urlparse(url).scheme + + def route(self, url=None, protocol=None, context=None): + """ + Returns the appropriate proxy given a URL or protocol. Arbitrary context data may also be passed where + available. + + Args: + url: The specific request URL for which the proxy will be used (if known) + protocol: The protocol in use (e.g. http or https) (if known) + context: Additional context to aid in proxy selection. May include e.g. the requesting client. + """ + if url and protocol is None: + protocol = self._get_protocol_from_url(url) + if protocol and protocol in settings.HTTP_PROXIES: + return { + protocol: settings.HTTP_PROXIES[protocol] + } + return settings.HTTP_PROXIES + + +def resolve_proxies(url=None, protocol=None, context=None): + """ + Return a dictionary of candidate proxies (compatible with the requests module), or None. + + Args: + url: The specific request URL for which the proxy will be used (optional) + protocol: The protocol in use (e.g. http or https) (optional) + context: Arbitrary additional context to aid in proxy selection (optional) + """ + context = context or {} + + for item in settings.PROXY_ROUTERS: + router = import_string(item) if type(item) is str else item + if proxies := router().route(url=url, protocol=protocol, context=context): + return proxies