diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index 770a3b258..9ba1d5dfd 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -7,13 +7,13 @@ from pathlib import Path from urllib.parse import urlparse from django import forms -from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.utils.translation import gettext as _ from netbox.data_backends import DataBackend from netbox.utils import register_data_backend from utilities.constants import HTTP_PROXY_SUPPORTED_SCHEMAS, HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS +from utilities.proxy import resolve_proxies from utilities.socks import ProxyPoolManager from .exceptions import SyncError @@ -70,18 +70,18 @@ class GitBackend(DataBackend): # Initialize backend config config = ConfigDict() - self.use_socks = False + self.socks_proxy = None # Apply HTTP proxy (if configured) - if settings.HTTP_PROXIES: - if proxy := settings.HTTP_PROXIES.get(self.url_scheme, None): - if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS: - raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}") + proxies = resolve_proxies(url=self.url, context={'client': self}) or {} + if proxy := proxies.get(self.url_scheme): + if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS: + raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}") - if self.url_scheme in ('http', 'https'): - config.set("http", "proxy", proxy) - if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS: - self.use_socks = True + if self.url_scheme in ('http', 'https'): + config.set("http", "proxy", proxy) + if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS: + self.socks_proxy = proxy return config @@ -98,8 +98,8 @@ class GitBackend(DataBackend): } # check if using socks for proxy - if so need to use custom pool_manager - if self.use_socks: - clone_args['pool_manager'] = ProxyPoolManager(settings.HTTP_PROXIES.get(self.url_scheme)) + if self.socks_proxy: + clone_args['pool_manager'] = ProxyPoolManager(self.socks_proxy) if self.url_scheme in ('http', 'https'): if self.params.get('username'): @@ -147,7 +147,7 @@ class S3Backend(DataBackend): # Initialize backend config return Boto3Config( - proxies=settings.HTTP_PROXIES, + proxies=resolve_proxies(url=self.url, context={'client': self}), ) @contextmanager diff --git a/netbox/core/jobs.py b/netbox/core/jobs.py index 891b1cbdb..b3dfaf1e7 100644 --- a/netbox/core/jobs.py +++ b/netbox/core/jobs.py @@ -5,6 +5,7 @@ import sys from django.conf import settings from netbox.jobs import JobRunner, system_job from netbox.search.backends import search_backend +from utilities.proxy import resolve_proxies from .choices import DataSourceStatusChoices, JobIntervalChoices from .exceptions import SyncError from .models import DataSource @@ -71,7 +72,7 @@ class SystemHousekeepingJob(JobRunner): url=settings.CENSUS_URL, params=census_data, timeout=3, - proxies=settings.HTTP_PROXIES + proxies=resolve_proxies(url=settings.CENSUS_URL) ) except requests.exceptions.RequestException: pass diff --git a/netbox/core/plugins.py b/netbox/core/plugins.py index e6d09711f..d31a699e4 100644 --- a/netbox/core/plugins.py +++ b/netbox/core/plugins.py @@ -11,6 +11,7 @@ from django.core.cache import cache from netbox.plugins import PluginConfig from netbox.registry import registry from utilities.datetime import datetime_from_timestamp +from utilities.proxy import resolve_proxies USER_AGENT_STRING = f'NetBox/{settings.RELEASE.version} {settings.RELEASE.edition}' CACHE_KEY_CATALOG_FEED = 'plugins-catalog-feed' @@ -120,10 +121,11 @@ def get_catalog_plugins(): def get_pages(): # TODO: pagination is currently broken in API payload = {'page': '1', 'per_page': '50'} + proxies = resolve_proxies(url=settings.PLUGIN_CATALOG_URL) first_page = session.get( settings.PLUGIN_CATALOG_URL, headers={'User-Agent': USER_AGENT_STRING}, - proxies=settings.HTTP_PROXIES, + proxies=proxies, timeout=3, params=payload ).json() @@ -135,7 +137,7 @@ def get_catalog_plugins(): next_page = session.get( settings.PLUGIN_CATALOG_URL, headers={'User-Agent': USER_AGENT_STRING}, - proxies=settings.HTTP_PROXIES, + proxies=proxies, timeout=3, params=payload ).json() diff --git a/netbox/extras/dashboard/widgets.py b/netbox/extras/dashboard/widgets.py index eeed5414f..54e6f5602 100644 --- a/netbox/extras/dashboard/widgets.py +++ b/netbox/extras/dashboard/widgets.py @@ -17,6 +17,7 @@ from core.models import ObjectType from extras.choices import BookmarkOrderingChoices from utilities.object_types import object_type_identifier, object_type_name from utilities.permissions import get_permission_for_model +from utilities.proxy import resolve_proxies from utilities.querydict import dict_to_querydict from utilities.templatetags.builtins.filters import render_markdown from utilities.views import get_viewname @@ -330,7 +331,7 @@ class RSSFeedWidget(DashboardWidget): response = requests.get( url=self.config['feed_url'], headers={'User-Agent': f'NetBox/{settings.RELEASE.version}'}, - proxies=settings.HTTP_PROXIES, + proxies=resolve_proxies(url=self.config['feed_url']), timeout=3 ) response.raise_for_status() diff --git a/netbox/extras/management/commands/housekeeping.py b/netbox/extras/management/commands/housekeeping.py index ade486fc0..ade20a118 100644 --- a/netbox/extras/management/commands/housekeeping.py +++ b/netbox/extras/management/commands/housekeeping.py @@ -11,6 +11,7 @@ from packaging import version from core.models import Job, ObjectChange from netbox.config import Config +from utilities.proxy import resolve_proxies class Command(BaseCommand): @@ -107,7 +108,7 @@ class Command(BaseCommand): response = requests.get( url=settings.RELEASE_CHECK_URL, headers=headers, - proxies=settings.HTTP_PROXIES + proxies=resolve_proxies(url=settings.RELEASE_CHECK_URL) ) response.raise_for_status() diff --git a/netbox/extras/webhooks.py b/netbox/extras/webhooks.py index 889c97ac2..7b1b7a55a 100644 --- a/netbox/extras/webhooks.py +++ b/netbox/extras/webhooks.py @@ -3,10 +3,10 @@ import hmac import logging import requests -from django.conf import settings from django_rq import job from jinja2.exceptions import TemplateError +from utilities.proxy import resolve_proxies from .constants import WEBHOOK_EVENT_TYPES logger = logging.getLogger('netbox.webhooks') @@ -63,9 +63,10 @@ def send_webhook(event_rule, model_name, event_type, data, timestamp, username, raise e # Prepare the HTTP request + url = webhook.render_payload_url(context) params = { 'method': webhook.http_method, - 'url': webhook.render_payload_url(context), + 'url': url, 'headers': headers, 'data': body.encode('utf8'), } @@ -88,7 +89,8 @@ def send_webhook(event_rule, model_name, event_type, data, timestamp, username, session.verify = webhook.ssl_verification if webhook.ca_file_path: session.verify = webhook.ca_file_path - response = session.send(prepared_request, proxies=settings.HTTP_PROXIES) + proxies = resolve_proxies(url=url, context={'webhook': webhook}) + response = session.send(prepared_request, proxies=proxies) if 200 <= response.status_code <= 299: logger.info(f"Request succeeded; response status {response.status_code}") diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index a17bb7730..748a1edf2 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -9,6 +9,7 @@ import warnings from django.contrib.messages import constants as messages from django.core.exceptions import ImproperlyConfigured, ValidationError from django.core.validators import URLValidator +from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ from netbox.config import PARAMS as CONFIG_PARAMS @@ -131,6 +132,7 @@ MEDIA_ROOT = getattr(configuration, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media' METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False) PLUGINS = getattr(configuration, 'PLUGINS', []) PLUGINS_CONFIG = getattr(configuration, 'PLUGINS_CONFIG', {}) +PROXY_ROUTERS = getattr(configuration, 'PROXY_ROUTERS', ['utilities.proxy.DefaultProxyRouter']) QUEUE_MAPPINGS = getattr(configuration, 'QUEUE_MAPPINGS', {}) REDIS = getattr(configuration, 'REDIS') # Required RELEASE_CHECK_URL = getattr(configuration, 'RELEASE_CHECK_URL', None) @@ -201,6 +203,14 @@ if RELEASE_CHECK_URL: "RELEASE_CHECK_URL must be a valid URL. Example: https://api.github.com/repos/netbox-community/netbox" ) +# Validate configured proxy routers +for path in PROXY_ROUTERS: + if type(path) is str: + try: + import_string(path) + except ImportError: + raise ImproperlyConfigured(f"Invalid proxy router path: {path}") + # # Database @@ -577,6 +587,7 @@ if SENTRY_ENABLED: sample_rate=SENTRY_SAMPLE_RATE, traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE, send_default_pii=SENTRY_SEND_DEFAULT_PII, + # TODO: Support proxy routing http_proxy=HTTP_PROXIES.get('http') if HTTP_PROXIES else None, https_proxy=HTTP_PROXIES.get('https') if HTTP_PROXIES else None ) diff --git a/netbox/utilities/proxy.py b/netbox/utilities/proxy.py new file mode 100644 index 000000000..85e942a65 --- /dev/null +++ b/netbox/utilities/proxy.py @@ -0,0 +1,46 @@ +from django.conf import settings +from django.utils.module_loading import import_string +from urllib.parse import urlparse + +__all__ = ( + 'DefaultProxyRouter', + 'resolve_proxies', +) + + +class DefaultProxyRouter: + """ + Base class for a proxy router. + """ + @staticmethod + def _get_protocol_from_url(url): + """ + Determine the applicable protocol (e.g. HTTP or HTTPS) from the given URL. + """ + return urlparse(url).scheme + + def route(self, url=None, protocol=None, context=None): + if url and protocol is None: + protocol = self._get_protocol_from_url(url) + if protocol and protocol in settings.HTTP_PROXIES: + return { + protocol: settings.HTTP_PROXIES[protocol] + } + return settings.HTTP_PROXIES + + +def resolve_proxies(url=None, protocol=None, context=None): + """ + Return a dictionary of candidate proxies (compatible with the requests module), or None. + + Args: + url: The specific request URL for which the proxy will be used (optional) + protocol: The protocol in use (e.g. http or https) (optional) + context: Arbitrary additional context to aid in proxy selection (optional) + """ + context = context or {} + + for item in settings.PROXY_ROUTERS: + router = import_string(item) if type(item) is str else item + if proxies := router.route(url=url, protocol=protocol, context=context): + return proxies