Closes #18627: Proxy routing (#18681)

* Introduce proxy routing

* Misc cleanup

* Document PROXY_ROUTERS parameter

docs/configuration/system.md

@@ -64,7 +64,7 @@ Email is sent from NetBox only for critical events or if configured for [logging
 
 ## HTTP_PROXIES
 
-Default: None
+Default: Empty
 
 A dictionary of HTTP proxies to use for outbound requests originating from NetBox (e.g. when sending webhook requests). Proxies should be specified by schema (HTTP and HTTPS) as per the [Python requests library documentation](https://requests.readthedocs.io/en/latest/user/advanced/#proxies). For example:
 
@@ -75,6 +75,8 @@ HTTP_PROXIES = {
 }
 ```
 
+If more flexibility is needed in determining which proxy to use for a given request, consider implementing one or more custom proxy routers via the [`PROXY_ROUTERS`](#proxy_routers) parameter.
+
 ---
 
 ## INTERNAL_IPS
@@ -160,6 +162,16 @@ The file path to the location where media files (such as image attachments) are
 
 ---
 
+## PROXY_ROUTERS
+
+Default: `["utilities.proxy.DefaultProxyRouter"]`
+
+A list of Python classes responsible for determining which proxy server(s) to use for outbound HTTP requests. Each item in the list can be the class itself or the dotted path to the class.
+
+The `route()` method on each class must return a dictionary of candidate proxies arranged by protocol (e.g. `http` and/or `https`), or None if no viable proxy can be determined. The default class, `DefaultProxyRouter`, simply returns the content of [`HTTP_PROXIES`](#http_proxies).
+
+---
+
 ## REPORTS_ROOT
 
 Default: `$INSTALL_ROOT/netbox/reports/`

netbox/core/data_backends.py

@@ -7,13 +7,13 @@ from pathlib import Path
 from urllib.parse import urlparse
 
 from django import forms
-from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.utils.translation import gettext as _
 
 from netbox.data_backends import DataBackend
 from netbox.utils import register_data_backend
 from utilities.constants import HTTP_PROXY_SUPPORTED_SCHEMAS, HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS
+from utilities.proxy import resolve_proxies
 from utilities.socks import ProxyPoolManager
 from .exceptions import SyncError
 
@@ -70,18 +70,18 @@ class GitBackend(DataBackend):
 
         # Initialize backend config
         config = ConfigDict()
-        self.use_socks = False
+        self.socks_proxy = None
 
         # Apply HTTP proxy (if configured)
-        if settings.HTTP_PROXIES:
+        proxies = resolve_proxies(url=self.url, context={'client': self}) or {}
-            if proxy := settings.HTTP_PROXIES.get(self.url_scheme, None):
+        if proxy := proxies.get(self.url_scheme):
-                if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS:
+            if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS:
-                    raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}")
+                raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}")
 
-                if self.url_scheme in ('http', 'https'):
+            if self.url_scheme in ('http', 'https'):
-                    config.set("http", "proxy", proxy)
+                config.set("http", "proxy", proxy)
-                    if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS:
+                if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS:
-                        self.use_socks = True
+                    self.socks_proxy = proxy
 
         return config
 
@@ -98,8 +98,8 @@ class GitBackend(DataBackend):
         }
 
         # check if using socks for proxy - if so need to use custom pool_manager
-        if self.use_socks:
+        if self.socks_proxy:
-            clone_args['pool_manager'] = ProxyPoolManager(settings.HTTP_PROXIES.get(self.url_scheme))
+            clone_args['pool_manager'] = ProxyPoolManager(self.socks_proxy)
 
         if self.url_scheme in ('http', 'https'):
             if self.params.get('username'):
@@ -147,7 +147,7 @@ class S3Backend(DataBackend):
 
         # Initialize backend config
         return Boto3Config(
-            proxies=settings.HTTP_PROXIES,
+            proxies=resolve_proxies(url=self.url, context={'client': self}),
         )
 
     @contextmanager

netbox/core/jobs.py

@@ -5,6 +5,7 @@ import sys
 from django.conf import settings
 from netbox.jobs import JobRunner, system_job
 from netbox.search.backends import search_backend
+from utilities.proxy import resolve_proxies
 from .choices import DataSourceStatusChoices, JobIntervalChoices
 from .exceptions import SyncError
 from .models import DataSource
@@ -71,7 +72,7 @@ class SystemHousekeepingJob(JobRunner):
                 url=settings.CENSUS_URL,
                 params=census_data,
                 timeout=3,
-                proxies=settings.HTTP_PROXIES
+                proxies=resolve_proxies(url=settings.CENSUS_URL)
             )
         except requests.exceptions.RequestException:
             pass

netbox/core/plugins.py

@@ -11,6 +11,7 @@ from django.core.cache import cache
 from netbox.plugins import PluginConfig
 from netbox.registry import registry
 from utilities.datetime import datetime_from_timestamp
+from utilities.proxy import resolve_proxies
 
 USER_AGENT_STRING = f'NetBox/{settings.RELEASE.version} {settings.RELEASE.edition}'
 CACHE_KEY_CATALOG_FEED = 'plugins-catalog-feed'
@@ -120,10 +121,11 @@ def get_catalog_plugins():
     def get_pages():
         # TODO: pagination is currently broken in API
         payload = {'page': '1', 'per_page': '50'}
+        proxies = resolve_proxies(url=settings.PLUGIN_CATALOG_URL)
         first_page = session.get(
             settings.PLUGIN_CATALOG_URL,
             headers={'User-Agent': USER_AGENT_STRING},
-            proxies=settings.HTTP_PROXIES,
+            proxies=proxies,
             timeout=3,
             params=payload
         ).json()
@@ -135,7 +137,7 @@ def get_catalog_plugins():
             next_page = session.get(
                 settings.PLUGIN_CATALOG_URL,
                 headers={'User-Agent': USER_AGENT_STRING},
-                proxies=settings.HTTP_PROXIES,
+                proxies=proxies,
                 timeout=3,
                 params=payload
             ).json()

netbox/extras/dashboard/widgets.py

@@ -17,6 +17,7 @@ from core.models import ObjectType
 from extras.choices import BookmarkOrderingChoices
 from utilities.object_types import object_type_identifier, object_type_name
 from utilities.permissions import get_permission_for_model
+from utilities.proxy import resolve_proxies
 from utilities.querydict import dict_to_querydict
 from utilities.templatetags.builtins.filters import render_markdown
 from utilities.views import get_viewname
@@ -330,7 +331,7 @@ class RSSFeedWidget(DashboardWidget):
             response = requests.get(
                 url=self.config['feed_url'],
                 headers={'User-Agent': f'NetBox/{settings.RELEASE.version}'},
-                proxies=settings.HTTP_PROXIES,
+                proxies=resolve_proxies(url=self.config['feed_url'], context={'client': self}),
                 timeout=3
             )
             response.raise_for_status()

netbox/extras/management/commands/housekeeping.py

@@ -11,6 +11,7 @@ from packaging import version
 
 from core.models import Job, ObjectChange
 from netbox.config import Config
+from utilities.proxy import resolve_proxies
 
 
 class Command(BaseCommand):
@@ -107,7 +108,7 @@ class Command(BaseCommand):
                 response = requests.get(
                     url=settings.RELEASE_CHECK_URL,
                     headers=headers,
-                    proxies=settings.HTTP_PROXIES
+                    proxies=resolve_proxies(url=settings.RELEASE_CHECK_URL)
                 )
                 response.raise_for_status()
 

netbox/extras/webhooks.py

@@ -3,10 +3,10 @@ import hmac
 import logging
 
 import requests
-from django.conf import settings
 from django_rq import job
 from jinja2.exceptions import TemplateError
 
+from utilities.proxy import resolve_proxies
 from .constants import WEBHOOK_EVENT_TYPES
 
 logger = logging.getLogger('netbox.webhooks')
@@ -63,9 +63,10 @@ def send_webhook(event_rule, model_name, event_type, data, timestamp, username,
         raise e
 
     # Prepare the HTTP request
+    url = webhook.render_payload_url(context)
     params = {
         'method': webhook.http_method,
-        'url': webhook.render_payload_url(context),
+        'url': url,
         'headers': headers,
         'data': body.encode('utf8'),
     }
@@ -88,7 +89,8 @@ def send_webhook(event_rule, model_name, event_type, data, timestamp, username,
         session.verify = webhook.ssl_verification
         if webhook.ca_file_path:
             session.verify = webhook.ca_file_path
-        response = session.send(prepared_request, proxies=settings.HTTP_PROXIES)
+        proxies = resolve_proxies(url=url, context={'client': webhook})
+        response = session.send(prepared_request, proxies=proxies)
 
     if 200 <= response.status_code <= 299:
         logger.info(f"Request succeeded; response status {response.status_code}")

netbox/netbox/settings.py

@@ -9,6 +9,7 @@ import warnings
 from django.contrib.messages import constants as messages
 from django.core.exceptions import ImproperlyConfigured, ValidationError
 from django.core.validators import URLValidator
+from django.utils.module_loading import import_string
 from django.utils.translation import gettext_lazy as _
 
 from netbox.config import PARAMS as CONFIG_PARAMS
@@ -116,7 +117,7 @@ EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', [])
 FIELD_CHOICES = getattr(configuration, 'FIELD_CHOICES', {})
 FILE_UPLOAD_MAX_MEMORY_SIZE = getattr(configuration, 'FILE_UPLOAD_MAX_MEMORY_SIZE', 2621440)
 GRAPHQL_MAX_ALIASES = getattr(configuration, 'GRAPHQL_MAX_ALIASES', 10)
-HTTP_PROXIES = getattr(configuration, 'HTTP_PROXIES', None)
+HTTP_PROXIES = getattr(configuration, 'HTTP_PROXIES', {})
 INTERNAL_IPS = getattr(configuration, 'INTERNAL_IPS', ('127.0.0.1', '::1'))
 ISOLATED_DEPLOYMENT = getattr(configuration, 'ISOLATED_DEPLOYMENT', False)
 JINJA2_FILTERS = getattr(configuration, 'JINJA2_FILTERS', {})
@@ -131,6 +132,7 @@ MEDIA_ROOT = getattr(configuration, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media'
 METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False)
 PLUGINS = getattr(configuration, 'PLUGINS', [])
 PLUGINS_CONFIG = getattr(configuration, 'PLUGINS_CONFIG', {})
+PROXY_ROUTERS = getattr(configuration, 'PROXY_ROUTERS', ['utilities.proxy.DefaultProxyRouter'])
 QUEUE_MAPPINGS = getattr(configuration, 'QUEUE_MAPPINGS', {})
 REDIS = getattr(configuration, 'REDIS')  # Required
 RELEASE_CHECK_URL = getattr(configuration, 'RELEASE_CHECK_URL', None)
@@ -201,6 +203,14 @@ if RELEASE_CHECK_URL:
             "RELEASE_CHECK_URL must be a valid URL. Example: https://api.github.com/repos/netbox-community/netbox"
         )
 
+# Validate configured proxy routers
+for path in PROXY_ROUTERS:
+    if type(path) is str:
+        try:
+            import_string(path)
+        except ImportError:
+            raise ImproperlyConfigured(f"Invalid path in PROXY_ROUTERS: {path}")
+
 
 #
 # Database
@@ -577,6 +587,7 @@ if SENTRY_ENABLED:
         sample_rate=SENTRY_SAMPLE_RATE,
         traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE,
         send_default_pii=SENTRY_SEND_DEFAULT_PII,
+        # TODO: Support proxy routing
         http_proxy=HTTP_PROXIES.get('http') if HTTP_PROXIES else None,
         https_proxy=HTTP_PROXIES.get('https') if HTTP_PROXIES else None
     )

netbox/utilities/proxy.py

@@ -0,0 +1,55 @@
+from django.conf import settings
+from django.utils.module_loading import import_string
+from urllib.parse import urlparse
+
+__all__ = (
+    'DefaultProxyRouter',
+    'resolve_proxies',
+)
+
+
+class DefaultProxyRouter:
+    """
+    Base class for a proxy router.
+    """
+    @staticmethod
+    def _get_protocol_from_url(url):
+        """
+        Determine the applicable protocol (e.g. HTTP or HTTPS) from the given URL.
+        """
+        return urlparse(url).scheme
+
+    def route(self, url=None, protocol=None, context=None):
+        """
+        Returns the appropriate proxy given a URL or protocol. Arbitrary context data may also be passed where
+        available.
+
+        Args:
+            url: The specific request URL for which the proxy will be used (if known)
+            protocol: The protocol in use (e.g. http or https) (if known)
+            context: Additional context to aid in proxy selection. May include e.g. the requesting client.
+        """
+        if url and protocol is None:
+            protocol = self._get_protocol_from_url(url)
+        if protocol and protocol in settings.HTTP_PROXIES:
+            return {
+                protocol: settings.HTTP_PROXIES[protocol]
+            }
+        return settings.HTTP_PROXIES
+
+
+def resolve_proxies(url=None, protocol=None, context=None):
+    """
+    Return a dictionary of candidate proxies (compatible with the requests module), or None.
+
+    Args:
+        url: The specific request URL for which the proxy will be used (optional)
+        protocol: The protocol in use (e.g. http or https) (optional)
+        context: Arbitrary additional context to aid in proxy selection (optional)
+    """
+    context = context or {}
+
+    for item in settings.PROXY_ROUTERS:
+        router = import_string(item) if type(item) is str else item
+        if proxies := router().route(url=url, protocol=protocol, context=context):
+            return proxies