Introduce proxy routing

netbox/core/data_backends.py

@@ -7,13 +7,13 @@ from pathlib import Path
 from urllib.parse import urlparse
 
 from django import forms
-from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.utils.translation import gettext as _
 
 from netbox.data_backends import DataBackend
 from netbox.utils import register_data_backend
 from utilities.constants import HTTP_PROXY_SUPPORTED_SCHEMAS, HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS
+from utilities.proxy import resolve_proxies
 from utilities.socks import ProxyPoolManager
 from .exceptions import SyncError
 
@@ -70,18 +70,18 @@ class GitBackend(DataBackend):
 
         # Initialize backend config
         config = ConfigDict()
-        self.use_socks = False
+        self.socks_proxy = None
 
         # Apply HTTP proxy (if configured)
-        if settings.HTTP_PROXIES:
+        proxies = resolve_proxies(url=self.url, context={'client': self}) or {}
-            if proxy := settings.HTTP_PROXIES.get(self.url_scheme, None):
+        if proxy := proxies.get(self.url_scheme):
-                if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS:
+            if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS:
-                    raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}")
+                raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}")
 
-                if self.url_scheme in ('http', 'https'):
+            if self.url_scheme in ('http', 'https'):
-                    config.set("http", "proxy", proxy)
+                config.set("http", "proxy", proxy)
-                    if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS:
+                if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS:
-                        self.use_socks = True
+                    self.socks_proxy = proxy
 
         return config
 
@@ -98,8 +98,8 @@ class GitBackend(DataBackend):
         }
 
         # check if using socks for proxy - if so need to use custom pool_manager
-        if self.use_socks:
+        if self.socks_proxy:
-            clone_args['pool_manager'] = ProxyPoolManager(settings.HTTP_PROXIES.get(self.url_scheme))
+            clone_args['pool_manager'] = ProxyPoolManager(self.socks_proxy)
 
         if self.url_scheme in ('http', 'https'):
             if self.params.get('username'):
@@ -147,7 +147,7 @@ class S3Backend(DataBackend):
 
         # Initialize backend config
         return Boto3Config(
-            proxies=settings.HTTP_PROXIES,
+            proxies=resolve_proxies(url=self.url, context={'client': self}),
         )
 
     @contextmanager

netbox/core/jobs.py

@@ -5,6 +5,7 @@ import sys
 from django.conf import settings
 from netbox.jobs import JobRunner, system_job
 from netbox.search.backends import search_backend
+from utilities.proxy import resolve_proxies
 from .choices import DataSourceStatusChoices, JobIntervalChoices
 from .exceptions import SyncError
 from .models import DataSource
@@ -71,7 +72,7 @@ class SystemHousekeepingJob(JobRunner):
                 url=settings.CENSUS_URL,
                 params=census_data,
                 timeout=3,
-                proxies=settings.HTTP_PROXIES
+                proxies=resolve_proxies(url=settings.CENSUS_URL)
             )
         except requests.exceptions.RequestException:
             pass

netbox/core/plugins.py

@@ -11,6 +11,7 @@ from django.core.cache import cache
 from netbox.plugins import PluginConfig
 from netbox.registry import registry
 from utilities.datetime import datetime_from_timestamp
+from utilities.proxy import resolve_proxies
 
 USER_AGENT_STRING = f'NetBox/{settings.RELEASE.version} {settings.RELEASE.edition}'
 CACHE_KEY_CATALOG_FEED = 'plugins-catalog-feed'
@@ -120,10 +121,11 @@ def get_catalog_plugins():
     def get_pages():
         # TODO: pagination is currently broken in API
         payload = {'page': '1', 'per_page': '50'}
+        proxies = resolve_proxies(url=settings.PLUGIN_CATALOG_URL)
         first_page = session.get(
             settings.PLUGIN_CATALOG_URL,
             headers={'User-Agent': USER_AGENT_STRING},
-            proxies=settings.HTTP_PROXIES,
+            proxies=proxies,
             timeout=3,
             params=payload
         ).json()
@@ -135,7 +137,7 @@ def get_catalog_plugins():
             next_page = session.get(
                 settings.PLUGIN_CATALOG_URL,
                 headers={'User-Agent': USER_AGENT_STRING},
-                proxies=settings.HTTP_PROXIES,
+                proxies=proxies,
                 timeout=3,
                 params=payload
             ).json()

netbox/extras/dashboard/widgets.py

@@ -17,6 +17,7 @@ from core.models import ObjectType
 from extras.choices import BookmarkOrderingChoices
 from utilities.object_types import object_type_identifier, object_type_name
 from utilities.permissions import get_permission_for_model
+from utilities.proxy import resolve_proxies
 from utilities.querydict import dict_to_querydict
 from utilities.templatetags.builtins.filters import render_markdown
 from utilities.views import get_viewname
@@ -330,7 +331,7 @@ class RSSFeedWidget(DashboardWidget):
             response = requests.get(
                 url=self.config['feed_url'],
                 headers={'User-Agent': f'NetBox/{settings.RELEASE.version}'},
-                proxies=settings.HTTP_PROXIES,
+                proxies=resolve_proxies(url=self.config['feed_url']),
                 timeout=3
             )
             response.raise_for_status()

netbox/extras/management/commands/housekeeping.py

@@ -11,6 +11,7 @@ from packaging import version
 
 from core.models import Job, ObjectChange
 from netbox.config import Config
+from utilities.proxy import resolve_proxies
 
 
 class Command(BaseCommand):
@@ -107,7 +108,7 @@ class Command(BaseCommand):
                 response = requests.get(
                     url=settings.RELEASE_CHECK_URL,
                     headers=headers,
-                    proxies=settings.HTTP_PROXIES
+                    proxies=resolve_proxies(url=settings.RELEASE_CHECK_URL)
                 )
                 response.raise_for_status()
 

netbox/extras/webhooks.py

@@ -3,10 +3,10 @@ import hmac
 import logging
 
 import requests
-from django.conf import settings
 from django_rq import job
 from jinja2.exceptions import TemplateError
 
+from utilities.proxy import resolve_proxies
 from .constants import WEBHOOK_EVENT_TYPES
 
 logger = logging.getLogger('netbox.webhooks')
@@ -63,9 +63,10 @@ def send_webhook(event_rule, model_name, event_type, data, timestamp, username,
         raise e
 
     # Prepare the HTTP request
+    url = webhook.render_payload_url(context)
     params = {
         'method': webhook.http_method,
-        'url': webhook.render_payload_url(context),
+        'url': url,
         'headers': headers,
         'data': body.encode('utf8'),
     }
@@ -88,7 +89,8 @@ def send_webhook(event_rule, model_name, event_type, data, timestamp, username,
         session.verify = webhook.ssl_verification
         if webhook.ca_file_path:
             session.verify = webhook.ca_file_path
-        response = session.send(prepared_request, proxies=settings.HTTP_PROXIES)
+        proxies = resolve_proxies(url=url, context={'webhook': webhook})
+        response = session.send(prepared_request, proxies=proxies)
 
     if 200 <= response.status_code <= 299:
         logger.info(f"Request succeeded; response status {response.status_code}")

netbox/netbox/settings.py

@@ -9,6 +9,7 @@ import warnings
 from django.contrib.messages import constants as messages
 from django.core.exceptions import ImproperlyConfigured, ValidationError
 from django.core.validators import URLValidator
+from django.utils.module_loading import import_string
 from django.utils.translation import gettext_lazy as _
 
 from netbox.config import PARAMS as CONFIG_PARAMS
@@ -131,6 +132,7 @@ MEDIA_ROOT = getattr(configuration, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media'
 METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False)
 PLUGINS = getattr(configuration, 'PLUGINS', [])
 PLUGINS_CONFIG = getattr(configuration, 'PLUGINS_CONFIG', {})
+PROXY_ROUTERS = getattr(configuration, 'PROXY_ROUTERS', ['utilities.proxy.DefaultProxyRouter'])
 QUEUE_MAPPINGS = getattr(configuration, 'QUEUE_MAPPINGS', {})
 REDIS = getattr(configuration, 'REDIS')  # Required
 RELEASE_CHECK_URL = getattr(configuration, 'RELEASE_CHECK_URL', None)
@@ -201,6 +203,14 @@ if RELEASE_CHECK_URL:
             "RELEASE_CHECK_URL must be a valid URL. Example: https://api.github.com/repos/netbox-community/netbox"
         )
 
+# Validate configured proxy routers
+for path in PROXY_ROUTERS:
+    if type(path) is str:
+        try:
+            import_string(path)
+        except ImportError:
+            raise ImproperlyConfigured(f"Invalid proxy router path: {path}")
+
 
 #
 # Database
@@ -577,6 +587,7 @@ if SENTRY_ENABLED:
         sample_rate=SENTRY_SAMPLE_RATE,
         traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE,
         send_default_pii=SENTRY_SEND_DEFAULT_PII,
+        # TODO: Support proxy routing
         http_proxy=HTTP_PROXIES.get('http') if HTTP_PROXIES else None,
         https_proxy=HTTP_PROXIES.get('https') if HTTP_PROXIES else None
     )

netbox/utilities/proxy.py

@@ -0,0 +1,46 @@
+from django.conf import settings
+from django.utils.module_loading import import_string
+from urllib.parse import urlparse
+
+__all__ = (
+    'DefaultProxyRouter',
+    'resolve_proxies',
+)
+
+
+class DefaultProxyRouter:
+    """
+    Base class for a proxy router.
+    """
+    @staticmethod
+    def _get_protocol_from_url(url):
+        """
+        Determine the applicable protocol (e.g. HTTP or HTTPS) from the given URL.
+        """
+        return urlparse(url).scheme
+
+    def route(self, url=None, protocol=None, context=None):
+        if url and protocol is None:
+            protocol = self._get_protocol_from_url(url)
+        if protocol and protocol in settings.HTTP_PROXIES:
+            return {
+                protocol: settings.HTTP_PROXIES[protocol]
+            }
+        return settings.HTTP_PROXIES
+
+
+def resolve_proxies(url=None, protocol=None, context=None):
+    """
+    Return a dictionary of candidate proxies (compatible with the requests module), or None.
+
+    Args:
+        url: The specific request URL for which the proxy will be used (optional)
+        protocol: The protocol in use (e.g. http or https) (optional)
+        context: Arbitrary additional context to aid in proxy selection (optional)
+    """
+    context = context or {}
+
+    for item in settings.PROXY_ROUTERS:
+        router = import_string(item) if type(item) is str else item
+        if proxies := router.route(url=url, protocol=protocol, context=context):
+            return proxies