From 92d8aa583a020f85140665583b6e9f71a5d90029 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Date: Tue, 1 Oct 2024 09:11:44 -0700 Subject: [PATCH] Add support for socks connection to Git backend (#17640) * Add support for socks connection to Git backend * cleanup socks detection * add documentation for installing python_socks * dont need lower() * cleanup * refactor Socks to utilities * fix imports * fix missing comma * Update docs/features/synchronized-data.md Co-authored-by: Jeremy Stretch * review feedback * Update docs/features/synchronized-data.md Co-authored-by: Jeremy Stretch * review changes --------- Co-authored-by: Jeremy Stretch --- docs/features/synchronized-data.md | 3 + netbox/core/data_backends.py | 20 +++++- netbox/utilities/constants.py | 4 ++ netbox/utilities/socks.py | 101 +++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 netbox/utilities/socks.py diff --git a/docs/features/synchronized-data.md b/docs/features/synchronized-data.md index 8c95c8779..23c79feed 100644 --- a/docs/features/synchronized-data.md +++ b/docs/features/synchronized-data.md @@ -13,6 +13,9 @@ To enable remote data synchronization, the NetBox administrator first designates !!! info Data backends which connect to external sources typically require the installation of one or more supporting Python libraries. The Git backend requires the [`dulwich`](https://www.dulwich.io/) package, and the S3 backend requires the [`boto3`](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) package. These must be installed within NetBox's environment to enable these backends. +!!! info + If you are configuring Git and have `HTTP_PROXIES` configured to use the SOCKS protocol, you will also need to install the [`python_socks`](https://pypi.org/project/python-socks/) Python library. + Each type of remote source has its own configuration parameters. For instance, a git source will ask the user to specify a branch and authentication credentials. Once the source has been created, a synchronization job is run to automatically replicate remote files in the local database. The following NetBox models can be associated with replicated data files: diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index 1b64f5f5c..8b36c6995 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -8,10 +8,13 @@ from urllib.parse import urlparse from django import forms from django.conf import settings +from django.core.exceptions import ImproperlyConfigured from django.utils.translation import gettext as _ from netbox.data_backends import DataBackend from netbox.utils import register_data_backend +from utilities.constants import HTTP_PROXY_SUPPORTED_SCHEMAS, HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS +from utilities.socks import ProxyPoolManager from .exceptions import SyncError __all__ = ( @@ -67,11 +70,18 @@ class GitBackend(DataBackend): # Initialize backend config config = ConfigDict() + self.use_socks = False # Apply HTTP proxy (if configured) - if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'): - if proxy := settings.HTTP_PROXIES.get(self.url_scheme): - config.set("http", "proxy", proxy) + if settings.HTTP_PROXIES: + if proxy := settings.HTTP_PROXIES.get(self.url_scheme, None): + if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS: + raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}") + + if self.url_scheme in ('http', 'https'): + config.set("http", "proxy", proxy) + if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS: + self.use_socks = True return config @@ -87,6 +97,10 @@ class GitBackend(DataBackend): "errstream": porcelain.NoneStream(), } + # check if using socks for proxy - if so need to use custom pool_manager + if self.use_socks: + clone_args['pool_manager'] = ProxyPoolManager(settings.HTTP_PROXIES.get(self.url_scheme)) + if self.url_scheme in ('http', 'https'): if self.params.get('username'): clone_args.update( diff --git a/netbox/utilities/constants.py b/netbox/utilities/constants.py index c7c26f6b3..2b93f2b96 100644 --- a/netbox/utilities/constants.py +++ b/netbox/utilities/constants.py @@ -93,3 +93,7 @@ HTML_ALLOWED_ATTRIBUTES = { "td": {"align"}, "th": {"align"}, } + +HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS = ['socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h'] +HTTP_PROXY_SOCK_RDNS_SCHEMAS = ['socks4h', 'socks4a', 'socks5h', 'socks5a'] +HTTP_PROXY_SUPPORTED_SCHEMAS = ['http', 'https', 'socks4', 'socks4a', 'socks4h', 'socks5', 'socks5a', 'socks5h'] diff --git a/netbox/utilities/socks.py b/netbox/utilities/socks.py new file mode 100644 index 000000000..bb0b6b250 --- /dev/null +++ b/netbox/utilities/socks.py @@ -0,0 +1,101 @@ +import logging + +from urllib.parse import urlparse +from urllib3 import PoolManager, HTTPConnectionPool, HTTPSConnectionPool +from urllib3.connection import HTTPConnection, HTTPSConnection +from .constants import HTTP_PROXY_SOCK_RDNS_SCHEMAS + + +logger = logging.getLogger('netbox.utilities') + + +class ProxyHTTPConnection(HTTPConnection): + """ + A Proxy connection class that uses a SOCK proxy - used to create + a urllib3 PoolManager that routes connections via the proxy. + This is for an HTTP (not HTTPS) connection + """ + use_rdns = False + + def __init__(self, *args, **kwargs): + socks_options = kwargs.pop('_socks_options') + self._proxy_url = socks_options['proxy_url'] + super().__init__(*args, **kwargs) + + def _new_conn(self): + try: + from python_socks.sync import Proxy + except ModuleNotFoundError as e: + logger.info("Configuring an HTTP proxy using SOCKS requires the python_socks library. Check that it has been installed.") + raise e + + proxy = Proxy.from_url(self._proxy_url, rdns=self.use_rdns) + return proxy.connect( + dest_host=self.host, + dest_port=self.port, + timeout=self.timeout + ) + + +class ProxyHTTPSConnection(ProxyHTTPConnection, HTTPSConnection): + """ + A Proxy connection class for an HTTPS (not HTTP) connection. + """ + pass + + +class RdnsProxyHTTPConnection(ProxyHTTPConnection): + """ + A Proxy connection class for an HTTP remote-dns connection. + I.E. socks4a, socks4h, socks5a, socks5h + """ + use_rdns = True + + +class RdnsProxyHTTPSConnection(ProxyHTTPSConnection): + """ + A Proxy connection class for an HTTPS remote-dns connection. + I.E. socks4a, socks4h, socks5a, socks5h + """ + use_rdns = True + + +class ProxyHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = ProxyHTTPConnection + + +class ProxyHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = ProxyHTTPSConnection + + +class RdnsProxyHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = RdnsProxyHTTPConnection + + +class RdnsProxyHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = RdnsProxyHTTPSConnection + + +class ProxyPoolManager(PoolManager): + def __init__(self, proxy_url, timeout=5, num_pools=10, headers=None, **connection_pool_kw): + # python_socks uses rdns param to denote remote DNS parsing and + # doesn't accept the 'h' or 'a' in the proxy URL + if use_rdns := urlparse(proxy_url).scheme in HTTP_PROXY_SOCK_RDNS_SCHEMAS: + proxy_url = proxy_url.replace('socks5h:', 'socks5:').replace('socks5a:', 'socks5:') + proxy_url = proxy_url.replace('socks4h:', 'socks4:').replace('socks4a:', 'socks4:') + + connection_pool_kw['_socks_options'] = {'proxy_url': proxy_url} + connection_pool_kw['timeout'] = timeout + + super().__init__(num_pools, headers, **connection_pool_kw) + + if use_rdns: + self.pool_classes_by_scheme = { + 'http': RdnsProxyHTTPConnectionPool, + 'https': RdnsProxyHTTPSConnectionPool, + } + else: + self.pool_classes_by_scheme = { + 'http': ProxyHTTPConnectionPool, + 'https': ProxyHTTPSConnectionPool, + }