From 3538eeda14a5ba1921299f2742593f0b9e434807 Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Thu, 13 Feb 2020 08:53:46 -0500 Subject: [PATCH 1/3] allow for redis sentinel connection --- netbox/netbox/configuration.example.py | 6 +++ netbox/netbox/settings.py | 55 ++++++++++++++++++++------ 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index 885c9e0ba..7002def9b 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -28,6 +28,9 @@ REDIS = { 'webhooks': { 'HOST': 'localhost', 'PORT': 6379, + # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '', 'DATABASE': 0, 'DEFAULT_TIMEOUT': 300, @@ -36,6 +39,9 @@ REDIS = { 'caching': { 'HOST': 'localhost', 'PORT': 6379, + # Comment out `HOST` and `PORT` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '', 'DATABASE': 1, 'DEFAULT_TIMEOUT': 300, diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 305f7efa8..4b6177318 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -170,14 +170,27 @@ if 'caching' not in REDIS: WEBHOOKS_REDIS = REDIS.get('webhooks', {}) WEBHOOKS_REDIS_HOST = WEBHOOKS_REDIS.get('HOST', 'localhost') WEBHOOKS_REDIS_PORT = WEBHOOKS_REDIS.get('PORT', 6379) +WEBHOOKS_REDIS_SENTINELS = WEBHOOKS_REDIS.get('SENTINELS', []) +WEBHOOKS_REDIS_USING_SENTINEL = all([ + isinstance(WEBHOOKS_REDIS_SENTINELS, (list, tuple)), + len(WEBHOOKS_REDIS_SENTINELS) > 0 +]) +WEBHOOKS_REDIS_SENTINEL_SERVICE = WEBHOOKS_REDIS.get('SENTINEL_SERVICE', 'default') WEBHOOKS_REDIS_PASSWORD = WEBHOOKS_REDIS.get('PASSWORD', '') WEBHOOKS_REDIS_DATABASE = WEBHOOKS_REDIS.get('DATABASE', 0) WEBHOOKS_REDIS_DEFAULT_TIMEOUT = WEBHOOKS_REDIS.get('DEFAULT_TIMEOUT', 300) WEBHOOKS_REDIS_SSL = WEBHOOKS_REDIS.get('SSL', False) + CACHING_REDIS = REDIS.get('caching', {}) CACHING_REDIS_HOST = CACHING_REDIS.get('HOST', 'localhost') CACHING_REDIS_PORT = CACHING_REDIS.get('PORT', 6379) +CACHING_REDIS_SENTINELS = CACHING_REDIS.get('SENTINELS', []) +CACHING_REDIS_USING_SENTINEL = all([ + isinstance(CACHING_REDIS_SENTINELS, (list, tuple)), + len(CACHING_REDIS_SENTINELS) > 0 +]) +CACHING_REDIS_SENTINEL_SERVICE = CACHING_REDIS.get('SENTINEL_SERVICE', 'default') CACHING_REDIS_PASSWORD = CACHING_REDIS.get('PASSWORD', '') CACHING_REDIS_DATABASE = CACHING_REDIS.get('DATABASE', 0) CACHING_REDIS_DEFAULT_TIMEOUT = CACHING_REDIS.get('DEFAULT_TIMEOUT', 300) @@ -394,28 +407,35 @@ if LDAP_CONFIG is not None: # # Caching # - -if CACHING_REDIS_SSL: - REDIS_CACHE_CON_STRING = 'rediss://' +if CACHING_REDIS_USING_SENTINEL: + CACHEOPS_SENTINEL = { + 'locations': CACHING_REDIS_SENTINELS, + 'service_name': CACHING_REDIS_SENTINEL_SERVICE, + 'db': CACHING_REDIS_DATABASE, + } else: - REDIS_CACHE_CON_STRING = 'redis://' + if CACHING_REDIS_SSL: + REDIS_CACHE_CON_STRING = 'rediss://' + else: + REDIS_CACHE_CON_STRING = 'redis://' -if CACHING_REDIS_PASSWORD: - REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, CACHING_REDIS_PASSWORD) + if CACHING_REDIS_PASSWORD: + REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, CACHING_REDIS_PASSWORD) -REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format( - REDIS_CACHE_CON_STRING, - CACHING_REDIS_HOST, - CACHING_REDIS_PORT, - CACHING_REDIS_DATABASE -) + REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format( + REDIS_CACHE_CON_STRING, + CACHING_REDIS_HOST, + CACHING_REDIS_PORT, + CACHING_REDIS_DATABASE + ) + CACHEOPS_REDIS = REDIS_CACHE_CON_STRING if not CACHE_TIMEOUT: CACHEOPS_ENABLED = False else: CACHEOPS_ENABLED = True -CACHEOPS_REDIS = REDIS_CACHE_CON_STRING + CACHEOPS_DEFAULTS = { 'timeout': CACHE_TIMEOUT } @@ -534,6 +554,15 @@ RQ_QUEUES = { 'PASSWORD': WEBHOOKS_REDIS_PASSWORD, 'DEFAULT_TIMEOUT': WEBHOOKS_REDIS_DEFAULT_TIMEOUT, 'SSL': WEBHOOKS_REDIS_SSL, + } if not WEBHOOKS_REDIS_USING_SENTINEL else { + 'SENTINELS': WEBHOOKS_REDIS_SENTINELS, + 'MASTER_NAME': WEBHOOKS_REDIS_SENTINEL_SERVICE, + 'DB': WEBHOOKS_REDIS_DATABASE, + 'PASSWORD': WEBHOOKS_REDIS_PASSWORD, + 'SOCKET_TIMEOUT': None, + 'CONNECTION_KWARGS': { + 'socket_connect_timeout': WEBHOOKS_REDIS_DEFAULT_TIMEOUT + }, } } From ba9a2956a8ac6076b4f9e74e857e6b4634e2a836 Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Thu, 13 Feb 2020 09:10:53 -0500 Subject: [PATCH 2/3] documentation on redis sentinel --- docs/configuration/required-settings.md | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/configuration/required-settings.md b/docs/configuration/required-settings.md index 70c6de276..81790eae0 100644 --- a/docs/configuration/required-settings.md +++ b/docs/configuration/required-settings.md @@ -88,6 +88,48 @@ REDIS = { It is highly recommended to keep the webhook and cache databases separate. Using the same database number on the same Redis instance for both may result in webhook processing data being lost during cache flushing events. +### Using Redis Sentinel + +If you are using [Redis Sentinel](https://redis.io/topics/sentinel) for high-availability purposes, there is minimal +configuration necessary to convert NetBox to recognize it. It requires the removal of the `HOST` and `PORT` keys from +above and the addition of two new keys. + +* `SENTINELS`: List of tuples or tuple of tuples with each inner tuple containing the name or IP address +of the Redis server and port for each sentinel instance to connect to +* `SENTINEL_SERVICE`: Name of the master / service to connect to + +Example: + +```python +REDIS = { + 'webhooks': { + 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'SENTINELS': [ + ('mysentinel.redis.example.com', 6379), + ('othersentinel.redis.example.com', 6379) + ], + 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } +} +``` + +!!! note: + It is possible to have only one or the other Redis configurations to use Sentinel functionality. It is possible + for example to have the webhook use sentinel via `HOST`/`PORT` and for caching to use Sentinel via + `SENTINELS`/`SENTINEL_SERVICE`. + + --- ## SECRET_KEY From 72f0e31b84820a7eaf508e567956c0d365f2afba Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Thu, 13 Feb 2020 10:27:56 -0500 Subject: [PATCH 3/3] fixed extraneous linting error --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 4b6177318..1a36cf424 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -409,7 +409,7 @@ if LDAP_CONFIG is not None: # if CACHING_REDIS_USING_SENTINEL: CACHEOPS_SENTINEL = { - 'locations': CACHING_REDIS_SENTINELS, + 'locations': CACHING_REDIS_SENTINELS, 'service_name': CACHING_REDIS_SENTINEL_SERVICE, 'db': CACHING_REDIS_DATABASE, }