implements #3282 - seperate webhooks and caching redis configs

docs/configuration/required-settings.md

@@ -24,7 +24,7 @@ NetBox requires access to a PostgreSQL database service to store data. This serv
 
 Example:
 
-```
+```python
 DATABASE = {
     'NAME': 'netbox',               # Database name
     'USER': 'netbox',               # PostgreSQL username
@@ -40,40 +40,48 @@ DATABASE = {
 
 [Redis](https://redis.io/) is an in-memory data store similar to memcached. While Redis has been an optional component of
 NetBox since the introduction of webhooks in version 2.4, it is required starting in 2.6 to support NetBox's caching
-functionality (as well as other planned features).
+functionality (as well as other planned features). In 2.7, the connection settings were broken down into two sections for
+webhooks and caching, allowing the user to connect to different Redis instances/databases per feature.
 
-Redis is configured using a configuration setting similar to `DATABASE`:
+Redis is configured using a configuration setting similar to `DATABASE` and these settings are the same for both of the `webhooks` and `caching` subsections:
 
 * `HOST` - Name or IP address of the Redis server (use `localhost` if running locally)
 * `PORT` - TCP port of the Redis service; leave blank for default port (6379)
 * `PASSWORD` - Redis password (if set)
-* `DATABASE` - Numeric database ID for webhooks
+* `DATABASE` - Numeric database ID
-* `CACHE_DATABASE` - Numeric database ID for caching
 * `DEFAULT_TIMEOUT` - Connection timeout in seconds
 * `SSL` - Use SSL connection to Redis
 
 Example:
 
-```
+```python
 REDIS = {
+    'webhooks': {
+        'HOST': 'redis.example.com',
+        'PORT': 1234,
+        'PASSWORD': 'foobar',
+        'DATABASE': 0,
+        'DEFAULT_TIMEOUT': 300,
+        'SSL': False,
+    },
+    'caching': {
         'HOST': 'localhost',
         'PORT': 6379,
         'PASSWORD': '',
-    'DATABASE': 0,
+        'DATABASE': 1,
-    'CACHE_DATABASE': 1,
         'DEFAULT_TIMEOUT': 300,
         'SSL': False,
     }
+}
 ```
 
 !!! note:
-    If you were using these settings in a prior release with webhooks, the `DATABASE` setting remains the same but
+    If you are upgrading from a version prior to v2.7, please note that the Redis connection configuration settings have
-    an additional `CACHE_DATABASE` setting has been added with a default value of 1 to support the caching backend. The
+    changed. Manual modification to bring the `REDIS` section inline with the above specification is necessary
-    `DATABASE` setting will be renamed in a future release of NetBox to better relay the meaning of the setting. 
 
 !!! warning:
-    It is highly recommended to keep the webhook and cache databases seperate. Using the same database number for both may result in webhook
+    It is highly recommended to keep the webhook and cache databases seperate. Using the same database number on the
-    processing data being lost in cache flushing events.
+    same Redis instance for both may result in webhook processing data being lost during cache flushing events.
 
 ---
 

docs/installation/2-netbox.md

@@ -138,14 +138,23 @@ Redis is a in-memory key-value store required as part of the NetBox installation
 
 ```python
 REDIS = {
+    'webhooks': {
+        'HOST': 'redis.example.com',
+        'PORT': 1234,
+        'PASSWORD': 'foobar',
+        'DATABASE': 0,
+        'DEFAULT_TIMEOUT': 300,
+        'SSL': False,
+    },
+    'caching': {
         'HOST': 'localhost',
         'PORT': 6379,
         'PASSWORD': '',
-    'DATABASE': 0,
+        'DATABASE': 1,
-    'CACHE_DATABASE': 1,
         'DEFAULT_TIMEOUT': 300,
         'SSL': False,
     }
+}
 ```
 
 ## SECRET_KEY

docs/release-notes/version-2.7.md

@@ -1,5 +1,53 @@
 v2.7.0 (FUTURE)
 
+## Changes
+
+### Redis Configuration ([#3282](https://github.com/netbox-community/netbox/issues/3282))
+
+v2.6.0 introduced caching and added the `CACHE_DATABASE` option to the existing `REDIS` database configuration section.
+This did not however, allow for using two different Redis connections for the seperate caching and webhooks features.
+This change separates the Redis connection configurations in the `REDIS` section into distinct `webhooks` and `caching` subsections.
+This requires modification of the `REDIS` section of the `configuration.py` file as follows:
+
+Old Redis configuration:
+```python
+REDIS = {
+    'HOST': 'localhost',
+    'PORT': 6379,
+    'PASSWORD': '',
+    'DATABASE': 0,
+    'CACHE_DATABASE': 1,
+    'DEFAULT_TIMEOUT': 300,
+    'SSL': False,
+}
+```
+
+New Redis configuration:
+```python
+REDIS = {
+    'webhooks': {
+        'HOST': 'redis.example.com',
+        'PORT': 1234,
+        'PASSWORD': 'foobar',
+        'DATABASE': 0,
+        'DEFAULT_TIMEOUT': 300,
+        'SSL': False,
+    },
+    'caching': {
+        'HOST': 'localhost',
+        'PORT': 6379,
+        'PASSWORD': '',
+        'DATABASE': 1,
+        'DEFAULT_TIMEOUT': 300,
+        'SSL': False,
+    }
+}
+```
+
+Note that `CACHE_DATABASE` has been removed and the connection settings have been duplicated for both `webhooks` and `caching`.
+This allows the user to make use of separate Redis instances and/or databases if desired.
+Full connection details are required in both sections, even if they are the same.
+
 ## Enhancements
 
 * [#2902](https://github.com/digitalocean/netbox/issues/2902) - Replace supervisord with systemd

netbox/extras/apps.py

@@ -21,11 +21,11 @@ class ExtrasConfig(AppConfig):
                 )
             try:
                 rs = redis.Redis(
-                    host=settings.REDIS_HOST,
+                    host=settings.WEBHOOKS_REDIS_HOST,
-                    port=settings.REDIS_PORT,
+                    port=settings.WEBHOOKS_REDIS_PORT,
-                    db=settings.REDIS_DATABASE,
+                    db=settings.WEBHOOKS_REDIS_DATABASE,
-                    password=settings.REDIS_PASSWORD or None,
+                    password=settings.WEBHOOKS_REDIS_PASSWORD or None,
-                    ssl=settings.REDIS_SSL,
+                    ssl=settings.WEBHOOKS_REDIS_SSL,
                 )
                 rs.ping()
             except redis.exceptions.ConnectionError:

netbox/netbox/configuration.example.py

@@ -26,15 +26,26 @@ DATABASE = {
 SECRET_KEY = ''
 
 # Redis database settings. The Redis database is used for caching and background processing such as webhooks
+# Seperate sections for webhooks and caching allow for connecting to seperate Redis instances/datbases if desired.
+# Full connection details are required in both sections, even if they are the same.
 REDIS = {
+    'webhooks': {
+        'HOST': 'redis.example.com',
+        'PORT': 1234,
+        'PASSWORD': 'foobar',
+        'DATABASE': 0,
+        'DEFAULT_TIMEOUT': 300,
+        'SSL': False,
+    },
+    'caching': {
         'HOST': 'localhost',
         'PORT': 6379,
         'PASSWORD': '',
-    'DATABASE': 0,
+        'DATABASE': 1,
-    'CACHE_DATABASE': 1,
         'DEFAULT_TIMEOUT': 300,
         'SSL': False,
     }
+}
 
 
 #########################

netbox/netbox/settings.py

@@ -118,13 +118,30 @@ DATABASES = {
 # Redis
 #
 
-REDIS_HOST = REDIS.get('HOST', 'localhost')
+if 'webhooks' not in REDIS:
-REDIS_PORT = REDIS.get('PORT', 6379)
+    raise ImproperlyConfigured(
-REDIS_PASSWORD = REDIS.get('PASSWORD', '')
+        "REDIS section in configuration.py is missing webhooks subsection."
-REDIS_DATABASE = REDIS.get('DATABASE', 0)
+    )
-REDIS_CACHE_DATABASE = REDIS.get('CACHE_DATABASE', 1)
+if 'caching' not in REDIS:
-REDIS_DEFAULT_TIMEOUT = REDIS.get('DEFAULT_TIMEOUT', 300)
+    raise ImproperlyConfigured(
-REDIS_SSL = REDIS.get('SSL', False)
+        "REDIS section in configuration.py is missing caching subsection."
+    )
+
+WEBHOOKS_REDIS = REDIS.get('webhooks', {})
+WEBHOOKS_REDIS_HOST = WEBHOOKS_REDIS.get('HOST', 'localhost')
+WEBHOOKS_REDIS_PORT = WEBHOOKS_REDIS.get('PORT', 6379)
+WEBHOOKS_REDIS_PASSWORD = WEBHOOKS_REDIS.get('PASSWORD', '')
+WEBHOOKS_REDIS_DATABASE = WEBHOOKS_REDIS.get('DATABASE', 0)
+WEBHOOKS_REDIS_DEFAULT_TIMEOUT = WEBHOOKS_REDIS.get('DEFAULT_TIMEOUT', 300)
+WEBHOOKS_REDIS_SSL = WEBHOOKS_REDIS.get('SSL', False)
+
+CACHING_REDIS = REDIS.get('caching', {})
+CACHING_REDIS_HOST = WEBHOOKS_REDIS.get('HOST', 'localhost')
+CACHING_REDIS_PORT = WEBHOOKS_REDIS.get('PORT', 6379)
+CACHING_REDIS_PASSWORD = WEBHOOKS_REDIS.get('PASSWORD', '')
+CACHING_REDIS_DATABASE = WEBHOOKS_REDIS.get('DATABASE', 0)
+CACHING_REDIS_DEFAULT_TIMEOUT = WEBHOOKS_REDIS.get('DEFAULT_TIMEOUT', 300)
+CACHING_REDIS_SSL = WEBHOOKS_REDIS.get('SSL', False)
 
 
 #
@@ -341,15 +358,20 @@ if LDAP_CONFIG is not None:
 # Caching
 #
 
-if REDIS_SSL:
+if CACHING_REDIS_SSL:
     REDIS_CACHE_CON_STRING = 'rediss://'
 else:
     REDIS_CACHE_CON_STRING = 'redis://'
 
-if REDIS_PASSWORD:
+if CACHING_REDIS_PASSWORD:
-    REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, REDIS_PASSWORD)
+    REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, CACHING_REDIS_PASSWORD)
 
-REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format(REDIS_CACHE_CON_STRING, REDIS_HOST, REDIS_PORT, REDIS_CACHE_DATABASE)
+REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format(
+    REDIS_CACHE_CON_STRING,
+    CACHING_REDIS_HOST,
+    CACHING_REDIS_PORT,
+    CACHING_REDIS_DATABASE
+)
 
 if not CACHE_TIMEOUT:
     CACHEOPS_ENABLED = False
@@ -467,12 +489,12 @@ SWAGGER_SETTINGS = {
 
 RQ_QUEUES = {
     'default': {
-        'HOST': REDIS_HOST,
+        'HOST': WEBHOOKS_REDIS_HOST,
-        'PORT': REDIS_PORT,
+        'PORT': WEBHOOKS_REDIS_PORT,
-        'DB': REDIS_DATABASE,
+        'DB': WEBHOOKS_REDIS_DATABASE,
-        'PASSWORD': REDIS_PASSWORD,
+        'PASSWORD': WEBHOOKS_REDIS_PASSWORD,
-        'DEFAULT_TIMEOUT': REDIS_DEFAULT_TIMEOUT,
+        'DEFAULT_TIMEOUT': WEBHOOKS_REDIS_DEFAULT_TIMEOUT,
-        'SSL': REDIS_SSL,
+        'SSL': WEBHOOKS_REDIS_SSL,
     }
 }