From 8b76db2bca5577a5b6c8c8e21526d7445a7274e1 Mon Sep 17 00:00:00 2001 From: maximumG Date: Wed, 7 Jul 2021 15:25:53 +0200 Subject: [PATCH 1/4] add: RQ queues for netbox core (high, default, low, check_release) --- netbox/netbox/settings.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index d71bc6486..fe51860a0 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -569,8 +569,10 @@ else: } RQ_QUEUES = { + 'high': RQ_PARAMS, 'default': RQ_PARAMS, # Webhooks - 'check_releases': RQ_PARAMS, + 'low': RQ_PARAMS, + 'check_release': RQ_PARAMS, } @@ -635,3 +637,14 @@ for plugin_name in PLUGINS: CACHEOPS.update({ "{}.{}".format(plugin_name, key): value for key, value in plugin_config.caching_config.items() }) + + # Create RQ queues dedicated to the plugin + # we use the plugin name as a prefix for queue name's defined in the plugin config + # ex: mysuperplugin.mysuperqueue1 + if type(plugin_config.queues) is not list: + raise ImproperlyConfigured( + "Plugin {} queues must be a list.".format(plugin_name) + ) + RQ_QUEUES.update({ + f"{plugin_name}.{queue}": RQ_PARAMS for queue in plugin_config.queues + }) From 995aa65f16765e9ca6e54de60d600b7bae671522 Mon Sep 17 00:00:00 2001 From: maximumG Date: Wed, 7 Jul 2021 15:26:33 +0200 Subject: [PATCH 2/4] feat: Netbox plugin can defined their own RQ queues --- netbox/extras/plugins/__init__.py | 3 +++ netbox/extras/tests/dummy_plugin/__init__.py | 5 +++++ netbox/extras/tests/test_plugins.py | 8 ++++++++ 3 files changed, 16 insertions(+) diff --git a/netbox/extras/plugins/__init__.py b/netbox/extras/plugins/__init__.py index 0ea461344..2413e1aa0 100644 --- a/netbox/extras/plugins/__init__.py +++ b/netbox/extras/plugins/__init__.py @@ -52,6 +52,9 @@ class PluginConfig(AppConfig): '*': {'ops': 'all'}, } + # Django-rq queues dedicated to the plugin + queues = [] + # Default integration paths. Plugin authors can override these to customize the paths to # integrated components. template_extensions = 'template_content.template_extensions' diff --git a/netbox/extras/tests/dummy_plugin/__init__.py b/netbox/extras/tests/dummy_plugin/__init__.py index 63f7d308e..83baf064f 100644 --- a/netbox/extras/tests/dummy_plugin/__init__.py +++ b/netbox/extras/tests/dummy_plugin/__init__.py @@ -12,6 +12,11 @@ class DummyPluginConfig(PluginConfig): middleware = [ 'extras.tests.dummy_plugin.middleware.DummyMiddleware' ] + queues = [ + 'testing-low', + 'testing-medium', + 'testing-high' + ] config = DummyPluginConfig diff --git a/netbox/extras/tests/test_plugins.py b/netbox/extras/tests/test_plugins.py index b47453aa7..74fd1d759 100644 --- a/netbox/extras/tests/test_plugins.py +++ b/netbox/extras/tests/test_plugins.py @@ -86,6 +86,14 @@ class PluginTest(TestCase): """ self.assertIn('extras.tests.dummy_plugin.*', settings.CACHEOPS) + def test_queues(self): + """ + Check that plugin queues are registered with the accurate name. + """ + self.assertIn('extras.tests.dummy_plugin.testing-low', settings.RQ_QUEUES) + self.assertIn('extras.tests.dummy_plugin.testing-medium', settings.RQ_QUEUES) + self.assertIn('extras.tests.dummy_plugin.testing-high', settings.RQ_QUEUES) + def test_min_version(self): """ Check enforcement of minimum NetBox version. From dce3e0763fcd94e687c72c2b68df0db4bebca3f5 Mon Sep 17 00:00:00 2001 From: maximumG Date: Fri, 9 Jul 2021 09:42:03 +0200 Subject: [PATCH 3/4] chore: documentation about netbox plugin queueing system --- docs/plugins/development.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/plugins/development.md b/docs/plugins/development.md index f008da2fb..7aab5ca8a 100644 --- a/docs/plugins/development.md +++ b/docs/plugins/development.md @@ -413,3 +413,31 @@ caching_config = { ``` See the [django-cacheops](https://github.com/Suor/django-cacheops) documentation for more detail on configuring caching. + +## Background Tasks + +By default, Netbox provides 3 differents [RQ](https://python-rq.org/) queues to run background jobs : *high*, *default* and *low*. +These 3 core queues can be used out-of-the-box by plugins to define background tasks. + +Plugins can also define dedicated queues. These queues can be configured under the PluginConfig class `queues` attribute. An example configuration +is below: + +```python +class MyPluginConfig(PluginConfig): + name = 'myplugin' + ... + queues = [ + 'queue1', + 'queue2', + 'queue-whatever-the-name' + ] +``` + +The PluginConfig above creates 3 queues with the following names: *myplugin.queue1*, *myplugin.queue2*, *myplugin.queue-whatever-the-name*. +As you can see, the queue's name is always preprended with the plugin's name, to avoid any name clashes between different plugins. + +In case you create dedicated queues for your plugin, it is strongly advised to also create a dedicated RQ worker instance. This instance should only listen to the queues defined in your plugin - to avoid impact between your background tasks and netbox internal tasks. + +``` +python manage.py rqworker myplugin.queue1 myplugin.queue2 myplugin.queue-whatever-the-name +``` \ No newline at end of file From a09e4bf82a2a7ec02fbb4ca7267baed28d421caa Mon Sep 17 00:00:00 2001 From: maximumG Date: Fri, 9 Jul 2021 10:03:51 +0200 Subject: [PATCH 4/4] chore: avoid the default RQ worker to listen to every queues by default and rather only listen to netbox core queues. --- contrib/netbox-rq.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/netbox-rq.service b/contrib/netbox-rq.service index 77d70910c..5b03777ed 100644 --- a/contrib/netbox-rq.service +++ b/contrib/netbox-rq.service @@ -11,7 +11,7 @@ User=netbox Group=netbox WorkingDirectory=/opt/netbox -ExecStart=/opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py rqworker +ExecStart=/opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py rqworker high default low Restart=on-failure RestartSec=30