diff --git a/contrib/netbox-rq.service b/contrib/netbox-rq.service index 77d70910c..5b03777ed 100644 --- a/contrib/netbox-rq.service +++ b/contrib/netbox-rq.service @@ -11,7 +11,7 @@ User=netbox Group=netbox WorkingDirectory=/opt/netbox -ExecStart=/opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py rqworker +ExecStart=/opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py rqworker high default low Restart=on-failure RestartSec=30 diff --git a/docs/plugins/development.md b/docs/plugins/development.md index edeb4a2eb..ed81eb891 100644 --- a/docs/plugins/development.md +++ b/docs/plugins/development.md @@ -383,4 +383,32 @@ class SiteAnimalCount(PluginTemplateExtension): }) template_extensions = [SiteAnimalCount] -``` \ No newline at end of file +``` + +## Background Tasks + +By default, Netbox provides 3 differents [RQ](https://python-rq.org/) queues to run background jobs : *high*, *default* and *low*. +These 3 core queues can be used out-of-the-box by plugins to define background tasks. + +Plugins can also define dedicated queues. These queues can be configured under the PluginConfig class `queues` attribute. An example configuration +is below: + +```python +class MyPluginConfig(PluginConfig): + name = 'myplugin' + ... + queues = [ + 'queue1', + 'queue2', + 'queue-whatever-the-name' + ] +``` + +The PluginConfig above creates 3 queues with the following names: *myplugin.queue1*, *myplugin.queue2*, *myplugin.queue-whatever-the-name*. +As you can see, the queue's name is always preprended with the plugin's name, to avoid any name clashes between different plugins. + +In case you create dedicated queues for your plugin, it is strongly advised to also create a dedicated RQ worker instance. This instance should only listen to the queues defined in your plugin - to avoid impact between your background tasks and netbox internal tasks. + +``` +python manage.py rqworker myplugin.queue1 myplugin.queue2 myplugin.queue-whatever-the-name +``` diff --git a/netbox/extras/plugins/__init__.py b/netbox/extras/plugins/__init__.py index a4d61f017..f9a7856ea 100644 --- a/netbox/extras/plugins/__init__.py +++ b/netbox/extras/plugins/__init__.py @@ -47,6 +47,9 @@ class PluginConfig(AppConfig): # Middleware classes provided by the plugin middleware = [] + # Django-rq queues dedicated to the plugin + queues = [] + # Default integration paths. Plugin authors can override these to customize the paths to # integrated components. template_extensions = 'template_content.template_extensions' diff --git a/netbox/extras/tests/dummy_plugin/__init__.py b/netbox/extras/tests/dummy_plugin/__init__.py index 63f7d308e..83baf064f 100644 --- a/netbox/extras/tests/dummy_plugin/__init__.py +++ b/netbox/extras/tests/dummy_plugin/__init__.py @@ -12,6 +12,11 @@ class DummyPluginConfig(PluginConfig): middleware = [ 'extras.tests.dummy_plugin.middleware.DummyMiddleware' ] + queues = [ + 'testing-low', + 'testing-medium', + 'testing-high' + ] config = DummyPluginConfig diff --git a/netbox/extras/tests/test_plugins.py b/netbox/extras/tests/test_plugins.py index 00a385bf5..2508ffb83 100644 --- a/netbox/extras/tests/test_plugins.py +++ b/netbox/extras/tests/test_plugins.py @@ -80,6 +80,14 @@ class PluginTest(TestCase): """ self.assertIn('extras.tests.dummy_plugin.middleware.DummyMiddleware', settings.MIDDLEWARE) + def test_queues(self): + """ + Check that plugin queues are registered with the accurate name. + """ + self.assertIn('extras.tests.dummy_plugin.testing-low', settings.RQ_QUEUES) + self.assertIn('extras.tests.dummy_plugin.testing-medium', settings.RQ_QUEUES) + self.assertIn('extras.tests.dummy_plugin.testing-high', settings.RQ_QUEUES) + def test_min_version(self): """ Check enforcement of minimum NetBox version. diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 09447a7a9..f3dc3af87 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -546,7 +546,9 @@ else: } RQ_QUEUES = { + 'high': RQ_PARAMS, 'default': RQ_PARAMS, + 'low': RQ_PARAMS, } @@ -599,3 +601,14 @@ for plugin_name in PLUGINS: plugin_middleware = plugin_config.middleware if plugin_middleware and type(plugin_middleware) in (list, tuple): MIDDLEWARE.extend(plugin_middleware) + + # Create RQ queues dedicated to the plugin + # we use the plugin name as a prefix for queue name's defined in the plugin config + # ex: mysuperplugin.mysuperqueue1 + if type(plugin_config.queues) is not list: + raise ImproperlyConfigured( + "Plugin {} queues must be a list.".format(plugin_name) + ) + RQ_QUEUES.update({ + f"{plugin_name}.{queue}": RQ_PARAMS for queue in plugin_config.queues + })