14731 plugins catalog (#16763)

* 14731 plugin catalog

* 14731 detal page

* 14731 plugin table

* 14731 cleanup

* 14731 cache API results

* 14731 fix install name

* 14731 filtering

* 14731 filtering

* 14731 fix detail view

* 14731 fix detail view

* 14731 sort / status

* 14731 sort / status

* 14731 cleanup detail view

* 14731 htmx plugin list

* 14731 align quicksearch

* 14731 remove pytz

* 14731 change to table

* 14731 change to table

* 14731 remove status from table

* 14731 quick search

* 14731 cleanup

* 14731 cleanup

* Employ datetime_from_timestamp() to parse timestamps

* 14731 review changes

* 14731 move to plugins.py file

* 14731 use dataclasses

* 14731 review changes

* Tweak table columns

* Use is_staff (for now) to evaluate user permission for plugin views

* Use table for ordering

* 7025 change to api fields

* 14731 tweaks

* Remove filtering for is_netboxlabs_supported

* Misc cleanup

* Update logic for determining whether to display plugin installation instructions

* 14731 review changes

* 14731 review changes

* 14731 review changes

* 14731 add user agent string, proxy settings

* Clean up templates

---------

Co-authored-by: Jeremy Stretch <jstretch@netboxlabs.com>

netbox/core/plugins.py

@@ -0,0 +1,209 @@
+import datetime
+import importlib
+import importlib.util
+from dataclasses import dataclass, field
+from typing import Optional
+
+import requests
+from django.conf import settings
+from django.core.cache import cache
+from django.utils.translation import gettext_lazy as _
+
+from netbox.plugins import PluginConfig
+from utilities.datetime import datetime_from_timestamp
+
+USER_AGENT_STRING = f'NetBox/{settings.RELEASE.version} {settings.RELEASE.edition}'
+
+
+@dataclass
+class PluginAuthor:
+    """
+    Identifying information for the author of a plugin.
+    """
+    name: str
+    org_id: str = ''
+    url: str = ''
+
+
+@dataclass
+class PluginVersion:
+    """
+    Details for a specific versioned release of a plugin.
+    """
+    date: datetime.datetime = None
+    version: str = ''
+    netbox_min_version: str = ''
+    netbox_max_version: str = ''
+    has_model: bool = False
+    is_certified: bool = False
+    is_feature: bool = False
+    is_integration: bool = False
+    is_netboxlabs_supported: bool = False
+
+
+@dataclass
+class Plugin:
+    """
+    The representation of a NetBox plugin in the catalog API.
+    """
+    id: str = ''
+    status: str = ''
+    title_short: str = ''
+    title_long: str = ''
+    tag_line: str = ''
+    description_short: str = ''
+    slug: str = ''
+    author: Optional[PluginAuthor] = None
+    created_at: datetime.datetime = None
+    updated_at: datetime.datetime = None
+    license_type: str = ''
+    homepage_url: str = ''
+    package_name_pypi: str = ''
+    config_name: str = ''
+    is_certified: bool = False
+    release_latest: PluginVersion = field(default_factory=PluginVersion)
+    release_recent_history: list[PluginVersion] = field(default_factory=list)
+    is_local: bool = False  # extra field for locally installed plugins
+    is_installed: bool = False
+    installed_version: str = ''
+
+
+def get_local_plugins():
+    """
+    Return a dictionary of all locally-installed plugins, mapped by name.
+    """
+    plugins = {}
+    for plugin_name in settings.PLUGINS:
+        plugin = importlib.import_module(plugin_name)
+        plugin_config: PluginConfig = plugin.config
+
+        plugins[plugin_config.name] = Plugin(
+            slug=plugin_config.name,
+            title_short=plugin_config.verbose_name,
+            tag_line=plugin_config.description,
+            description_short=plugin_config.description,
+            is_local=True,
+            is_installed=True,
+            installed_version=plugin_config.version,
+        )
+
+    return plugins
+
+
+def get_catalog_plugins():
+    """
+    Return a dictionary of all entries in the plugins catalog, mapped by name.
+    """
+    session = requests.Session()
+    plugins = {}
+
+    def get_pages():
+        # TODO: pagination is currently broken in API
+        payload = {'page': '1', 'per_page': '50'}
+        first_page = session.get(
+            settings.PLUGIN_CATALOG_URL,
+            headers={'User-Agent': USER_AGENT_STRING},
+            proxies=settings.HTTP_PROXIES,
+            timeout=3,
+            params=payload
+        ).json()
+        yield first_page
+        num_pages = first_page['metadata']['pagination']['last_page']
+
+        for page in range(2, num_pages + 1):
+            payload['page'] = page
+            next_page = session.get(
+                settings.PLUGIN_CATALOG_URL,
+                headers={'User-Agent': USER_AGENT_STRING},
+                proxies=settings.HTTP_PROXIES,
+                timeout=3,
+                params=payload
+            ).json()
+            yield next_page
+
+    for page in get_pages():
+        for data in page['data']:
+
+            # Populate releases
+            releases = []
+            for version in data['release_recent_history']:
+                releases.append(
+                    PluginVersion(
+                        date=datetime_from_timestamp(version['date']),
+                        version=version['version'],
+                        netbox_min_version=version['netbox_min_version'],
+                        netbox_max_version=version['netbox_max_version'],
+                        has_model=version['has_model'],
+                        is_certified=version['is_certified'],
+                        is_feature=version['is_feature'],
+                        is_integration=version['is_integration'],
+                        is_netboxlabs_supported=version['is_netboxlabs_supported'],
+                    )
+                )
+            releases = sorted(releases, key=lambda x: x.date, reverse=True)
+            latest_release = PluginVersion(
+                date=datetime_from_timestamp(data['release_latest']['date']),
+                version=data['release_latest']['version'],
+                netbox_min_version=data['release_latest']['netbox_min_version'],
+                netbox_max_version=data['release_latest']['netbox_max_version'],
+                has_model=data['release_latest']['has_model'],
+                is_certified=data['release_latest']['is_certified'],
+                is_feature=data['release_latest']['is_feature'],
+                is_integration=data['release_latest']['is_integration'],
+                is_netboxlabs_supported=data['release_latest']['is_netboxlabs_supported'],
+            )
+
+            # Populate author (if any)
+            if data['author']:
+                print(data['author'])
+                author = PluginAuthor(
+                    name=data['author']['name'],
+                    org_id=data['author']['org_id'],
+                    url=data['author']['url'],
+                )
+            else:
+                author = None
+
+            # Populate plugin data
+            plugins[data['slug']] = Plugin(
+                id=data['id'],
+                status=data['status'],
+                title_short=data['title_short'],
+                title_long=data['title_long'],
+                tag_line=data['tag_line'],
+                description_short=data['description_short'],
+                slug=data['slug'],
+                author=author,
+                created_at=datetime_from_timestamp(data['created_at']),
+                updated_at=datetime_from_timestamp(data['updated_at']),
+                license_type=data['license_type'],
+                homepage_url=data['homepage_url'],
+                package_name_pypi=data['package_name_pypi'],
+                config_name=data['config_name'],
+                is_certified=data['is_certified'],
+                release_latest=latest_release,
+                release_recent_history=releases,
+            )
+
+    return plugins
+
+
+def get_plugins():
+    """
+    Return a dictionary of all plugins (both catalog and locally installed), mapped by name.
+    """
+    local_plugins = get_local_plugins()
+    catalog_plugins = cache.get('plugins-catalog-feed')
+    if not catalog_plugins:
+        catalog_plugins = get_catalog_plugins()
+        cache.set('plugins-catalog-feed', catalog_plugins, 3600)
+
+    plugins = catalog_plugins
+    for k, v in local_plugins.items():
+        if k in plugins:
+            plugins[k].is_local = True
+            plugins[k].is_installed = True
+        else:
+            plugins[k] = v
+
+    return plugins

netbox/core/tables/plugins.py

@@ -1,39 +1,80 @@
 import django_tables2 as tables
 from django.utils.translation import gettext_lazy as _
-from netbox.tables import BaseTable
+
+from netbox.tables import BaseTable, columns
 
 __all__ = (
-    'PluginTable',
+    'CatalogPluginTable',
+    'PluginVersionTable',
 )
 
 
-class PluginTable(BaseTable):
+class PluginVersionTable(BaseTable):
-    name = tables.Column(
-        accessor=tables.A('verbose_name'),
-        verbose_name=_('Name')
-    )
     version = tables.Column(
         verbose_name=_('Version')
     )
-    package = tables.Column(
+    last_updated = columns.DateTimeColumn(
-        accessor=tables.A('name'),
+        accessor=tables.A('date'),
-        verbose_name=_('Package')
+        timespec='minutes',
+        verbose_name=_('Last Updated')
     )
-    author = tables.Column(
+    min_version = tables.Column(
-        verbose_name=_('Author')
+        accessor=tables.A('netbox_min_version'),
+        verbose_name=_('Minimum NetBox Version')
     )
-    author_email = tables.Column(
+    max_version = tables.Column(
-        verbose_name=_('Author Email')
+        accessor=tables.A('netbox_max_version'),
-    )
+        verbose_name=_('Maximum NetBox Version')
-    description = tables.Column(
-        verbose_name=_('Description')
     )
 
     class Meta(BaseTable.Meta):
-        empty_text = _('No plugins found')
+        empty_text = _('No plugin data found')
         fields = (
-            'name', 'version', 'package', 'author', 'author_email', 'description',
+            'version', 'last_updated', 'min_version', 'max_version',
         )
         default_columns = (
-            'name', 'version', 'package', 'description',
+            'version', 'last_updated', 'min_version', 'max_version',
         )
+        orderable = False
+
+
+class CatalogPluginTable(BaseTable):
+    title_short = tables.Column(
+        linkify=('core:plugin', [tables.A('slug')]),
+        verbose_name=_('Name')
+    )
+    author = tables.Column(
+        accessor=tables.A('author.name'),
+        verbose_name=_('Author')
+    )
+    is_local = columns.BooleanColumn(
+        verbose_name=_('Local')
+    )
+    is_installed = columns.BooleanColumn(
+        verbose_name=_('Installed')
+    )
+    is_certified = columns.BooleanColumn(
+        verbose_name=_('Certified')
+    )
+    created_at = columns.DateTimeColumn(
+        verbose_name=_('Published')
+    )
+    updated_at = columns.DateTimeColumn(
+        verbose_name=_('Updated')
+    )
+    installed_version = tables.Column(
+        verbose_name=_('Installed version')
+    )
+
+    class Meta(BaseTable.Meta):
+        empty_text = _('No plugin data found')
+        fields = (
+            'title_short', 'author', 'is_local', 'is_installed', 'is_certified', 'created_at', 'updated_at',
+            'installed_version',
+        )
+        default_columns = (
+            'title_short', 'author', 'is_local', 'is_installed', 'is_certified', 'created_at', 'updated_at',
+        )
+        # List installed plugins first, then certified plugins, then
+        # everything else (with each tranche ordered alphabetically)
+        order_by = ('-is_installed', '-is_certified', 'name')

netbox/core/urls.py

@@ -49,4 +49,8 @@ urlpatterns = (
 
     # System
     path('system/', views.SystemView.as_view(), name='system'),
+
+    # Plugins
+    path('plugins/', views.PluginListView.as_view(), name='plugin_list'),
+    path('plugins/<str:name>/', views.PluginView.as_view(), name='plugin'),
 )

netbox/core/views.py

@@ -2,7 +2,6 @@ import json
 import platform
 
 from django import __version__ as DJANGO_VERSION
-from django.apps import apps
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.mixins import UserPassesTestMixin
@@ -36,6 +35,8 @@ from utilities.query import count_related
 from utilities.views import ContentTypePermissionRequiredMixin, GetRelatedModelsMixin, register_model_view
 from . import filtersets, forms, tables
 from .models import *
+from .plugins import get_plugins
+from .tables import CatalogPluginTable, PluginVersionTable
 
 
 #
@@ -581,7 +582,7 @@ class WorkerView(BaseRQView):
 
 
 #
-# Plugins
+# System
 #
 
 class SystemView(UserPassesTestMixin, View):
@@ -614,12 +615,6 @@ class SystemView(UserPassesTestMixin, View):
             'rq_worker_count': Worker.count(get_connection('default')),
         }
 
-        # Plugins
-        plugins = [
-            # Look up app config by package name
-            apps.get_app_config(plugin.rsplit('.', 1)[-1]) for plugin in settings.PLUGINS
-        ]
-
         # Configuration
         try:
             config = ConfigRevision.objects.get(pk=cache.get('config_version'))
@@ -631,9 +626,6 @@ class SystemView(UserPassesTestMixin, View):
         if 'export' in request.GET:
             data = {
                 **stats,
-                'plugins': {
-                    plugin.name: plugin.version for plugin in plugins
-                },
                 'config': {
                     k: config.data[k] for k in sorted(config.data)
                 },
@@ -642,11 +634,58 @@ class SystemView(UserPassesTestMixin, View):
             response['Content-Disposition'] = 'attachment; filename="netbox.json"'
             return response
 
-        plugins_table = tables.PluginTable(plugins, orderable=False)
-        plugins_table.configure(request)
-
         return render(request, 'core/system.html', {
             'stats': stats,
-            'plugins_table': plugins_table,
             'config': config,
         })
+
+
+#
+# Plugins
+#
+
+class PluginListView(UserPassesTestMixin, View):
+
+    def test_func(self):
+        return self.request.user.is_staff
+
+    def get(self, request):
+        q = request.GET.get('q', None)
+
+        plugins = get_plugins().values()
+        if q:
+            plugins = [obj for obj in plugins if q.casefold() in obj.title_short.casefold()]
+
+        table = CatalogPluginTable(plugins, user=request.user)
+        table.configure(request)
+
+        # If this is an HTMX request, return only the rendered table HTML
+        if htmx_partial(request):
+            return render(request, 'htmx/table.html', {
+                'table': table,
+            })
+
+        return render(request, 'core/plugin_list.html', {
+            'table': table,
+        })
+
+
+class PluginView(UserPassesTestMixin, View):
+
+    def test_func(self):
+        return self.request.user.is_staff
+
+    def get(self, request, name):
+
+        plugins = get_plugins()
+        if name not in plugins:
+            raise Http404(_("Plugin {name} not found").format(name=name))
+        plugin = plugins[name]
+
+        table = PluginVersionTable(plugin.release_recent_history, user=request.user)
+        table.configure(request)
+
+        return render(request, 'core/plugin.html', {
+            'plugin': plugin,
+            'table': table,
+        })

netbox/netbox/navigation/menu.py

@@ -437,6 +437,11 @@ ADMIN_MENU = Menu(
                     link_text=_('System'),
                     auth_required=True
                 ),
+                MenuItem(
+                    link='core:plugin_list',
+                    link_text=_('Plugins'),
+                    auth_required=True
+                ),
                 MenuItem(
                     link='core:configrevision_list',
                     link_text=_('Configuration History'),

netbox/netbox/settings.py

@@ -769,6 +769,8 @@ STRAWBERRY_DJANGO = {
 # Plugins
 #
 
+PLUGIN_CATALOG_URL = 'https://api.netbox.oss.netboxlabs.com/v1/plugins'
+
 # Register any configured plugins
 for plugin_name in PLUGINS:
     try:

netbox/templates/core/inc/plugin_installation.html

@@ -0,0 +1,29 @@
+<p>You can install this plugin from the command line with PyPI.</p>
+<p>The following commands may be helpful; always refer to <a href="{{ plugin.homepage_url }}" target="_blank">the plugin's own documentation <i class="mdi mdi-launch"></i></a> and the <a href="https://netboxlabs.com/docs/netbox/en/stable/plugins/installation/" target="_blank">Installing a Plugin unit <i class="mdi mdi-launch"></i></a> of the NetBox documentation.</p>
+<p>1. Enter the NetBox virtual environment and install the plugin package:</p>
+
+<pre class="block">
+source /opt/netbox/venv/bin/activate
+pip install {{ plugin.slug }}
+</pre>
+
+<p>2. In /opt/netbox/netbox/netbox/configuration.py, add the plugin to the PLUGINS list:</p>
+
+<pre class="block">
+PLUGINS=［
+"{{ plugin.config_name }}",
+]
+</pre>
+
+<p>3. Still from the NetBox virtual environment, run database migrations and collect static files:</p>
+
+<pre class="block">
+python3 /opt/netbox/netbox/netbox/manage.py migrate
+python3 /opt/netbox/netbox/netbox/manage.py collectstatic
+</pre>
+
+<p>4. Restart the NetBox services to complete the plugin installation:</p>
+
+<pre class="block">
+sudo systemctl restart netbox netbox-rq
+</pre>

netbox/templates/core/plugin.html

@@ -0,0 +1,113 @@
+{% extends 'generic/object.html' %}
+{% load helpers %}
+{% load form_helpers %}
+{% load i18n %}
+
+{% block title %}{{ plugin.title_short }}{% endblock %}
+
+{% block object_identifier %}
+{% endblock object_identifier %}
+
+{% block breadcrumbs %}
+  <li class="breadcrumb-item"><a href="{% url 'core:plugin_list' %}">{% trans "Plugins" %}</a></li>
+{% endblock breadcrumbs %}
+
+{% block subtitle %}
+  <span class="text-secondary fs-5">
+    {% checkmark plugin.is_installed %}
+    {% if plugin.is_installed %}
+      v{{ plugin.installed_version }} {% trans "installed" %}
+    {% else %}
+      {% trans "Not installed" %}
+    {% endif %}
+  </span>
+{% endblock %}
+
+{% block controls %}{% endblock %}
+
+{% block tabs %}
+  <ul class="nav nav-tabs" role="tablist">
+    <li class="nav-item" role="presentation">
+      <a class="nav-link active" id="overview-tab" data-bs-toggle="tab" data-bs-target="#overview" type="button" role="tab" aria-controls="edit-form" aria-selected="true">
+        {% trans "Overview" %}
+      </a>
+    </li>
+    {% if True or not plugin.is_local and 'commercial' not in settings.RELEASE.features %}
+      <li class="nav-item" role="presentation">
+        <button class="nav-link" id="install-tab" data-bs-toggle="tab" data-bs-target="#install" type="button" role="tab" aria-controls="object-list" aria-selected="false">
+          {% trans "Install" %}
+        </button>
+      </li>
+    {% endif %}
+  </ul>
+{% endblock tabs %}
+
+{% block content %}
+  <div class="tab-pane show active" id="overview" role="tabpanel" aria-labelledby="overview-tab">
+    <div class="row">
+      <div class="col col-6">
+        <div class="card">
+          <h5 class="card-header">{% trans "Plugin Details" %}</h5>
+          <table class="table table-hover attr-table">
+            <tr>
+              <th scope="row">{% trans "Name" %}</th>
+              <td>{{ plugin.title_short }}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "Summary" %}</th>
+              <td>{{ plugin.tag_line|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "Author" %}</th>
+              <td>{{ plugin.author.name|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "URL" %}</th>
+              <td>
+                {% if plugin.homepage_url %}
+                  <a href="{{ plugin.homepage_url }}">{{ plugin.homepage_url }}</a>
+                {% else %}
+                  {{ ''|placeholder }}
+                {% endif %}
+              </td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "License" %}</th>
+              <td>{{ plugin.license_type|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "Description" %}</th>
+              <td>{{ plugin.description_short|markdown }}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "Certified" %}</th>
+              <td>{% checkmark plugin.is_certified %}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "Local" %}</th>
+              <td>{% checkmark plugin.is_local %}</td>
+            </tr>
+          </table>
+        </div>
+      </div>
+      <div class="col col-6">
+        <div class="card">
+          <h5 class="card-header">{% trans "Version History" %}</h5>
+          <div class="htmx-container table-responsive" id="object_list">
+            {% include 'htmx/table.html' %}
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+  {% if True or not plugin.is_local and 'commercial' not in settings.RELEASE.features %}
+    <div class="tab-pane" id="install" role="tabpanel" aria-labelledby="install-tab">
+      <div class="card">
+        <h5 class="card-header">{% trans "Local Installation Instructions" %}</h5>
+        <div class="card-body">
+          {% include 'core/inc/plugin_installation.html' %}
+        </div>
+      </div>
+    </div>
+  {% endif %}
+{% endblock content %}

netbox/templates/core/plugin_list.html

@@ -0,0 +1,16 @@
+{% extends 'generic/object_list.html' %}
+{% load buttons %}
+{% load helpers %}
+{% load i18n %}
+{% load render_table from django_tables2 %}
+
+{% block title %}{% trans "Plugins" %}{% endblock %}
+
+{% block tabs %}
+  <ul class="nav nav-tabs px-3">
+    <li class="nav-item" role="presentation">
+      <a class="nav-link active" role="tab">{% trans "Plugins" %}</a>
+    </li>
+  </ul>
+{% endblock tabs %}
+

netbox/templates/core/system.html

@@ -78,16 +78,6 @@
     </div>
   </div>
 
-  {# Plugins #}
-  <div class="row mb-3">
-    <div class="col col-md-12">
-      <div class="card">
-        <h5 class="card-header">{% trans "Plugins" %}</h5>
-        {% render_table plugins_table %}
-      </div>
-    </div>
-  </div>
-
   {# Configuration #}
   <div class="row mb-3">
     <div class="col col-md-12">