14731 plugins catalog (#16763)

* 14731 plugin catalog

* 14731 detal page

* 14731 plugin table

* 14731 cleanup

* 14731 cache API results

* 14731 fix install name

* 14731 filtering

* 14731 filtering

* 14731 fix detail view

* 14731 fix detail view

* 14731 sort / status

* 14731 sort / status

* 14731 cleanup detail view

* 14731 htmx plugin list

* 14731 align quicksearch

* 14731 remove pytz

* 14731 change to table

* 14731 change to table

* 14731 remove status from table

* 14731 quick search

* 14731 cleanup

* 14731 cleanup

* Employ datetime_from_timestamp() to parse timestamps

* 14731 review changes

* 14731 move to plugins.py file

* 14731 use dataclasses

* 14731 review changes

* Tweak table columns

* Use is_staff (for now) to evaluate user permission for plugin views

* Use table for ordering

* 7025 change to api fields

* 14731 tweaks

* Remove filtering for is_netboxlabs_supported

* Misc cleanup

* Update logic for determining whether to display plugin installation instructions

* 14731 review changes

* 14731 review changes

* 14731 review changes

* 14731 add user agent string, proxy settings

* Clean up templates

---------

Co-authored-by: Jeremy Stretch <jstretch@netboxlabs.com>
This commit is contained in:
Arthur Hanson 2024-07-26 01:58:48 +07:00 committed by GitHub
parent 8409ca9fd2
commit 1d6987bca0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 493 additions and 45 deletions

209
netbox/core/plugins.py Normal file
View File

@ -0,0 +1,209 @@
import datetime
import importlib
import importlib.util
from dataclasses import dataclass, field
from typing import Optional
import requests
from django.conf import settings
from django.core.cache import cache
from django.utils.translation import gettext_lazy as _
from netbox.plugins import PluginConfig
from utilities.datetime import datetime_from_timestamp
USER_AGENT_STRING = f'NetBox/{settings.RELEASE.version} {settings.RELEASE.edition}'
@dataclass
class PluginAuthor:
"""
Identifying information for the author of a plugin.
"""
name: str
org_id: str = ''
url: str = ''
@dataclass
class PluginVersion:
"""
Details for a specific versioned release of a plugin.
"""
date: datetime.datetime = None
version: str = ''
netbox_min_version: str = ''
netbox_max_version: str = ''
has_model: bool = False
is_certified: bool = False
is_feature: bool = False
is_integration: bool = False
is_netboxlabs_supported: bool = False
@dataclass
class Plugin:
"""
The representation of a NetBox plugin in the catalog API.
"""
id: str = ''
status: str = ''
title_short: str = ''
title_long: str = ''
tag_line: str = ''
description_short: str = ''
slug: str = ''
author: Optional[PluginAuthor] = None
created_at: datetime.datetime = None
updated_at: datetime.datetime = None
license_type: str = ''
homepage_url: str = ''
package_name_pypi: str = ''
config_name: str = ''
is_certified: bool = False
release_latest: PluginVersion = field(default_factory=PluginVersion)
release_recent_history: list[PluginVersion] = field(default_factory=list)
is_local: bool = False # extra field for locally installed plugins
is_installed: bool = False
installed_version: str = ''
def get_local_plugins():
"""
Return a dictionary of all locally-installed plugins, mapped by name.
"""
plugins = {}
for plugin_name in settings.PLUGINS:
plugin = importlib.import_module(plugin_name)
plugin_config: PluginConfig = plugin.config
plugins[plugin_config.name] = Plugin(
slug=plugin_config.name,
title_short=plugin_config.verbose_name,
tag_line=plugin_config.description,
description_short=plugin_config.description,
is_local=True,
is_installed=True,
installed_version=plugin_config.version,
)
return plugins
def get_catalog_plugins():
"""
Return a dictionary of all entries in the plugins catalog, mapped by name.
"""
session = requests.Session()
plugins = {}
def get_pages():
# TODO: pagination is currently broken in API
payload = {'page': '1', 'per_page': '50'}
first_page = session.get(
settings.PLUGIN_CATALOG_URL,
headers={'User-Agent': USER_AGENT_STRING},
proxies=settings.HTTP_PROXIES,
timeout=3,
params=payload
).json()
yield first_page
num_pages = first_page['metadata']['pagination']['last_page']
for page in range(2, num_pages + 1):
payload['page'] = page
next_page = session.get(
settings.PLUGIN_CATALOG_URL,
headers={'User-Agent': USER_AGENT_STRING},
proxies=settings.HTTP_PROXIES,
timeout=3,
params=payload
).json()
yield next_page
for page in get_pages():
for data in page['data']:
# Populate releases
releases = []
for version in data['release_recent_history']:
releases.append(
PluginVersion(
date=datetime_from_timestamp(version['date']),
version=version['version'],
netbox_min_version=version['netbox_min_version'],
netbox_max_version=version['netbox_max_version'],
has_model=version['has_model'],
is_certified=version['is_certified'],
is_feature=version['is_feature'],
is_integration=version['is_integration'],
is_netboxlabs_supported=version['is_netboxlabs_supported'],
)
)
releases = sorted(releases, key=lambda x: x.date, reverse=True)
latest_release = PluginVersion(
date=datetime_from_timestamp(data['release_latest']['date']),
version=data['release_latest']['version'],
netbox_min_version=data['release_latest']['netbox_min_version'],
netbox_max_version=data['release_latest']['netbox_max_version'],
has_model=data['release_latest']['has_model'],
is_certified=data['release_latest']['is_certified'],
is_feature=data['release_latest']['is_feature'],
is_integration=data['release_latest']['is_integration'],
is_netboxlabs_supported=data['release_latest']['is_netboxlabs_supported'],
)
# Populate author (if any)
if data['author']:
print(data['author'])
author = PluginAuthor(
name=data['author']['name'],
org_id=data['author']['org_id'],
url=data['author']['url'],
)
else:
author = None
# Populate plugin data
plugins[data['slug']] = Plugin(
id=data['id'],
status=data['status'],
title_short=data['title_short'],
title_long=data['title_long'],
tag_line=data['tag_line'],
description_short=data['description_short'],
slug=data['slug'],
author=author,
created_at=datetime_from_timestamp(data['created_at']),
updated_at=datetime_from_timestamp(data['updated_at']),
license_type=data['license_type'],
homepage_url=data['homepage_url'],
package_name_pypi=data['package_name_pypi'],
config_name=data['config_name'],
is_certified=data['is_certified'],
release_latest=latest_release,
release_recent_history=releases,
)
return plugins
def get_plugins():
"""
Return a dictionary of all plugins (both catalog and locally installed), mapped by name.
"""
local_plugins = get_local_plugins()
catalog_plugins = cache.get('plugins-catalog-feed')
if not catalog_plugins:
catalog_plugins = get_catalog_plugins()
cache.set('plugins-catalog-feed', catalog_plugins, 3600)
plugins = catalog_plugins
for k, v in local_plugins.items():
if k in plugins:
plugins[k].is_local = True
plugins[k].is_installed = True
else:
plugins[k] = v
return plugins

View File

@ -1,39 +1,80 @@
import django_tables2 as tables
from django.utils.translation import gettext_lazy as _
from netbox.tables import BaseTable
from netbox.tables import BaseTable, columns
__all__ = (
'PluginTable',
'CatalogPluginTable',
'PluginVersionTable',
)
class PluginTable(BaseTable):
name = tables.Column(
accessor=tables.A('verbose_name'),
verbose_name=_('Name')
)
class PluginVersionTable(BaseTable):
version = tables.Column(
verbose_name=_('Version')
)
package = tables.Column(
accessor=tables.A('name'),
verbose_name=_('Package')
last_updated = columns.DateTimeColumn(
accessor=tables.A('date'),
timespec='minutes',
verbose_name=_('Last Updated')
)
author = tables.Column(
verbose_name=_('Author')
min_version = tables.Column(
accessor=tables.A('netbox_min_version'),
verbose_name=_('Minimum NetBox Version')
)
author_email = tables.Column(
verbose_name=_('Author Email')
)
description = tables.Column(
verbose_name=_('Description')
max_version = tables.Column(
accessor=tables.A('netbox_max_version'),
verbose_name=_('Maximum NetBox Version')
)
class Meta(BaseTable.Meta):
empty_text = _('No plugins found')
empty_text = _('No plugin data found')
fields = (
'name', 'version', 'package', 'author', 'author_email', 'description',
'version', 'last_updated', 'min_version', 'max_version',
)
default_columns = (
'name', 'version', 'package', 'description',
'version', 'last_updated', 'min_version', 'max_version',
)
orderable = False
class CatalogPluginTable(BaseTable):
title_short = tables.Column(
linkify=('core:plugin', [tables.A('slug')]),
verbose_name=_('Name')
)
author = tables.Column(
accessor=tables.A('author.name'),
verbose_name=_('Author')
)
is_local = columns.BooleanColumn(
verbose_name=_('Local')
)
is_installed = columns.BooleanColumn(
verbose_name=_('Installed')
)
is_certified = columns.BooleanColumn(
verbose_name=_('Certified')
)
created_at = columns.DateTimeColumn(
verbose_name=_('Published')
)
updated_at = columns.DateTimeColumn(
verbose_name=_('Updated')
)
installed_version = tables.Column(
verbose_name=_('Installed version')
)
class Meta(BaseTable.Meta):
empty_text = _('No plugin data found')
fields = (
'title_short', 'author', 'is_local', 'is_installed', 'is_certified', 'created_at', 'updated_at',
'installed_version',
)
default_columns = (
'title_short', 'author', 'is_local', 'is_installed', 'is_certified', 'created_at', 'updated_at',
)
# List installed plugins first, then certified plugins, then
# everything else (with each tranche ordered alphabetically)
order_by = ('-is_installed', '-is_certified', 'name')

View File

@ -49,4 +49,8 @@ urlpatterns = (
# System
path('system/', views.SystemView.as_view(), name='system'),
# Plugins
path('plugins/', views.PluginListView.as_view(), name='plugin_list'),
path('plugins/<str:name>/', views.PluginView.as_view(), name='plugin'),
)

View File

@ -2,7 +2,6 @@ import json
import platform
from django import __version__ as DJANGO_VERSION
from django.apps import apps
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.mixins import UserPassesTestMixin
@ -36,6 +35,8 @@ from utilities.query import count_related
from utilities.views import ContentTypePermissionRequiredMixin, GetRelatedModelsMixin, register_model_view
from . import filtersets, forms, tables
from .models import *
from .plugins import get_plugins
from .tables import CatalogPluginTable, PluginVersionTable
#
@ -581,7 +582,7 @@ class WorkerView(BaseRQView):
#
# Plugins
# System
#
class SystemView(UserPassesTestMixin, View):
@ -614,12 +615,6 @@ class SystemView(UserPassesTestMixin, View):
'rq_worker_count': Worker.count(get_connection('default')),
}
# Plugins
plugins = [
# Look up app config by package name
apps.get_app_config(plugin.rsplit('.', 1)[-1]) for plugin in settings.PLUGINS
]
# Configuration
try:
config = ConfigRevision.objects.get(pk=cache.get('config_version'))
@ -631,9 +626,6 @@ class SystemView(UserPassesTestMixin, View):
if 'export' in request.GET:
data = {
**stats,
'plugins': {
plugin.name: plugin.version for plugin in plugins
},
'config': {
k: config.data[k] for k in sorted(config.data)
},
@ -642,11 +634,58 @@ class SystemView(UserPassesTestMixin, View):
response['Content-Disposition'] = 'attachment; filename="netbox.json"'
return response
plugins_table = tables.PluginTable(plugins, orderable=False)
plugins_table.configure(request)
return render(request, 'core/system.html', {
'stats': stats,
'plugins_table': plugins_table,
'config': config,
})
#
# Plugins
#
class PluginListView(UserPassesTestMixin, View):
def test_func(self):
return self.request.user.is_staff
def get(self, request):
q = request.GET.get('q', None)
plugins = get_plugins().values()
if q:
plugins = [obj for obj in plugins if q.casefold() in obj.title_short.casefold()]
table = CatalogPluginTable(plugins, user=request.user)
table.configure(request)
# If this is an HTMX request, return only the rendered table HTML
if htmx_partial(request):
return render(request, 'htmx/table.html', {
'table': table,
})
return render(request, 'core/plugin_list.html', {
'table': table,
})
class PluginView(UserPassesTestMixin, View):
def test_func(self):
return self.request.user.is_staff
def get(self, request, name):
plugins = get_plugins()
if name not in plugins:
raise Http404(_("Plugin {name} not found").format(name=name))
plugin = plugins[name]
table = PluginVersionTable(plugin.release_recent_history, user=request.user)
table.configure(request)
return render(request, 'core/plugin.html', {
'plugin': plugin,
'table': table,
})

View File

@ -437,6 +437,11 @@ ADMIN_MENU = Menu(
link_text=_('System'),
auth_required=True
),
MenuItem(
link='core:plugin_list',
link_text=_('Plugins'),
auth_required=True
),
MenuItem(
link='core:configrevision_list',
link_text=_('Configuration History'),

View File

@ -769,6 +769,8 @@ STRAWBERRY_DJANGO = {
# Plugins
#
PLUGIN_CATALOG_URL = 'https://api.netbox.oss.netboxlabs.com/v1/plugins'
# Register any configured plugins
for plugin_name in PLUGINS:
try:

View File

@ -0,0 +1,29 @@
<p>You can install this plugin from the command line with PyPI.</p>
<p>The following commands may be helpful; always refer to <a href="{{ plugin.homepage_url }}" target="_blank">the plugin's own documentation <i class="mdi mdi-launch"></i></a> and the <a href="https://netboxlabs.com/docs/netbox/en/stable/plugins/installation/" target="_blank">Installing a Plugin unit <i class="mdi mdi-launch"></i></a> of the NetBox documentation.</p>
<p>1. Enter the NetBox virtual environment and install the plugin package:</p>
<pre class="block">
source /opt/netbox/venv/bin/activate
pip install {{ plugin.slug }}
</pre>
<p>2. In /opt/netbox/netbox/netbox/configuration.py, add the plugin to the PLUGINS list:</p>
<pre class="block">
PLUGINS=
"{{ plugin.config_name }}",
]
</pre>
<p>3. Still from the NetBox virtual environment, run database migrations and collect static files:</p>
<pre class="block">
python3 /opt/netbox/netbox/netbox/manage.py migrate
python3 /opt/netbox/netbox/netbox/manage.py collectstatic
</pre>
<p>4. Restart the NetBox services to complete the plugin installation:</p>
<pre class="block">
sudo systemctl restart netbox netbox-rq
</pre>

View File

@ -0,0 +1,113 @@
{% extends 'generic/object.html' %}
{% load helpers %}
{% load form_helpers %}
{% load i18n %}
{% block title %}{{ plugin.title_short }}{% endblock %}
{% block object_identifier %}
{% endblock object_identifier %}
{% block breadcrumbs %}
<li class="breadcrumb-item"><a href="{% url 'core:plugin_list' %}">{% trans "Plugins" %}</a></li>
{% endblock breadcrumbs %}
{% block subtitle %}
<span class="text-secondary fs-5">
{% checkmark plugin.is_installed %}
{% if plugin.is_installed %}
v{{ plugin.installed_version }} {% trans "installed" %}
{% else %}
{% trans "Not installed" %}
{% endif %}
</span>
{% endblock %}
{% block controls %}{% endblock %}
{% block tabs %}
<ul class="nav nav-tabs" role="tablist">
<li class="nav-item" role="presentation">
<a class="nav-link active" id="overview-tab" data-bs-toggle="tab" data-bs-target="#overview" type="button" role="tab" aria-controls="edit-form" aria-selected="true">
{% trans "Overview" %}
</a>
</li>
{% if True or not plugin.is_local and 'commercial' not in settings.RELEASE.features %}
<li class="nav-item" role="presentation">
<button class="nav-link" id="install-tab" data-bs-toggle="tab" data-bs-target="#install" type="button" role="tab" aria-controls="object-list" aria-selected="false">
{% trans "Install" %}
</button>
</li>
{% endif %}
</ul>
{% endblock tabs %}
{% block content %}
<div class="tab-pane show active" id="overview" role="tabpanel" aria-labelledby="overview-tab">
<div class="row">
<div class="col col-6">
<div class="card">
<h5 class="card-header">{% trans "Plugin Details" %}</h5>
<table class="table table-hover attr-table">
<tr>
<th scope="row">{% trans "Name" %}</th>
<td>{{ plugin.title_short }}</td>
</tr>
<tr>
<th scope="row">{% trans "Summary" %}</th>
<td>{{ plugin.tag_line|placeholder }}</td>
</tr>
<tr>
<th scope="row">{% trans "Author" %}</th>
<td>{{ plugin.author.name|placeholder }}</td>
</tr>
<tr>
<th scope="row">{% trans "URL" %}</th>
<td>
{% if plugin.homepage_url %}
<a href="{{ plugin.homepage_url }}">{{ plugin.homepage_url }}</a>
{% else %}
{{ ''|placeholder }}
{% endif %}
</td>
</tr>
<tr>
<th scope="row">{% trans "License" %}</th>
<td>{{ plugin.license_type|placeholder }}</td>
</tr>
<tr>
<th scope="row">{% trans "Description" %}</th>
<td>{{ plugin.description_short|markdown }}</td>
</tr>
<tr>
<th scope="row">{% trans "Certified" %}</th>
<td>{% checkmark plugin.is_certified %}</td>
</tr>
<tr>
<th scope="row">{% trans "Local" %}</th>
<td>{% checkmark plugin.is_local %}</td>
</tr>
</table>
</div>
</div>
<div class="col col-6">
<div class="card">
<h5 class="card-header">{% trans "Version History" %}</h5>
<div class="htmx-container table-responsive" id="object_list">
{% include 'htmx/table.html' %}
</div>
</div>
</div>
</div>
</div>
{% if True or not plugin.is_local and 'commercial' not in settings.RELEASE.features %}
<div class="tab-pane" id="install" role="tabpanel" aria-labelledby="install-tab">
<div class="card">
<h5 class="card-header">{% trans "Local Installation Instructions" %}</h5>
<div class="card-body">
{% include 'core/inc/plugin_installation.html' %}
</div>
</div>
</div>
{% endif %}
{% endblock content %}

View File

@ -0,0 +1,16 @@
{% extends 'generic/object_list.html' %}
{% load buttons %}
{% load helpers %}
{% load i18n %}
{% load render_table from django_tables2 %}
{% block title %}{% trans "Plugins" %}{% endblock %}
{% block tabs %}
<ul class="nav nav-tabs px-3">
<li class="nav-item" role="presentation">
<a class="nav-link active" role="tab">{% trans "Plugins" %}</a>
</li>
</ul>
{% endblock tabs %}

View File

@ -78,16 +78,6 @@
</div>
</div>
{# Plugins #}
<div class="row mb-3">
<div class="col col-md-12">
<div class="card">
<h5 class="card-header">{% trans "Plugins" %}</h5>
{% render_table plugins_table %}
</div>
</div>
</div>
{# Configuration #}
<div class="row mb-3">
<div class="col col-md-12">