From ef432754ee322ad4046ec4d1dff0b0607eb247a3 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 8 Aug 2019 21:19:14 -0400 Subject: [PATCH 01/42] Started 2.7 branch --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 090122e37..55f45565d 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -12,7 +12,7 @@ from django.core.exceptions import ImproperlyConfigured # Environment setup # -VERSION = '2.6.3-dev' +VERSION = '2.7-beta1' # Hostname HOSTNAME = platform.node() From dccda62f2d52e71d4130daeda4c098bcbf12d956 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 8 Aug 2019 21:33:20 -0400 Subject: [PATCH 02/42] Closes #2745: Remove topology maps --- base_requirements.txt | 4 - docs/additional-features/topology-maps.md | 17 -- docs/installation/2-netbox.md | 4 +- mkdocs.yml | 1 - netbox/dcim/views.py | 4 +- netbox/extras/admin.py | 14 +- netbox/extras/api/serializers.py | 15 +- netbox/extras/api/urls.py | 3 - netbox/extras/api/views.py | 34 +--- netbox/extras/constants.py | 10 -- netbox/extras/filters.py | 20 +-- .../migrations/0024_remove_topology_maps.py | 16 ++ netbox/extras/models.py | 153 +----------------- netbox/netbox/views.py | 3 +- netbox/templates/dcim/site.html | 19 --- netbox/templates/home.html | 23 --- requirements.txt | 1 - 17 files changed, 26 insertions(+), 315 deletions(-) delete mode 100644 docs/additional-features/topology-maps.md create mode 100644 netbox/extras/migrations/0024_remove_topology_maps.py diff --git a/base_requirements.txt b/base_requirements.txt index f0f6cfe38..ca3f4ba6f 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -54,10 +54,6 @@ djangorestframework # https://github.com/axnsan12/drf-yasg drf-yasg[validation] -# Python interface to the graphviz graph rendering utility -# https://github.com/xflr6/graphviz -graphviz - # Simple markup language for rendering HTML # https://github.com/Python-Markdown/markdown # py-gfm requires Markdown<3.0 diff --git a/docs/additional-features/topology-maps.md b/docs/additional-features/topology-maps.md deleted file mode 100644 index 21bbe404d..000000000 --- a/docs/additional-features/topology-maps.md +++ /dev/null @@ -1,17 +0,0 @@ -# Topology Maps - -NetBox can generate simple topology maps from the physical network connections recorded in its database. First, you'll need to create a topology map definition under the admin UI at Extras > Topology Maps. - -Each topology map is associated with a site. A site can have multiple topology maps, which might each illustrate a different aspect of its infrastructure (for example, production versus backend infrastructure). - -To define the scope of a topology map, decide which devices you want to include. The map will only include interface connections with both points terminated on an included device. Specify the devices to include in the **device patterns** field by entering a list of [regular expressions](https://en.wikipedia.org/wiki/Regular_expression) matching device names. For example, if you wanted to include "mgmt-switch1" through "mgmt-switch99", you might use the regex `mgmt-switch\d+`. - -Each line of the **device patterns** field represents a hierarchical layer within the topology map. For example, you might map a traditional network with core, distribution, and access tiers like this: - -``` -core-switch-[abcd] -dist-switch\d -access-switch\d+;oob-switch\d+ -``` - -Note that you can combine multiple regexes onto one line using semicolons. The order in which regexes are listed on a line is significant: devices matching the first regex will be rendered first, and subsequent groups will be rendered to the right of those. diff --git a/docs/installation/2-netbox.md b/docs/installation/2-netbox.md index 30894670e..c825c3590 100644 --- a/docs/installation/2-netbox.md +++ b/docs/installation/2-netbox.md @@ -5,14 +5,14 @@ This section of the documentation discusses installing and configuring the NetBo **Ubuntu** ```no-highlight -# apt-get install -y python3 python3-pip python3-dev build-essential libxml2-dev libxslt1-dev libffi-dev graphviz libpq-dev libssl-dev redis-server zlib1g-dev +# apt-get install -y python3 python3-pip python3-dev build-essential libxml2-dev libxslt1-dev libffi-dev libpq-dev libssl-dev redis-server zlib1g-dev ``` **CentOS** ```no-highlight # yum install -y epel-release -# yum install -y gcc python36 python36-devel python36-setuptools libxml2-devel libxslt-devel libffi-devel graphviz openssl-devel redhat-rpm-config redis +# yum install -y gcc python36 python36-devel python36-setuptools libxml2-devel libxslt-devel libffi-devel openssl-devel redhat-rpm-config redis # easy_install-3.6 pip # ln -s /usr/bin/python36 /usr/bin/python3 ``` diff --git a/mkdocs.yml b/mkdocs.yml index 99f77d06c..9cbf38484 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -32,7 +32,6 @@ pages: - Context Data: 'additional-features/context-data.md' - Export Templates: 'additional-features/export-templates.md' - Graphs: 'additional-features/graphs.md' - - Topology Maps: 'additional-features/topology-maps.md' - Reports: 'additional-features/reports.md' - Webhooks: 'additional-features/webhooks.md' - Change Logging: 'additional-features/change-logging.md' diff --git a/netbox/dcim/views.py b/netbox/dcim/views.py index 5ddaf15ed..ef7666153 100644 --- a/netbox/dcim/views.py +++ b/netbox/dcim/views.py @@ -16,7 +16,7 @@ from django.utils.safestring import mark_safe from django.views.generic import View from circuits.models import Circuit -from extras.models import Graph, TopologyMap, GRAPH_TYPE_INTERFACE, GRAPH_TYPE_SITE +from extras.models import Graph, GRAPH_TYPE_INTERFACE, GRAPH_TYPE_SITE from extras.views import ObjectConfigContextView from ipam.models import Prefix, VLAN from ipam.tables import InterfaceIPAddressTable, InterfaceVLANTable @@ -207,14 +207,12 @@ class SiteView(PermissionRequiredMixin, View): 'vm_count': VirtualMachine.objects.filter(cluster__site=site).count(), } rack_groups = RackGroup.objects.filter(site=site).annotate(rack_count=Count('racks')) - topology_maps = TopologyMap.objects.filter(site=site) show_graphs = Graph.objects.filter(type=GRAPH_TYPE_SITE).exists() return render(request, 'dcim/site.html', { 'site': site, 'stats': stats, 'rack_groups': rack_groups, - 'topology_maps': topology_maps, 'show_graphs': show_graphs, }) diff --git a/netbox/extras/admin.py b/netbox/extras/admin.py index d93b04037..ca469d711 100644 --- a/netbox/extras/admin.py +++ b/netbox/extras/admin.py @@ -3,7 +3,7 @@ from django.contrib import admin from netbox.admin import admin_site from utilities.forms import LaxURLField -from .models import CustomField, CustomFieldChoice, CustomLink, Graph, ExportTemplate, TopologyMap, Webhook +from .models import CustomField, CustomFieldChoice, CustomLink, Graph, ExportTemplate, Webhook def order_content_types(field): @@ -137,15 +137,3 @@ class ExportTemplateForm(forms.ModelForm): class ExportTemplateAdmin(admin.ModelAdmin): list_display = ['name', 'content_type', 'description', 'mime_type', 'file_extension'] form = ExportTemplateForm - - -# -# Topology maps -# - -@admin.register(TopologyMap, site=admin_site) -class TopologyMapAdmin(admin.ModelAdmin): - list_display = ['name', 'slug', 'site'] - prepopulated_fields = { - 'slug': ['name'], - } diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index abf0d8cf5..7ab825ea5 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -10,8 +10,7 @@ from dcim.api.nested_serializers import ( from dcim.models import Device, DeviceRole, Platform, Rack, Region, Site from extras.constants import * from extras.models import ( - ConfigContext, ExportTemplate, Graph, ImageAttachment, ObjectChange, ReportResult, TopologyMap, - Tag + ConfigContext, ExportTemplate, Graph, ImageAttachment, ObjectChange, ReportResult, Tag, ) from tenancy.api.nested_serializers import NestedTenantSerializer, NestedTenantGroupSerializer from tenancy.models import Tenant, TenantGroup @@ -69,18 +68,6 @@ class ExportTemplateSerializer(ValidatedModelSerializer): ] -# -# Topology maps -# - -class TopologyMapSerializer(ValidatedModelSerializer): - site = NestedSiteSerializer() - - class Meta: - model = TopologyMap - fields = ['id', 'name', 'slug', 'site', 'device_patterns', 'description'] - - # # Tags # diff --git a/netbox/extras/api/urls.py b/netbox/extras/api/urls.py index c135280ea..ddfe2107c 100644 --- a/netbox/extras/api/urls.py +++ b/netbox/extras/api/urls.py @@ -26,9 +26,6 @@ router.register(r'graphs', views.GraphViewSet) # Export templates router.register(r'export-templates', views.ExportTemplateViewSet) -# Topology maps -router.register(r'topology-maps', views.TopologyMapViewSet) - # Tags router.register(r'tags', views.TagViewSet) diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index 44e010cd2..cbc851c4d 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -2,8 +2,7 @@ from collections import OrderedDict from django.contrib.contenttypes.models import ContentType from django.db.models import Count -from django.http import Http404, HttpResponse -from django.shortcuts import get_object_or_404 +from django.http import Http404 from rest_framework.decorators import action from rest_framework.exceptions import PermissionDenied from rest_framework.response import Response @@ -11,8 +10,7 @@ from rest_framework.viewsets import ReadOnlyModelViewSet, ViewSet from extras import filters from extras.models import ( - ConfigContext, CustomFieldChoice, ExportTemplate, Graph, ImageAttachment, ObjectChange, ReportResult, TopologyMap, - Tag, + ConfigContext, CustomFieldChoice, ExportTemplate, Graph, ImageAttachment, ObjectChange, ReportResult, Tag, ) from extras.reports import get_report, get_reports from utilities.api import FieldChoicesViewSet, IsAuthenticatedOrLoginNotRequired, ModelViewSet @@ -115,34 +113,6 @@ class ExportTemplateViewSet(ModelViewSet): filterset_class = filters.ExportTemplateFilter -# -# Topology maps -# - -class TopologyMapViewSet(ModelViewSet): - queryset = TopologyMap.objects.select_related('site') - serializer_class = serializers.TopologyMapSerializer - filterset_class = filters.TopologyMapFilter - - @action(detail=True) - def render(self, request, pk): - - tmap = get_object_or_404(TopologyMap, pk=pk) - img_format = 'png' - - try: - data = tmap.render(img_format=img_format) - except Exception as e: - return HttpResponse( - "There was an error generating the requested graph: %s" % e - ) - - response = HttpResponse(data, content_type='image/{}'.format(img_format)) - response['Content-Disposition'] = 'inline; filename="{}.{}"'.format(tmap.slug, img_format) - - return response - - # # Tags # diff --git a/netbox/extras/constants.py b/netbox/extras/constants.py index b72ae8c08..51add5a4c 100644 --- a/netbox/extras/constants.py +++ b/netbox/extras/constants.py @@ -134,16 +134,6 @@ TEMPLATE_LANGUAGE_CHOICES = ( (TEMPLATE_LANGUAGE_JINJA2, 'Jinja2'), ) -# Topology map types -TOPOLOGYMAP_TYPE_NETWORK = 1 -TOPOLOGYMAP_TYPE_CONSOLE = 2 -TOPOLOGYMAP_TYPE_POWER = 3 -TOPOLOGYMAP_TYPE_CHOICES = ( - (TOPOLOGYMAP_TYPE_NETWORK, 'Network'), - (TOPOLOGYMAP_TYPE_CONSOLE, 'Console'), - (TOPOLOGYMAP_TYPE_POWER, 'Power'), -) - # Change log actions OBJECTCHANGE_ACTION_CREATE = 1 OBJECTCHANGE_ACTION_UPDATE = 2 diff --git a/netbox/extras/filters.py b/netbox/extras/filters.py index 49e879fe4..bb8c78e2e 100644 --- a/netbox/extras/filters.py +++ b/netbox/extras/filters.py @@ -5,7 +5,7 @@ from django.db.models import Q from dcim.models import DeviceRole, Platform, Region, Site from tenancy.models import Tenant, TenantGroup from .constants import CF_FILTER_DISABLED, CF_FILTER_EXACT, CF_TYPE_BOOLEAN, CF_TYPE_SELECT -from .models import ConfigContext, CustomField, Graph, ExportTemplate, ObjectChange, Tag, TopologyMap +from .models import ConfigContext, CustomField, Graph, ExportTemplate, ObjectChange, Tag class CustomFieldFilter(django_filters.Filter): @@ -103,24 +103,6 @@ class TagFilter(django_filters.FilterSet): ) -class TopologyMapFilter(django_filters.FilterSet): - site_id = django_filters.ModelMultipleChoiceFilter( - field_name='site', - queryset=Site.objects.all(), - label='Site', - ) - site = django_filters.ModelMultipleChoiceFilter( - field_name='site__slug', - queryset=Site.objects.all(), - to_field_name='slug', - label='Site (slug)', - ) - - class Meta: - model = TopologyMap - fields = ['name', 'slug'] - - class ConfigContextFilter(django_filters.FilterSet): q = django_filters.CharFilter( method='search', diff --git a/netbox/extras/migrations/0024_remove_topology_maps.py b/netbox/extras/migrations/0024_remove_topology_maps.py new file mode 100644 index 000000000..c019f4cec --- /dev/null +++ b/netbox/extras/migrations/0024_remove_topology_maps.py @@ -0,0 +1,16 @@ +# Generated by Django 2.2 on 2019-08-09 01:26 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('extras', '0023_fix_tag_sequences'), + ] + + operations = [ + migrations.DeleteModel( + name='TopologyMap', + ), + ] diff --git a/netbox/extras/models.py b/netbox/extras/models.py index c5df5c2e5..68889dc33 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -7,17 +7,14 @@ from django.contrib.contenttypes.models import ContentType from django.contrib.postgres.fields import JSONField from django.core.validators import ValidationError from django.db import models -from django.db.models import F, Q from django.http import HttpResponse from django.template import Template, Context from django.urls import reverse -import graphviz from jinja2 import Environment from taggit.models import TagBase, GenericTaggedItemBase -from dcim.constants import CONNECTION_STATUS_CONNECTED from utilities.fields import ColorField -from utilities.utils import deepmerge, foreground_color, model_names_to_filter_dict +from utilities.utils import deepmerge, model_names_to_filter_dict from .constants import * from .querysets import ConfigContextQuerySet @@ -496,154 +493,6 @@ class ExportTemplate(models.Model): return response -# -# Topology maps -# - -class TopologyMap(models.Model): - name = models.CharField( - max_length=50, - unique=True - ) - slug = models.SlugField( - unique=True - ) - type = models.PositiveSmallIntegerField( - choices=TOPOLOGYMAP_TYPE_CHOICES, - default=TOPOLOGYMAP_TYPE_NETWORK - ) - site = models.ForeignKey( - to='dcim.Site', - on_delete=models.CASCADE, - related_name='topology_maps', - blank=True, - null=True - ) - device_patterns = models.TextField( - help_text='Identify devices to include in the diagram using regular ' - 'expressions, one per line. Each line will result in a new ' - 'tier of the drawing. Separate multiple regexes within a ' - 'line using semicolons. Devices will be rendered in the ' - 'order they are defined.' - ) - description = models.CharField( - max_length=100, - blank=True - ) - - class Meta: - ordering = ['name'] - - def __str__(self): - return self.name - - @property - def device_sets(self): - if not self.device_patterns: - return None - return [line.strip() for line in self.device_patterns.split('\n')] - - def render(self, img_format='png'): - - from dcim.models import Device - - # Construct the graph - if self.type == TOPOLOGYMAP_TYPE_NETWORK: - G = graphviz.Graph - else: - G = graphviz.Digraph - self.graph = G() - self.graph.graph_attr['ranksep'] = '1' - seen = set() - for i, device_set in enumerate(self.device_sets): - - subgraph = G(name='sg{}'.format(i)) - subgraph.graph_attr['rank'] = 'same' - subgraph.graph_attr['directed'] = 'true' - - # Add a pseudonode for each device_set to enforce hierarchical layout - subgraph.node('set{}'.format(i), label='', shape='none', width='0') - if i: - self.graph.edge('set{}'.format(i - 1), 'set{}'.format(i), style='invis') - - # Add each device to the graph - devices = [] - for query in device_set.strip(';').split(';'): # Split regexes on semicolons - devices += Device.objects.filter(name__regex=query).select_related('device_role') - # Remove duplicate devices - devices = [d for d in devices if d.id not in seen] - seen.update([d.id for d in devices]) - for d in devices: - bg_color = '#{}'.format(d.device_role.color) - fg_color = '#{}'.format(foreground_color(d.device_role.color)) - subgraph.node(d.name, style='filled', fillcolor=bg_color, fontcolor=fg_color, fontname='sans') - - # Add an invisible connection to each successive device in a set to enforce horizontal order - for j in range(0, len(devices) - 1): - subgraph.edge(devices[j].name, devices[j + 1].name, style='invis') - - self.graph.subgraph(subgraph) - - # Compile list of all devices - device_superset = Q() - for device_set in self.device_sets: - for query in device_set.split(';'): # Split regexes on semicolons - device_superset = device_superset | Q(name__regex=query) - devices = Device.objects.filter(*(device_superset,)) - - # Draw edges depending on graph type - if self.type == TOPOLOGYMAP_TYPE_NETWORK: - self.add_network_connections(devices) - elif self.type == TOPOLOGYMAP_TYPE_CONSOLE: - self.add_console_connections(devices) - elif self.type == TOPOLOGYMAP_TYPE_POWER: - self.add_power_connections(devices) - - return self.graph.pipe(format=img_format) - - def add_network_connections(self, devices): - - from circuits.models import CircuitTermination - from dcim.models import Interface - - # Add all interface connections to the graph - connected_interfaces = Interface.objects.select_related( - '_connected_interface__device' - ).filter( - Q(device__in=devices) | Q(_connected_interface__device__in=devices), - _connected_interface__isnull=False, - pk__lt=F('_connected_interface') - ) - for interface in connected_interfaces: - style = 'solid' if interface.connection_status == CONNECTION_STATUS_CONNECTED else 'dashed' - self.graph.edge(interface.device.name, interface.connected_endpoint.device.name, style=style) - - # Add all circuits to the graph - for termination in CircuitTermination.objects.filter(term_side='A', connected_endpoint__device__in=devices): - peer_termination = termination.get_peer_termination() - if (peer_termination is not None and peer_termination.interface is not None and - peer_termination.interface.device in devices): - self.graph.edge(termination.interface.device.name, peer_termination.interface.device.name, color='blue') - - def add_console_connections(self, devices): - - from dcim.models import ConsolePort - - # Add all console connections to the graph - for cp in ConsolePort.objects.filter(device__in=devices, connected_endpoint__device__in=devices): - style = 'solid' if cp.connection_status == CONNECTION_STATUS_CONNECTED else 'dashed' - self.graph.edge(cp.connected_endpoint.device.name, cp.device.name, style=style) - - def add_power_connections(self, devices): - - from dcim.models import PowerPort - - # Add all power connections to the graph - for pp in PowerPort.objects.filter(device__in=devices, _connected_poweroutlet__device__in=devices): - style = 'solid' if pp.connection_status == CONNECTION_STATUS_CONNECTED else 'dashed' - self.graph.edge(pp.connected_endpoint.device.name, pp.device.name, style=style) - - # # Image attachments # diff --git a/netbox/netbox/views.py b/netbox/netbox/views.py index 146bba6db..9d431be41 100644 --- a/netbox/netbox/views.py +++ b/netbox/netbox/views.py @@ -21,7 +21,7 @@ from dcim.tables import ( CableTable, DeviceDetailTable, DeviceTypeTable, PowerFeedTable, RackTable, RackGroupTable, SiteTable, VirtualChassisTable, ) -from extras.models import ObjectChange, ReportResult, TopologyMap +from extras.models import ObjectChange, ReportResult from ipam.filters import AggregateFilter, IPAddressFilter, PrefixFilter, VLANFilter, VRFFilter from ipam.models import Aggregate, IPAddress, Prefix, VLAN, VRF from ipam.tables import AggregateTable, IPAddressTable, PrefixTable, VLANTable, VRFTable @@ -222,7 +222,6 @@ class HomeView(View): return render(request, self.template_name, { 'search_form': SearchForm(), 'stats': stats, - 'topology_maps': TopologyMap.objects.filter(site__isnull=True), 'report_results': ReportResult.objects.order_by('-created')[:10], 'changelog': ObjectChange.objects.select_related('user', 'changed_object_type')[:50] }) diff --git a/netbox/templates/dcim/site.html b/netbox/templates/dcim/site.html index 0e38d2967..8af48968c 100644 --- a/netbox/templates/dcim/site.html +++ b/netbox/templates/dcim/site.html @@ -285,25 +285,6 @@ {% endif %} -
-
- Topology Maps -
- {% if topology_maps %} - - {% for tm in topology_maps %} - - - - - {% endfor %} -
{{ tm }}{{ tm.description }}
- {% else %} -
- None -
- {% endif %} -
{% include 'inc/modal.html' with modal_name='graphs' %} diff --git a/netbox/templates/home.html b/netbox/templates/home.html index 8d483568f..be63b19c5 100644 --- a/netbox/templates/home.html +++ b/netbox/templates/home.html @@ -259,29 +259,6 @@
-
-
- Global Topology Maps -
- {% if topology_maps and perms.extras.view_topologymap %} - - {% for tm in topology_maps %} - - - - - {% endfor %} -
{{ tm }}{{ tm.description }}
- {% elif perms.extras.view_topologymap %} -
- None found -
- {% else %} -
- No permission -
- {% endif %} -
Reports diff --git a/requirements.txt b/requirements.txt index 3ad165a4b..f2a5c84ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,6 @@ django-taggit-serializer==0.1.7 django-timezone-field==3.0 djangorestframework==3.9.4 drf-yasg[validation]==1.16.0 -graphviz==0.10.1 Jinja2==2.10.1 Markdown==2.6.11 netaddr==0.7.19 From 15b55f5e628f33926fc3f14262ecdf9b59fa8bb1 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 8 Aug 2019 21:35:59 -0400 Subject: [PATCH 03/42] Remove deprecated form_factor accessor on Interface and InterfaceTemplate --- netbox/dcim/api/serializers.py | 12 ++++-------- netbox/dcim/models.py | 32 -------------------------------- 2 files changed, 4 insertions(+), 40 deletions(-) diff --git a/netbox/dcim/api/serializers.py b/netbox/dcim/api/serializers.py index e9526fa41..19a80a809 100644 --- a/netbox/dcim/api/serializers.py +++ b/netbox/dcim/api/serializers.py @@ -241,12 +241,10 @@ class PowerOutletTemplateSerializer(ValidatedModelSerializer): class InterfaceTemplateSerializer(ValidatedModelSerializer): device_type = NestedDeviceTypeSerializer() type = ChoiceField(choices=IFACE_TYPE_CHOICES, required=False) - # TODO: Remove in v2.7 (backward-compatibility for form_factor) - form_factor = ChoiceField(choices=IFACE_TYPE_CHOICES, required=False) class Meta: model = InterfaceTemplate - fields = ['id', 'device_type', 'name', 'type', 'form_factor', 'mgmt_only'] + fields = ['id', 'device_type', 'name', 'type', 'mgmt_only'] class RearPortTemplateSerializer(ValidatedModelSerializer): @@ -437,8 +435,6 @@ class PowerPortSerializer(TaggitSerializer, ConnectedEndpointSerializer): class InterfaceSerializer(TaggitSerializer, ConnectedEndpointSerializer): device = NestedDeviceSerializer() type = ChoiceField(choices=IFACE_TYPE_CHOICES, required=False) - # TODO: Remove in v2.7 (backward-compatibility for form_factor) - form_factor = ChoiceField(choices=IFACE_TYPE_CHOICES, required=False) lag = NestedInterfaceSerializer(required=False, allow_null=True) mode = ChoiceField(choices=IFACE_MODE_CHOICES, required=False, allow_null=True) untagged_vlan = NestedVLANSerializer(required=False, allow_null=True) @@ -454,9 +450,9 @@ class InterfaceSerializer(TaggitSerializer, ConnectedEndpointSerializer): class Meta: model = Interface fields = [ - 'id', 'device', 'name', 'type', 'form_factor', 'enabled', 'lag', 'mtu', 'mac_address', 'mgmt_only', - 'description', 'connected_endpoint_type', 'connected_endpoint', 'connection_status', 'cable', 'mode', - 'untagged_vlan', 'tagged_vlans', 'tags', 'count_ipaddresses', + 'id', 'device', 'name', 'type', 'enabled', 'lag', 'mtu', 'mac_address', 'mgmt_only', 'description', + 'connected_endpoint_type', 'connected_endpoint', 'connection_status', 'cable', 'mode', 'untagged_vlan', + 'tagged_vlans', 'tags', 'count_ipaddresses', ] # TODO: This validation should be handled by Interface.clean() diff --git a/netbox/dcim/models.py b/netbox/dcim/models.py index 4c22c9549..fb8ab1dc5 100644 --- a/netbox/dcim/models.py +++ b/netbox/dcim/models.py @@ -1181,22 +1181,6 @@ class InterfaceTemplate(ComponentTemplateModel): def __str__(self): return self.name - # TODO: Remove in v2.7 - @property - def form_factor(self): - """ - Backward-compatibility for form_factor - """ - return self.type - - # TODO: Remove in v2.7 - @form_factor.setter - def form_factor(self, value): - """ - Backward-compatibility for form_factor - """ - self.type = value - def instantiate(self, device): return Interface( device=device, @@ -2342,22 +2326,6 @@ class Interface(CableTermination, ComponentModel): object_data=serialize_object(self) ).save() - # TODO: Remove in v2.7 - @property - def form_factor(self): - """ - Backward-compatibility for form_factor - """ - return self.type - - # TODO: Remove in v2.7 - @form_factor.setter - def form_factor(self, value): - """ - Backward-compatibility for form_factor - """ - self.type = value - @property def connected_endpoint(self): if self._connected_interface: From a6511632ad96788b4d0aebde0890ed973112e323 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Wed, 14 Aug 2019 20:28:23 -0400 Subject: [PATCH 04/42] closes #3407 - Added code coverage reporting to the CI pipeline --- .gitignore | 1 + .travis.yml | 1 + CHANGELOG.md | 8 ++++++++ scripts/cibuild.sh | 10 +++++++++- 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d859bad28..e02e72710 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ fabfile.py gunicorn_config.py .DS_Store .vscode +.coverage diff --git a/.travis.yml b/.travis.yml index 29fa87b64..872121c21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ python: install: - pip install -r requirements.txt - pip install pycodestyle + - pip install coverage before_script: - psql --version - psql -U postgres -c 'SELECT version();' diff --git a/CHANGELOG.md b/CHANGELOG.md index eaf8cd930..5216d32a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +v2.7.0 (FUTURE) + +## Housekeeping + +* [#3407](https://github.com/netbox-community/netbox/issues/3407) - Added code coverage reporting to the CI pipeline + +--- + v2.6.3 (FUTURE) ## Bug Fixes diff --git a/scripts/cibuild.sh b/scripts/cibuild.sh index c0a49b5ef..282000b0a 100755 --- a/scripts/cibuild.sh +++ b/scripts/cibuild.sh @@ -41,13 +41,21 @@ sed -i -e "s/ALLOWED_HOSTS = \[\]/ALLOWED_HOSTS = \['*'\]/g" $CONFIG sed -i -e "s/SECRET_KEY = ''/SECRET_KEY = 'netboxci'/g" $CONFIG # Run NetBox tests -./netbox/manage.py test netbox/ +coverage run --source="netbox/" netbox/manage.py test netbox/ RC=$? if [[ $RC != 0 ]]; then echo -e "\n$(info) one or more tests failed, failing build." EXIT=$RC fi +# Show code coverage report +coverage report --skip-covered --omit *migrations* +RC=$? +if [[ $RC != 0 ]]; then + echo -e "\n$(info) failed to generate code coverage report." + EXIT=$RC +fi + # Show build duration END=$(date +%s) echo "$(info) exiting with code $EXIT after $(($END - $START)) seconds." From 480db83f3950a783fcf008e0e5cba86c409d1a4f Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 5 Sep 2019 10:25:44 -0400 Subject: [PATCH 05/42] Renumber remove_topology_maps migration --- ...024_remove_topology_maps.py => 0026_remove_topology_maps.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename netbox/extras/migrations/{0024_remove_topology_maps.py => 0026_remove_topology_maps.py} (82%) diff --git a/netbox/extras/migrations/0024_remove_topology_maps.py b/netbox/extras/migrations/0026_remove_topology_maps.py similarity index 82% rename from netbox/extras/migrations/0024_remove_topology_maps.py rename to netbox/extras/migrations/0026_remove_topology_maps.py index c019f4cec..40b36d3b5 100644 --- a/netbox/extras/migrations/0024_remove_topology_maps.py +++ b/netbox/extras/migrations/0026_remove_topology_maps.py @@ -6,7 +6,7 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ('extras', '0023_fix_tag_sequences'), + ('extras', '0025_objectchange_time_index'), ] operations = [ From 9e258dd31ee1771b61e873c6495647bf45b16fee Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Fri, 6 Sep 2019 11:46:35 -0500 Subject: [PATCH 06/42] Fixes: #2902 - Implements systemd (#3017) * Closes #2902 - Migrate to systemd from supervisord * Closes #2902 - Migrate to systemd from supervisord * Update systemd unit and environment file * Add gunicorn.conf * Update documentation and CHANGELOG. Moved parameters around on service file * Update Gitignore --- .gitignore | 3 + CHANGELOG.md | 7 +- contrib/gunicorn.conf | 22 +++++ contrib/netbox-rq.service | 24 +++++ contrib/netbox.env | 15 ++++ contrib/netbox.service | 24 +++++ contrib/netbox@.service | 24 +++++ docs/installation/3-http-daemon.md | 105 +++++++++++++++++----- docs/installation/index.md | 2 + docs/installation/migrating-to-systemd.md | 105 ++++++++++++++++++++++ docs/installation/upgrading.md | 9 +- 11 files changed, 311 insertions(+), 29 deletions(-) create mode 100644 contrib/gunicorn.conf create mode 100644 contrib/netbox-rq.service create mode 100644 contrib/netbox.env create mode 100644 contrib/netbox.service create mode 100644 contrib/netbox@.service create mode 100644 docs/installation/migrating-to-systemd.md diff --git a/.gitignore b/.gitignore index 38154baaa..2183b50a3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ fabfile.py *.swp gunicorn_config.py +gunicorn.conf +netbox.log +netbox.pid .DS_Store .vscode .coverage diff --git a/CHANGELOG.md b/CHANGELOG.md index 70feb952e..ad372eb62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ -<<<<<<< HEAD v2.7.0 (FUTURE) +## Enhancements + +* [#2902](https://github.com/digitalocean/netbox/issues/2902) - Replace supervisord with systemd + ## Housekeeping * [#3407](https://github.com/netbox-community/netbox/issues/3407) - Added code coverage reporting to the CI pipeline @@ -51,7 +54,7 @@ v2.6.2 (2019-08-02) ## Bug Fixes -* [#3018](https://github.com/netbox-community/netbox/issues/3018) - Components connected via a cable must have an equal number of positions +* [#3018](https://github.com/netbox-community/netbox/issues/301 8) - Components connected via a cable must have an equal number of positions * [#3289](https://github.com/netbox-community/netbox/issues/3289) - Prevent position from being nullified when moving a device to a new rack * [#3293](https://github.com/netbox-community/netbox/issues/3293) - Enable filtering device components by multiple device IDs * [#3315](https://github.com/netbox-community/netbox/issues/3315) - Enable filtering devices/interfaces by multiple MAC addresses diff --git a/contrib/gunicorn.conf b/contrib/gunicorn.conf new file mode 100644 index 000000000..9ecf81008 --- /dev/null +++ b/contrib/gunicorn.conf @@ -0,0 +1,22 @@ +# Bind is the ip and port that the Netbox WSGI should bind to +# +bind='127.0.0.1:8001' + +# Workers is the number of workers that GUnicorn should spawn. +# Workers should be: cores * 2 + 1. So if you have 8 cores, it would be 17. +# +workers=3 + +# Threads +# The number of threads for handling requests +# +threads=3 + +# Timeout is the timeout between gunicorn receiving a request and returning a response (or failing with a 500 error) +# +timeout=120 + +# ErrorLog +# ErrorLog is the logfile for the ErrorLog +# +errorlog='/opt/netbox/netbox.log' \ No newline at end of file diff --git a/contrib/netbox-rq.service b/contrib/netbox-rq.service new file mode 100644 index 000000000..4b364d6bc --- /dev/null +++ b/contrib/netbox-rq.service @@ -0,0 +1,24 @@ +[Unit] +Description=Netbox RQ Worker +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +EnvironmentFile=/etc/sysconfig/netbox.env + +User=www-data +Group=www-data + +WorkingDirectory=${WorkingDirectory} + +ExecStart=/usr/bin/python3 ${WorkingDirectory}/netbox/manage.py rqworker + +Restart=on-failure +RestartSec=30 +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/contrib/netbox.env b/contrib/netbox.env new file mode 100644 index 000000000..c64cc587a --- /dev/null +++ b/contrib/netbox.env @@ -0,0 +1,15 @@ +# Name is the Process Name +# +Name = 'Netbox' + +# ConfigPath is the path to the gunicorn config file. +# +ConfigPath=/opt/netbox/gunicorn.conf + +# WorkingDirectory is the Working Directory for Netbox. +# +WorkingDirectory=/opt/netbox/ + +# PidPath is the path to the pid for the netbox WSGI +# +PidPath=/opt/netbox/netbox.pid \ No newline at end of file diff --git a/contrib/netbox.service b/contrib/netbox.service new file mode 100644 index 000000000..76fb0e8ac --- /dev/null +++ b/contrib/netbox.service @@ -0,0 +1,24 @@ +[Unit] +Description=Netbox WSGI +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +EnvironmentFile=/etc/sysconfig/netbox.env + +User=www-data +Group=www-data +PIDFile=${PidPath} +WorkingDirectory=${WorkingDirectory} + +ExecStart=/usr/local/bin/gunicorn --pid ${PidPath} --pythonpath ${WorkingDirectory}/netbox --config ${ConfigPath} netbox.wsgi + +Restart=on-failure +RestartSec=30 +PrivateTmp=true + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/contrib/netbox@.service b/contrib/netbox@.service new file mode 100644 index 000000000..8616ccc52 --- /dev/null +++ b/contrib/netbox@.service @@ -0,0 +1,24 @@ +[Unit] +Description=Netbox WSGI +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +EnvironmentFile=/etc/sysconfig/netbox.%i.env + +User=www-data +Group=www-data +PIDFile=${PidPath} +WorkingDirectory=${WorkingDirectory} + +ExecStart=/usr/local/bin/gunicorn --pid ${PidPath} --pythonpath ${WorkingDirectory}/netbox --config ${ConfigPath} netbox.wsgi + +Restart=on-failure +RestartSec=30 +PrivateTmp=true + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/docs/installation/3-http-daemon.md b/docs/installation/3-http-daemon.md index dcf16101e..209afee0c 100644 --- a/docs/installation/3-http-daemon.md +++ b/docs/installation/3-http-daemon.md @@ -1,4 +1,4 @@ -We'll set up a simple WSGI front end using [gunicorn](http://gunicorn.org/) for the purposes of this guide. For web servers, we provide example configurations for both [nginx](https://www.nginx.com/resources/wiki/) and [Apache](http://httpd.apache.org/docs/2.4). (You are of course free to use whichever combination of HTTP and WSGI services you'd like.) We'll also use [supervisord](http://supervisord.org/) to enable service persistence. +We'll set up a simple WSGI front end using [gunicorn](http://gunicorn.org/) for the purposes of this guide. For web servers, we provide example configurations for both [nginx](https://www.nginx.com/resources/wiki/) and [Apache](http://httpd.apache.org/docs/2.4). (You are of course free to use whichever combination of HTTP and WSGI services you'd like.) We'll use systemd to enable service persistence. !!! info For the sake of brevity, only Ubuntu 18.04 instructions are provided here, but this sort of web server and WSGI configuration is not unique to NetBox. Please consult your distribution's documentation for assistance if needed. @@ -108,42 +108,99 @@ Install gunicorn: # pip3 install gunicorn ``` -Save the following configuration in the root netbox installation path as `gunicorn_config.py` (e.g. `/opt/netbox/gunicorn_config.py` per our example installation). Be sure to verify the location of the gunicorn executable on your server (e.g. `which gunicorn`) and to update the `pythonpath` variable if needed. If using CentOS/RHEL, change the username from `www-data` to `nginx` or `apache`. +# systemd configuration + +Copy or link contrib/netbox.service and contrib/netbox-rq.service to /etc/systemd/system/netbox.service and /etc/systemd/system/netbox-rq.service ```no-highlight -command = '/usr/bin/gunicorn' -pythonpath = '/opt/netbox/netbox' -bind = '127.0.0.1:8001' -workers = 3 -user = 'www-data' +# cp contrib/netbox.service to /etc/systemd/system/netbox.service +# cp contrib/netbox-rq.service to /etc/systemd/system/netbox-rq.service ``` -# supervisord Installation - -Install supervisor: +Edit /etc/systemd/system/netbox.service and /etc/systemd/system/netbox-rq.service. Be sure to verify the location of the gunicorn executable on your server (e.g. `which gunicorn`). If using CentOS/RHEL, change the username from `www-data` to `nginx` or `apache`: ```no-highlight -# apt-get install -y supervisor +/usr/local/bin/gunicorn --pid ${PidPath} --pythonpath ${WorkingDirectory}/netbox --config ${ConfigPath} netbox.wsgi ``` -Save the following as `/etc/supervisor/conf.d/netbox.conf`. Update the `command` and `directory` paths as needed. If using CentOS/RHEL, change the username from `www-data` to `nginx` or `apache`. - ```no-highlight -[program:netbox] -command = gunicorn -c /opt/netbox/gunicorn_config.py netbox.wsgi -directory = /opt/netbox/netbox/ -user = www-data - -[program:netbox-rqworker] -command = python3 /opt/netbox/netbox/manage.py rqworker -directory = /opt/netbox/netbox/ -user = www-data +User=www-data +Group=www-data ``` -Then, restart the supervisor service to detect and run the gunicorn service: +Copy contrib/netbox.env to /etc/sysconfig/netbox.env ```no-highlight -# service supervisor restart +# cp contrib/netbox.env to /etc/sysconfig/netbox.env +``` + +Edit /etc/sysconfig/netbox.env and change the settings as required. Update the `WorkingDirectory` variable if needed. + +```no-highlight +# Name is the Process Name +# +Name = 'Netbox' + +# ConfigPath is the path to the gunicorn config file. +# +ConfigPath=/opt/netbox/gunicorn.conf + +# WorkingDirectory is the Working Directory for Netbox. +# +WorkingDirectory=/opt/netbox/ + +# PidPath is the path to the pid for the netbox WSGI +# +PidPath=/var/run/netbox.pid +``` + +Copy contrib/gunicorn.conf to gunicorn.conf + +```no-highlight +# cp contrib/gunicorn.conf to gunicorn.conf +``` + +Edit gunicorn.conf and change the settings as required. + +``` +# Bind is the ip and port that the Netbox WSGI should bind to +# +bind='127.0.0.1:8001' + +# Workers is the number of workers that GUnicorn should spawn. +# Workers should be: cores * 2 + 1. So if you have 8 cores, it would be 17. +# +workers=3 + +# Threads +# The number of threads for handling requests +# Threads should be: cores * 2 + 1. So if you have 4 cores, it would be 9. +# +threads=3 + +# Timeout is the timeout between gunicorn receiving a request and returning a response (or failing with a 500 error) +# +timeout=120 + +# ErrorLog +# ErrorLog is the logfile for the ErrorLog +# +errorlog='/opt/netbox/netbox.log' +``` + +Then, restart the systemd daemon service to detect the netbox service and start the netbox service: + +```no-highlight +# systemctl daemon-reload +# systemctl start netbox.service +# systemctl enable netbox.service +``` + +If using webhooks, also start the Redis worker: + +```no-highlight +# systemctl start netbox-rq.service +# systemctl enable netbox-rq.service ``` At this point, you should be able to connect to the nginx HTTP service at the server name or IP address you provided. If you are unable to connect, check that the nginx service is running and properly configured. If you receive a 502 (bad gateway) error, this indicates that gunicorn is misconfigured or not running. diff --git a/docs/installation/index.md b/docs/installation/index.md index 54daa62e3..4962eb7d0 100644 --- a/docs/installation/index.md +++ b/docs/installation/index.md @@ -12,3 +12,5 @@ The following sections detail how to set up a new instance of NetBox: If you are upgrading from an existing installation, please consult the [upgrading guide](upgrading.md). NetBox v2.5 and later requires Python 3.5 or higher. Please see the instructions for [migrating to Python 3](migrating-to-python3.md) if you are still using Python 2. + +Netbox v2.5.9 and later moved to using systemd instead of supervisord. Please see the instructions for [migrating to systemd](migrating-to-systemd.md) if you are still using supervisord. diff --git a/docs/installation/migrating-to-systemd.md b/docs/installation/migrating-to-systemd.md new file mode 100644 index 000000000..8f207c6ad --- /dev/null +++ b/docs/installation/migrating-to-systemd.md @@ -0,0 +1,105 @@ +# Migration + +Migration is not required, as supervisord will still continue to function. + +## Ubuntu + +### Remove supervisord: + +```no-highlight +# apt-get remove -y supervisord +``` + +### systemd configuration: + +Copy or link contrib/netbox.service and contrib/netbox-rq.service to /etc/systemd/system/netbox.service and /etc/systemd/system/netbox-rq.service + +```no-highlight +# cp contrib/netbox.service /etc/systemd/system/netbox.service +# cp contrib/netbox-rq.service /etc/systemd/system/netbox-rq.service +``` + +Edit /etc/systemd/system/netbox.service and /etc/systemd/system/netbox-rq.service. Be sure to verify the location of the gunicorn executable on your server (e.g. `which gunicorn`). If using CentOS/RHEL. Change the username from `www-data` to `nginx` or `apache`: + +```no-highlight +/usr/local/bin/gunicorn --pid ${PidPath} --pythonpath ${WorkingDirectory}/netbox --config ${ConfigPath} netbox.wsgi +``` + +```no-highlight +User=www-data +Group=www-data +``` + +Copy contrib/netbox.env to /etc/sysconfig/netbox.env + +```no-highlight +# cp contrib/netbox.env /etc/sysconfig/netbox.env +``` + +Edit /etc/sysconfig/netbox.env and change the settings as required. Update the `WorkingDirectory` variable if needed. + +```no-highlight +# Name is the Process Name +# +Name = 'Netbox' + +# ConfigPath is the path to the gunicorn config file. +# +ConfigPath=/opt/netbox/gunicorn.conf + +# WorkingDirectory is the Working Directory for Netbox. +# +WorkingDirectory=/opt/netbox/ + +# PidPath is the path to the pid for the netbox WSGI +# +PidPath=/var/run/netbox.pid +``` + +Copy contrib/gunicorn.conf to gunicorn.conf + +```no-highlight +# cp contrib/gunicorn.conf to gunicorn.conf +``` + +Edit gunicorn.conf and change the settings as required. + +``` +# Bind is the ip and port that the Netbox WSGI should bind to +# +bind='127.0.0.1:8001' + +# Workers is the number of workers that GUnicorn should spawn. +# Workers should be: cores * 2 + 1. So if you have 8 cores, it would be 17. +# +workers=3 + +# Threads +# The number of threads for handling requests +# +threads=3 + +# Timeout is the timeout between gunicorn receiving a request and returning a response (or failing with a 500 error) +# +timeout=120 + +# ErrorLog +# ErrorLog is the logfile for the ErrorLog +# +errorlog='/opt/netbox/netbox.log' +``` + +Then, restart the systemd daemon service to detect the netbox service and start the netbox service: + +```no-highlight +# systemctl daemon-reload +# systemctl start netbox.service +# systemctl enable netbox.service +``` + +If using webhooks, also start the Redis worker: + +```no-highlight +# systemctl start netbox-rq.service +# systemctl enable netbox-rq.service +``` \ No newline at end of file diff --git a/docs/installation/upgrading.md b/docs/installation/upgrading.md index c13582e96..3de4b319b 100644 --- a/docs/installation/upgrading.md +++ b/docs/installation/upgrading.md @@ -80,14 +80,17 @@ This script: # Restart the WSGI Service -Finally, restart the WSGI service to run the new code. If you followed this guide for the initial installation, this is done using `supervisorctl`: +Finally, restart the WSGI service to run the new code. If you followed this guide for the initial installation, this is done using `systemctl: ```no-highlight -# sudo supervisorctl restart netbox +# sudo systemctl restart netbox ``` If using webhooks, also restart the Redis worker: ```no-highlight -# sudo supervisorctl restart netbox-rqworker +# sudo systemctl restart netbox-rqworker ``` + +!!! note + It's possible you are still using supervisord instead of the linux native systemd. If you are still using supervisord you can restart the services by either restarting supervisord or by using supervisorctl to restart netbox. \ No newline at end of file From f8fdca49680fe0de8b635628a2149d2920fb1f28 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 5 Sep 2019 17:23:56 -0400 Subject: [PATCH 07/42] Initial work on JSON/YAML-based DeviceType import --- netbox/dcim/forms.py | 33 +++++++++- netbox/dcim/urls.py | 3 +- netbox/dcim/views.py | 10 ++- netbox/templates/dcim/device_import.html | 2 +- .../templates/dcim/device_import_child.html | 2 +- netbox/templates/secrets/secret_import.html | 2 +- .../templates/utilities/obj_bulk_import.html | 60 ++++++++++++++++++ netbox/templates/utilities/obj_import.html | 30 +-------- netbox/utilities/forms.py | 37 ++++++++++- netbox/utilities/views.py | 62 ++++++++++++++++++- 10 files changed, 202 insertions(+), 39 deletions(-) create mode 100644 netbox/templates/utilities/obj_bulk_import.html diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 4abbcdd71..08aaf02b0 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -22,7 +22,8 @@ from utilities.forms import ( APISelect, APISelectMultiple, add_blank_choice, ArrayFieldSelectMultiple, BootstrapMixin, BulkEditForm, BulkEditNullBooleanSelect, ChainedFieldsMixin, ChainedModelChoiceField, ColorSelect, CommentField, ComponentForm, ConfirmationForm, CSVChoiceField, ExpandableNameField, FilterChoiceField, FlexibleModelChoiceField, JSONField, - SelectWithPK, SmallTextarea, SlugField, StaticSelect2, StaticSelect2Multiple, BOOLEAN_WITH_BLANK_CHOICES + SelectWithPK, SmallTextarea, SlugField, StaticSelect2, StaticSelect2Multiple, MultiObjectField, + BOOLEAN_WITH_BLANK_CHOICES, ) from virtualization.models import Cluster, ClusterGroup from .constants import * @@ -831,6 +832,36 @@ class DeviceTypeCSVForm(forms.ModelForm): } +class InterfaceTemplateImportForm(BootstrapMixin, forms.ModelForm): + name_pattern = ExpandableNameField( + label='Name' + ) + + class Meta: + model = InterfaceTemplate + fields = [ + 'type', 'mgmt_only', + ] + + +class DeviceTypeImportForm(forms.ModelForm): + manufacturer = forms.ModelChoiceField( + queryset=Manufacturer.objects.all(), + to_field_name='name' + ) + interfaces = MultiObjectField( + form=InterfaceTemplateImportForm, + required=False + ) + + class Meta: + model = DeviceType + fields = [ + 'manufacturer', 'model', 'slug', 'part_number', 'u_height', 'is_full_depth', 'subdevice_role', + 'interfaces', + ] + + class DeviceTypeBulkEditForm(BootstrapMixin, AddRemoveTagsForm, CustomFieldBulkEditForm): pk = forms.ModelMultipleChoiceField( queryset=DeviceType.objects.all(), diff --git a/netbox/dcim/urls.py b/netbox/dcim/urls.py index ae1f05757..a9d24dcb1 100644 --- a/netbox/dcim/urls.py +++ b/netbox/dcim/urls.py @@ -82,7 +82,8 @@ urlpatterns = [ # Device types path(r'device-types/', views.DeviceTypeListView.as_view(), name='devicetype_list'), path(r'device-types/add/', views.DeviceTypeCreateView.as_view(), name='devicetype_add'), - path(r'device-types/import/', views.DeviceTypeBulkImportView.as_view(), name='devicetype_import'), + # path(r'device-types/import/', views.DeviceTypeBulkImportView.as_view(), name='devicetype_import'), + path(r'device-types/import/', views.DeviceTypeImportView.as_view(), name='devicetype_import'), path(r'device-types/edit/', views.DeviceTypeBulkEditView.as_view(), name='devicetype_bulk_edit'), path(r'device-types/delete/', views.DeviceTypeBulkDeleteView.as_view(), name='devicetype_bulk_delete'), path(r'device-types//', views.DeviceTypeView.as_view(), name='devicetype'), diff --git a/netbox/dcim/views.py b/netbox/dcim/views.py index fe98c93f3..f78ce700b 100644 --- a/netbox/dcim/views.py +++ b/netbox/dcim/views.py @@ -13,6 +13,7 @@ from django.urls import reverse from django.utils.html import escape from django.utils.http import is_safe_url from django.utils.safestring import mark_safe +from django.utils.text import slugify from django.views.generic import View from circuits.models import Circuit @@ -25,7 +26,7 @@ from utilities.paginator import EnhancedPaginator from utilities.utils import csv_format from utilities.views import ( BulkComponentCreateView, BulkDeleteView, BulkEditView, BulkImportView, ComponentCreateView, GetReturnURLMixin, - ObjectDeleteView, ObjectEditView, ObjectListView, + ObjectImportView, ObjectDeleteView, ObjectEditView, ObjectListView, ) from virtualization.models import VirtualMachine from . import filters, forms, tables @@ -654,6 +655,13 @@ class DeviceTypeDeleteView(PermissionRequiredMixin, ObjectDeleteView): default_return_url = 'dcim:devicetype_list' +class DeviceTypeImportView(PermissionRequiredMixin, ObjectImportView): + permission_required = 'dcim.add_devicetype' + model = DeviceType + model_form = forms.DeviceTypeImportForm + default_return_url = 'dcim:devicetype_import' + + class DeviceTypeBulkImportView(PermissionRequiredMixin, BulkImportView): permission_required = 'dcim.add_devicetype' model_form = forms.DeviceTypeCSVForm diff --git a/netbox/templates/dcim/device_import.html b/netbox/templates/dcim/device_import.html index 85ebfbbc6..2f3a0ea8f 100644 --- a/netbox/templates/dcim/device_import.html +++ b/netbox/templates/dcim/device_import.html @@ -1,4 +1,4 @@ -{% extends 'utilities/obj_import.html' %} +{% extends 'utilities/obj_bulk_import.html' %} {% block tabs %} {% include 'dcim/inc/device_import_header.html' %} diff --git a/netbox/templates/dcim/device_import_child.html b/netbox/templates/dcim/device_import_child.html index 406d239d7..346196382 100644 --- a/netbox/templates/dcim/device_import_child.html +++ b/netbox/templates/dcim/device_import_child.html @@ -1,4 +1,4 @@ -{% extends 'utilities/obj_import.html' %} +{% extends 'utilities/obj_bulk_import.html' %} {% block tabs %} {% include 'dcim/inc/device_import_header.html' with active_tab='child_import' %} diff --git a/netbox/templates/secrets/secret_import.html b/netbox/templates/secrets/secret_import.html index 169f16b11..bf2f06ae9 100644 --- a/netbox/templates/secrets/secret_import.html +++ b/netbox/templates/secrets/secret_import.html @@ -1,4 +1,4 @@ -{% extends 'utilities/obj_import.html' %} +{% extends 'utilities/obj_bulk_import.html' %} {% load static %} {% block content %} diff --git a/netbox/templates/utilities/obj_bulk_import.html b/netbox/templates/utilities/obj_bulk_import.html new file mode 100644 index 000000000..97b093a02 --- /dev/null +++ b/netbox/templates/utilities/obj_bulk_import.html @@ -0,0 +1,60 @@ +{% extends '_base.html' %} +{% load helpers %} +{% load form_helpers %} + +{% block content %} +

{% block title %}{{ obj_type|bettertitle }} Bulk Import{% endblock %}

+{% block tabs %}{% endblock %} +
+
+ {% if form.non_field_errors %} +
+
Errors
+
+ {{ form.non_field_errors }} +
+
+ {% endif %} +
+ {% csrf_token %} + {% render_form form %} +
+
+ + {% if return_url %} + Cancel + {% endif %} +
+
+
+
+
+ {% if fields %} +

CSV Format

+ + + + + + + {% for name, field in fields.items %} + + + + + + {% endfor %} +
FieldRequiredDescription
{{ name }}{% if field.required %}{% endif %} + {{ field.help_text|default:field.label }} + {% if field.choices %} +
Choices: {{ field|example_choices }} + {% elif field|widget_type == 'dateinput' %} +
Format: YYYY-MM-DD + {% elif field|widget_type == 'checkboxinput' %} +
Specify "true" or "false" + {% endif %} +
+ {% endif %} +
+
+{% endblock %} diff --git a/netbox/templates/utilities/obj_import.html b/netbox/templates/utilities/obj_import.html index 89621a3c3..d2da2bc94 100644 --- a/netbox/templates/utilities/obj_import.html +++ b/netbox/templates/utilities/obj_import.html @@ -6,7 +6,7 @@

{% block title %}{{ obj_type|bettertitle }} Import{% endblock %}

{% block tabs %}{% endblock %}
-
+
{% if form.non_field_errors %}
Errors
@@ -28,33 +28,5 @@
-
- {% if fields %} -

CSV Format

- - - - - - - {% for name, field in fields.items %} - - - - - - {% endfor %} -
FieldRequiredDescription
{{ name }}{% if field.required %}{% endif %} - {{ field.help_text|default:field.label }} - {% if field.choices %} -
Choices: {{ field|example_choices }} - {% elif field|widget_type == 'dateinput' %} -
Format: YYYY-MM-DD - {% elif field|widget_type == 'checkboxinput' %} -
Specify "true" or "false" - {% endif %} -
- {% endif %} -
{% endblock %} diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index 1d4671bf6..a083e968a 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -6,8 +6,7 @@ from io import StringIO from django import forms from django.conf import settings from django.contrib.postgres.forms.jsonb import JSONField as _JSONField, InvalidJSONInput -from django.db.models import Count -from django.urls import reverse_lazy +from django.utils.html import mark_safe from mptt.forms import TreeNodeMultipleChoiceField from .constants import * @@ -554,6 +553,24 @@ class SlugField(forms.SlugField): self.widget.attrs['slug-source'] = slug_source +class MultiObjectField(forms.Field): + """ + Use this field to relay data to another form for validation. Useful when importing data via JSON/YAML. + """ + def __init__(self, form, *args, **kwargs): + self.form = form + super().__init__(*args, **kwargs) + + def clean(self, value): + + for obj in value: + subform = self.form(obj) + if not subform.is_valid(): + raise forms.ValidationError(mark_safe(subform.errors.items())) + + return value + + class FilterChoiceIterator(forms.models.ModelChoiceIterator): def __iter__(self): @@ -721,3 +738,19 @@ class BulkEditForm(forms.Form): # Copy any nullable fields defined in Meta if hasattr(self.Meta, 'nullable_fields'): self.nullable_fields = self.Meta.nullable_fields + + +class ImportForm(BootstrapMixin, forms.Form): + """ + Generic form for creating an object from JSON/YAML data + """ + data = forms.CharField( + widget=forms.Textarea + ) + format = forms.ChoiceField( + choices=( + ('json', 'JSON'), + ('yaml', 'YAML') + ), + initial='yaml' + ) diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index bbc58f134..179d47820 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -1,4 +1,6 @@ +import json import sys +import yaml from copy import deepcopy from django.conf import settings @@ -26,7 +28,7 @@ from extras.querysets import CustomFieldQueryset from utilities.forms import BootstrapMixin, CSVDataField from utilities.utils import csv_format from .error_handlers import handle_protectederror -from .forms import ConfirmationForm +from .forms import ConfirmationForm, ImportForm from .paginator import EnhancedPaginator @@ -393,6 +395,62 @@ class BulkCreateView(GetReturnURLMixin, View): }) +class ObjectImportView(GetReturnURLMixin, View): + """ + Import a single object (YAML or JSON format). + """ + model = None + model_form = None + template_name = 'utilities/obj_import.html' + + def create_object(self, data): + raise NotImplementedError("View must implement object creation logic") + + def get(self, request): + + form = ImportForm() + + return render(request, self.template_name, { + 'form': form, + 'obj_type': self.model._meta.verbose_name, + 'return_url': self.get_return_url(request), + }) + + def post(self, request): + + form = ImportForm(request.POST) + + if form.is_valid(): + + # Process object data + if form.cleaned_data['format'] == 'json': + data = json.loads(form.cleaned_data['data']) + else: + data = yaml.load(form.cleaned_data['data']) + + # Initialize model form + model_form = self.model_form(data) + + if model_form.is_valid(): + + obj = model_form.save(commit=False) + # assert False, model_form.cleaned_data['interfaces'] + + messages.success(request, "Imported object: {}".format(obj)) + return redirect(self.get_return_url(request)) + + else: + # Replicate model form errors for display + for field, err in model_form.errors.items(): + form.add_error(None, "{}: {}".format(field, err)) + + return render(request, self.template_name, { + 'form': form, + 'obj_type': self.model._meta.verbose_name, + 'return_url': self.get_return_url(request), + }) + + class BulkImportView(GetReturnURLMixin, View): """ Import objects in bulk (CSV format). @@ -404,7 +462,7 @@ class BulkImportView(GetReturnURLMixin, View): """ model_form = None table = None - template_name = 'utilities/obj_import.html' + template_name = 'utilities/obj_bulk_import.html' widget_attrs = {} def _import_form(self, *args, **kwargs): From 60b70b6c7b1f55cf5a65ccd8c3af9b5d3533ce89 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 20 Sep 2019 15:16:14 -0400 Subject: [PATCH 08/42] Add RearPortTemplate power_port field --- netbox/dcim/forms.py | 5 +++++ netbox/utilities/views.py | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 9f10cef2d..df289d32f 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -919,6 +919,11 @@ class InterfaceTemplateImportForm(ComponentTemplateImportForm): class FrontPortTemplateImportForm(ComponentTemplateImportForm): + power_port = forms.ModelChoiceField( + queryset=RearPortTemplate.objects.all(), + to_field_name='name', + required=False + ) class Meta: model = FrontPortTemplate diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index a7a308b09..06267392a 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -437,7 +437,9 @@ class ObjectImportView(GetReturnURLMixin, View): with transaction.atomic(): obj = model_form.save() - messages.success(request, "Imported object: {}".format(obj)) + messages.success(request, mark_safe('Imported object: {}'.format( + obj.get_absolute_url(), obj + ))) return redirect(self.get_return_url(request)) else: From 5049c6c0a1a91641f04a50e7bcec9b3b9678a743 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 20 Sep 2019 15:57:44 -0400 Subject: [PATCH 09/42] Add test for DeviceType import --- netbox/dcim/forms.py | 9 ++-- netbox/dcim/tests/test_forms.py | 94 +++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index df289d32f..02bc61857 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -966,14 +966,14 @@ class DeviceTypeImportForm(forms.ModelForm): form=InterfaceTemplateImportForm, required=False ) - front_ports = MultiObjectField( - form=FrontPortTemplateImportForm, - required=False - ) rear_ports = MultiObjectField( form=RearPortTemplateImportForm, required=False ) + front_ports = MultiObjectField( + form=FrontPortTemplateImportForm, + required=False + ) class Meta: model = DeviceType @@ -994,7 +994,6 @@ class DeviceTypeImportForm(forms.ModelForm): data.update({ 'device_type': instance.pk }) - print(data) form = field.form(data) if form.is_valid(): form.save() diff --git a/netbox/dcim/tests/test_forms.py b/netbox/dcim/tests/test_forms.py index 2f333ea69..c0b94541a 100644 --- a/netbox/dcim/tests/test_forms.py +++ b/netbox/dcim/tests/test_forms.py @@ -8,6 +8,100 @@ def get_id(model, slug): return model.objects.get(slug=slug).id +DEVICETYPE_DATA = { + 'manufacturer': 'Generic', + 'model': 'TEST-1000', + 'slug': 'test-1000', + 'u_height': 2, + 'console_ports': [ + {'name': 'Console Port 1'}, + {'name': 'Console Port 2'}, + {'name': 'Console Port 3'}, + ], + 'console_server_ports': [ + {'name': 'Console Server Port 1'}, + {'name': 'Console Server Port 2'}, + {'name': 'Console Server Port 3'}, + ], + 'power_ports': [ + {'name': 'Power Port 1'}, + {'name': 'Power Port 2'}, + {'name': 'Power Port 3'}, + ], + 'power_outlets': [ + {'name': 'Power Outlet 1', 'power_port': 'Power Port 1', 'feed_leg': POWERFEED_LEG_A}, + {'name': 'Power Outlet 2', 'power_port': 'Power Port 1', 'feed_leg': POWERFEED_LEG_A}, + {'name': 'Power Outlet 3', 'power_port': 'Power Port 1', 'feed_leg': POWERFEED_LEG_A}, + ], + 'interfaces': [ + {'name': 'Interface 1', 'type': IFACE_TYPE_1GE_FIXED, 'mgmt_only': True}, + {'name': 'Interface 2', 'type': IFACE_TYPE_1GE_FIXED}, + {'name': 'Interface 3', 'type': IFACE_TYPE_1GE_FIXED}, + ], + 'rear_ports': [ + {'name': 'Rear Port 1', 'type': PORT_TYPE_8P8C}, + {'name': 'Rear Port 2', 'type': PORT_TYPE_8P8C}, + {'name': 'Rear Port 3', 'type': PORT_TYPE_8P8C}, + ], + 'front_ports': [ + {'name': 'Front Port 1', 'type': PORT_TYPE_8P8C, 'rear_port': 'Rear Port 1'}, + {'name': 'Front Port 2', 'type': PORT_TYPE_8P8C, 'rear_port': 'Rear Port 2'}, + {'name': 'Front Port 3', 'type': PORT_TYPE_8P8C, 'rear_port': 'Rear Port 3'}, + ] +} + + +class DeviceTypeImportTestCase(TestCase): + + def setUp(self): + + Manufacturer(name='Generic', slug='generic').save() + + def test_import_devicetype_yaml(self): + + form = DeviceTypeImportForm(DEVICETYPE_DATA) + + self.assertTrue(form.is_valid(), "Form validation failed: {}".format(form.errors)) + + form.save() + dt = DeviceType.objects.get(model='TEST-1000') + + # Verify all of the components were created + self.assertEqual(dt.consoleport_templates.count(), 3) + cp1 = ConsolePortTemplate.objects.first() + self.assertEqual(cp1.name, 'Console Port 1') + + self.assertEqual(dt.consoleserverport_templates.count(), 3) + csp1 = ConsoleServerPortTemplate.objects.first() + self.assertEqual(csp1.name, 'Console Server Port 1') + + self.assertEqual(dt.powerport_templates.count(), 3) + pp1 = PowerPortTemplate.objects.first() + self.assertEqual(pp1.name, 'Power Port 1') + + self.assertEqual(dt.poweroutlet_templates.count(), 3) + po1 = PowerOutletTemplate.objects.first() + self.assertEqual(po1.name, 'Power Outlet 1') + self.assertEqual(po1.power_port, pp1) + self.assertEqual(po1.feed_leg, POWERFEED_LEG_A) + + self.assertEqual(dt.interface_templates.count(), 4) + iface1 = Interface.objects.first() + self.assertEqual(iface1.name, 'Interface 1') + self.assertEqual(iface1.type, IFACE_TYPE_1GE_FIXED) + self.assertTrue(iface1.mgmt_only) + + self.assertEqual(dt.rearport_templates.count(), 3) + rp1 = FrontPortTemplate.objects.first() + self.assertEqual(rp1.name, 'Rear Port 1') + + self.assertEqual(dt.frontport_templates.count(), 3) + fp1 = FrontPortTemplate.objects.first() + self.assertEqual(fp1.name, 'Front Port 1') + self.assertEqual(fp1.rear_port, rp1) + self.assertEqual(fp1.rear_port_position, 1) + + class DeviceTestCase(TestCase): fixtures = ['dcim', 'ipam'] From 15b2a7eab0612ab43eda6f17f37b4eabe4b34097 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 24 Sep 2019 15:58:23 -0400 Subject: [PATCH 10/42] Fix form rendering; enable toggling of redirect to imported object --- netbox/dcim/forms.py | 2 +- netbox/templates/utilities/obj_import.html | 5 +++-- netbox/utilities/views.py | 10 +++++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 02bc61857..1af04a567 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -941,7 +941,7 @@ class RearPortTemplateImportForm(ComponentTemplateImportForm): ] -class DeviceTypeImportForm(forms.ModelForm): +class DeviceTypeImportForm(BootstrapMixin, forms.ModelForm): manufacturer = forms.ModelChoiceField( queryset=Manufacturer.objects.all(), to_field_name='name' diff --git a/netbox/templates/utilities/obj_import.html b/netbox/templates/utilities/obj_import.html index d2da2bc94..d0ba99295 100644 --- a/netbox/templates/utilities/obj_import.html +++ b/netbox/templates/utilities/obj_import.html @@ -15,12 +15,13 @@
{% endif %} -
+ {% csrf_token %} {% render_form form %}
- + + {% if return_url %} Cancel {% endif %} diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index 06267392a..3a9b19c76 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -440,7 +440,15 @@ class ObjectImportView(GetReturnURLMixin, View): messages.success(request, mark_safe('Imported object: {}'.format( obj.get_absolute_url(), obj ))) - return redirect(self.get_return_url(request)) + + if '_addanother' in request.POST: + return redirect(request.get_full_path()) + + return_url = form.cleaned_data.get('return_url') + if return_url is not None and is_safe_url(url=return_url, allowed_hosts=request.get_host()): + return redirect(return_url) + else: + return redirect(self.get_return_url(request, obj)) else: # Replicate model form errors for display From 2621f17cdef78d70d8ab1b1ddbba468eed25da92 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 24 Sep 2019 16:03:05 -0400 Subject: [PATCH 11/42] Remove legacy CSV-based DeviceType import --- netbox/dcim/forms.py | 25 ------------------------- netbox/dcim/urls.py | 1 - netbox/dcim/views.py | 7 ------- 3 files changed, 33 deletions(-) diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 1af04a567..9f465414d 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -829,31 +829,6 @@ class DeviceTypeForm(BootstrapMixin, CustomFieldForm): } -class DeviceTypeCSVForm(forms.ModelForm): - manufacturer = forms.ModelChoiceField( - queryset=Manufacturer.objects.all(), - required=True, - to_field_name='name', - help_text='Manufacturer name', - error_messages={ - 'invalid_choice': 'Manufacturer not found.', - } - ) - subdevice_role = CSVChoiceField( - choices=SUBDEVICE_ROLE_CHOICES, - required=False, - help_text='Parent/child status' - ) - - class Meta: - model = DeviceType - fields = DeviceType.csv_headers - help_texts = { - 'model': 'Model name', - 'slug': 'URL-friendly slug', - } - - class ComponentTemplateImportForm(BootstrapMixin, forms.ModelForm): def clean_device_type(self): diff --git a/netbox/dcim/urls.py b/netbox/dcim/urls.py index d7ba0cb0e..8c5a72727 100644 --- a/netbox/dcim/urls.py +++ b/netbox/dcim/urls.py @@ -82,7 +82,6 @@ urlpatterns = [ # Device types path(r'device-types/', views.DeviceTypeListView.as_view(), name='devicetype_list'), path(r'device-types/add/', views.DeviceTypeCreateView.as_view(), name='devicetype_add'), - # path(r'device-types/import/', views.DeviceTypeBulkImportView.as_view(), name='devicetype_import'), path(r'device-types/import/', views.DeviceTypeImportView.as_view(), name='devicetype_import'), path(r'device-types/edit/', views.DeviceTypeBulkEditView.as_view(), name='devicetype_bulk_edit'), path(r'device-types/delete/', views.DeviceTypeBulkDeleteView.as_view(), name='devicetype_bulk_delete'), diff --git a/netbox/dcim/views.py b/netbox/dcim/views.py index 7bd8868f3..56119a775 100644 --- a/netbox/dcim/views.py +++ b/netbox/dcim/views.py @@ -663,13 +663,6 @@ class DeviceTypeImportView(PermissionRequiredMixin, ObjectImportView): default_return_url = 'dcim:devicetype_import' -class DeviceTypeBulkImportView(PermissionRequiredMixin, BulkImportView): - permission_required = 'dcim.add_devicetype' - model_form = forms.DeviceTypeCSVForm - table = tables.DeviceTypeTable - default_return_url = 'dcim:devicetype_list' - - class DeviceTypeBulkEditView(PermissionRequiredMixin, BulkEditView): permission_required = 'dcim.change_devicetype' queryset = DeviceType.objects.prefetch_related('manufacturer').annotate(instance_count=Count('instances')) From 30ee232654bc7d9220f92c74fda9f15d12802059 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 24 Sep 2019 16:13:52 -0400 Subject: [PATCH 12/42] Move JSON/YAML data valdiation to ImportForm --- netbox/utilities/forms.py | 26 ++++++++++++++++++++++++-- netbox/utilities/views.py | 10 +--------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index af4e92dd0..dae9abda3 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -2,11 +2,11 @@ import csv import json import re from io import StringIO +import yaml from django import forms from django.conf import settings from django.contrib.postgres.forms.jsonb import JSONField as _JSONField, InvalidJSONInput -from django.utils.html import mark_safe from mptt.forms import TreeNodeMultipleChoiceField from .constants import * @@ -747,7 +747,8 @@ class ImportForm(BootstrapMixin, forms.Form): Generic form for creating an object from JSON/YAML data """ data = forms.CharField( - widget=forms.Textarea + widget=forms.Textarea, + help_text="Enter object data in JSON or YAML format." ) format = forms.ChoiceField( choices=( @@ -756,3 +757,24 @@ class ImportForm(BootstrapMixin, forms.Form): ), initial='yaml' ) + + def clean(self): + + data = self.cleaned_data['data'] + format = self.cleaned_data['format'] + + # Process JSON/YAML data + if format == 'json': + try: + self.cleaned_data['data'] = json.loads(data) + except json.decoder.JSONDecodeError as err: + raise forms.ValidationError({ + 'data': "Invalid JSON data: {}".format(err) + }) + else: + try: + self.cleaned_data['data'] = yaml.load(data) + except yaml.scanner.ScannerError as err: + raise forms.ValidationError({ + 'data': "Invalid YAML data: {}".format(err) + }) diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index 3a9b19c76..b5406e145 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -420,18 +420,10 @@ class ObjectImportView(GetReturnURLMixin, View): def post(self, request): form = ImportForm(request.POST) - if form.is_valid(): - # Process object data - if form.cleaned_data['format'] == 'json': - data = json.loads(form.cleaned_data['data']) - else: - data = yaml.load(form.cleaned_data['data']) - # Initialize model form - model_form = self.model_form(data) - + model_form = self.model_form(form.cleaned_data['data']) if model_form.is_valid(): with transaction.atomic(): From 0615d368f2b3d0077a6f335d90adcb063414ad3c Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 24 Sep 2019 16:51:59 -0400 Subject: [PATCH 13/42] Force validation of individual objects within a MultiObjectField --- netbox/dcim/forms.py | 21 ++++++++++----------- netbox/utilities/forms.py | 8 ++++++++ netbox/utilities/views.py | 9 +++++++-- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 9f465414d..6c145e9fa 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -848,7 +848,7 @@ class ConsolePortTemplateImportForm(ComponentTemplateImportForm): class Meta: model = ConsolePortTemplate fields = [ - 'device_type', 'name', + 'name', ] @@ -857,7 +857,7 @@ class ConsoleServerPortTemplateImportForm(ComponentTemplateImportForm): class Meta: model = ConsoleServerPortTemplate fields = [ - 'device_type', 'name', + 'name', ] @@ -866,7 +866,7 @@ class PowerPortTemplateImportForm(ComponentTemplateImportForm): class Meta: model = PowerPortTemplate fields = [ - 'device_type', 'name', 'maximum_draw', 'allocated_draw', + 'name', 'maximum_draw', 'allocated_draw', ] @@ -880,7 +880,7 @@ class PowerOutletTemplateImportForm(ComponentTemplateImportForm): class Meta: model = PowerOutletTemplate fields = [ - 'device_type', 'name', 'power_port', 'feed_leg', + 'name', 'power_port', 'feed_leg', ] @@ -889,7 +889,7 @@ class InterfaceTemplateImportForm(ComponentTemplateImportForm): class Meta: model = InterfaceTemplate fields = [ - 'device_type', 'name', 'type', 'mgmt_only', + 'name', 'type', 'mgmt_only', ] @@ -903,7 +903,7 @@ class FrontPortTemplateImportForm(ComponentTemplateImportForm): class Meta: model = FrontPortTemplate fields = [ - 'device_type', 'name', 'type', 'rear_port', 'rear_port_position', + 'name', 'type', 'rear_port', 'rear_port_position', ] @@ -912,7 +912,7 @@ class RearPortTemplateImportForm(ComponentTemplateImportForm): class Meta: model = RearPortTemplate fields = [ - 'device_type', 'name', 'type', 'positions', + 'name', 'type', 'positions', ] @@ -966,12 +966,11 @@ class DeviceTypeImportForm(BootstrapMixin, forms.ModelForm): for field_name, field in self.fields.items(): if isinstance(field, MultiObjectField): for data in self.cleaned_data[field_name]: - data.update({ - 'device_type': instance.pk - }) form = field.form(data) if form.is_valid(): - form.save() + component = form.save(commit=False) + component.device_type = instance + component.save() return instance diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index dae9abda3..10390ca5e 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -570,6 +570,14 @@ class MultiObjectField(forms.Field): if value is None: return list() + for i, obj_data in enumerate(value, start=1): + form = self.form(obj_data) + if not form.is_valid(): + errors = [ + "Object {} {}: {}".format(i, field, errors) for field, errors in form.errors.items() + ] + raise forms.ValidationError(errors) + return value diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index b5406e145..fe39263b1 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -443,9 +443,14 @@ class ObjectImportView(GetReturnURLMixin, View): return redirect(self.get_return_url(request, obj)) else: + # Replicate model form errors for display - for field, err in model_form.errors.items(): - form.add_error(None, "{}: {}".format(field, err)) + for field, errors in model_form.errors.items(): + for err in errors: + if field == '__all__': + form.add_error(None, err) + else: + form.add_error(None, "{}: {}".format(field, err)) return render(request, self.template_name, { 'form': form, From 47f1febfc96a54c543ba58b507caa6bfaebe66e6 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 25 Sep 2019 16:06:09 -0400 Subject: [PATCH 14/42] Capture import form field default values --- netbox/utilities/views.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index fe39263b1..b46a50ef3 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -423,7 +423,17 @@ class ObjectImportView(GetReturnURLMixin, View): if form.is_valid(): # Initialize model form - model_form = self.model_form(form.cleaned_data['data']) + data = form.cleaned_data['data'] + model_form = self.model_form(data) + + # Assign default values for any fields which were not specified. We have to do this manually because passing + # 'initial=' to the form on initialization merely sets default values for the widgets. Since widgets are not + # used for YAML/JSON import, we first bind the imported data normally, then update the form's data with the + # applicable field defaults as needed prior to form validation. + for field_name, field in model_form.fields.items(): + if field_name not in data and hasattr(field, 'initial'): + model_form.data[field_name] = field.initial + if model_form.is_valid(): with transaction.atomic(): From 5f3528cf744033a823d6bbccd3c90d0185f95ffb Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 25 Sep 2019 16:19:22 -0400 Subject: [PATCH 15/42] Capture MultiObjectField default form field values --- netbox/utilities/forms.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index 10390ca5e..41d577c68 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -571,7 +571,15 @@ class MultiObjectField(forms.Field): return list() for i, obj_data in enumerate(value, start=1): + + # Bind object data to form form = self.form(obj_data) + + # Assign default values for required fields that have not been defined + for field_name, field in form.fields.items(): + if field_name not in obj_data and hasattr(field, 'initial'): + form.data[field_name] = field.initial + if not form.is_valid(): errors = [ "Object {} {}: {}".format(i, field, errors) for field, errors in form.errors.items() From 36d4f0d259920cd11a0877877d9ef5f794379065 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 25 Sep 2019 16:39:04 -0400 Subject: [PATCH 16/42] Fix typo --- netbox/dcim/tests/test_forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/dcim/tests/test_forms.py b/netbox/dcim/tests/test_forms.py index c0b94541a..aed7ee9aa 100644 --- a/netbox/dcim/tests/test_forms.py +++ b/netbox/dcim/tests/test_forms.py @@ -92,7 +92,7 @@ class DeviceTypeImportTestCase(TestCase): self.assertTrue(iface1.mgmt_only) self.assertEqual(dt.rearport_templates.count(), 3) - rp1 = FrontPortTemplate.objects.first() + rp1 = RearPortTemplate.objects.first() self.assertEqual(rp1.name, 'Rear Port 1') self.assertEqual(dt.frontport_templates.count(), 3) From edc1b52f65286b2e2127cb9d283a4906169d68c9 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 27 Sep 2019 16:51:12 -0400 Subject: [PATCH 17/42] Adopted a different approach to importing related objects --- netbox/dcim/forms.py | 246 ++++++++++++++------------------ netbox/dcim/tests/test_forms.py | 24 +++- netbox/dcim/views.py | 12 +- netbox/utilities/forms.py | 33 ----- netbox/utilities/views.py | 30 +++- 5 files changed, 167 insertions(+), 178 deletions(-) diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index c2818e267..ea663fbcc 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -24,8 +24,7 @@ from utilities.forms import ( APISelect, APISelectMultiple, add_blank_choice, ArrayFieldSelectMultiple, BootstrapMixin, BulkEditForm, BulkEditNullBooleanSelect, ChainedFieldsMixin, ChainedModelChoiceField, ColorSelect, CommentField, ComponentForm, ConfirmationForm, CSVChoiceField, ExpandableNameField, FilterChoiceField, FlexibleModelChoiceField, JSONField, - SelectWithPK, SmallTextarea, SlugField, StaticSelect2, StaticSelect2Multiple, MultiObjectField, - BOOLEAN_WITH_BLANK_CHOICES, + SelectWithPK, SmallTextarea, SlugField, StaticSelect2, StaticSelect2Multiple, BOOLEAN_WITH_BLANK_CHOICES, ) from virtualization.models import Cluster, ClusterGroup from .constants import * @@ -829,126 +828,11 @@ class DeviceTypeForm(BootstrapMixin, CustomFieldForm): } -class ComponentTemplateImportForm(BootstrapMixin, forms.ModelForm): - - def clean_device_type(self): - - data = self.cleaned_data['device_type'] - - # Limit fields referencing other components to the parent DeviceType - for field_name, field in self.fields.items(): - if isinstance(field, forms.ModelChoiceField) and not field_name == 'device_type': - field.queryset = field.queryset.filter(device_type=data) - - return data - - -class ConsolePortTemplateImportForm(ComponentTemplateImportForm): - - class Meta: - model = ConsolePortTemplate - fields = [ - 'name', - ] - - -class ConsoleServerPortTemplateImportForm(ComponentTemplateImportForm): - - class Meta: - model = ConsoleServerPortTemplate - fields = [ - 'name', - ] - - -class PowerPortTemplateImportForm(ComponentTemplateImportForm): - - class Meta: - model = PowerPortTemplate - fields = [ - 'name', 'maximum_draw', 'allocated_draw', - ] - - -class PowerOutletTemplateImportForm(ComponentTemplateImportForm): - power_port = forms.ModelChoiceField( - queryset=PowerPortTemplate.objects.all(), - to_field_name='name', - required=False - ) - - class Meta: - model = PowerOutletTemplate - fields = [ - 'name', 'power_port', 'feed_leg', - ] - - -class InterfaceTemplateImportForm(ComponentTemplateImportForm): - - class Meta: - model = InterfaceTemplate - fields = [ - 'name', 'type', 'mgmt_only', - ] - - -class FrontPortTemplateImportForm(ComponentTemplateImportForm): - power_port = forms.ModelChoiceField( - queryset=RearPortTemplate.objects.all(), - to_field_name='name', - required=False - ) - - class Meta: - model = FrontPortTemplate - fields = [ - 'name', 'type', 'rear_port', 'rear_port_position', - ] - - -class RearPortTemplateImportForm(ComponentTemplateImportForm): - - class Meta: - model = RearPortTemplate - fields = [ - 'name', 'type', 'positions', - ] - - class DeviceTypeImportForm(BootstrapMixin, forms.ModelForm): manufacturer = forms.ModelChoiceField( queryset=Manufacturer.objects.all(), to_field_name='name' ) - console_ports = MultiObjectField( - form=ConsolePortTemplateImportForm, - required=False - ) - console_server_ports = MultiObjectField( - form=ConsoleServerPortTemplateImportForm, - required=False - ) - power_ports = MultiObjectField( - form=PowerPortTemplateImportForm, - required=False - ) - power_outlets = MultiObjectField( - form=PowerOutletTemplateImportForm, - required=False - ) - interfaces = MultiObjectField( - form=InterfaceTemplateImportForm, - required=False - ) - rear_ports = MultiObjectField( - form=RearPortTemplateImportForm, - required=False - ) - front_ports = MultiObjectField( - form=FrontPortTemplateImportForm, - required=False - ) class Meta: model = DeviceType @@ -956,24 +840,6 @@ class DeviceTypeImportForm(BootstrapMixin, forms.ModelForm): 'manufacturer', 'model', 'slug', 'part_number', 'u_height', 'is_full_depth', 'subdevice_role', ] - def save(self, commit=True): - - instance = super().save(commit) - - if commit: - - # Save related components - for field_name, field in self.fields.items(): - if isinstance(field, MultiObjectField): - for data in self.cleaned_data[field_name]: - form = field.form(data) - if form.is_valid(): - component = form.save(commit=False) - component.device_type = instance - component.save() - - return instance - class DeviceTypeBulkEditForm(BootstrapMixin, AddRemoveTagsForm, CustomFieldBulkEditForm): pk = forms.ModelMultipleChoiceField( @@ -1334,6 +1200,116 @@ class DeviceBayTemplateCreateForm(ComponentForm): ) +# +# Component template import forms +# + +class ComponentTemplateImportForm(BootstrapMixin, forms.ModelForm): + + def __init__(self, device_type, data=None, *args, **kwargs): + + # Must pass the parent DeviceType on form initialization + data.update({ + 'device_type': device_type.pk, + }) + print(data) + + super().__init__(data, *args, **kwargs) + + def clean_device_type(self): + + data = self.cleaned_data['device_type'] + + # Limit fields referencing other components to the parent DeviceType + for field_name, field in self.fields.items(): + if isinstance(field, forms.ModelChoiceField) and field_name != 'device_type': + field.queryset = field.queryset.filter(device_type=data) + + return data + + +class ConsolePortTemplateImportForm(ComponentTemplateImportForm): + + class Meta: + model = ConsolePortTemplate + fields = [ + 'device_type', 'name', + ] + + +class ConsoleServerPortTemplateImportForm(ComponentTemplateImportForm): + + class Meta: + model = ConsoleServerPortTemplate + fields = [ + 'device_type', 'name', + ] + + +class PowerPortTemplateImportForm(ComponentTemplateImportForm): + + class Meta: + model = PowerPortTemplate + fields = [ + 'device_type', 'name', 'maximum_draw', 'allocated_draw', + ] + + +class PowerOutletTemplateImportForm(ComponentTemplateImportForm): + power_port = forms.ModelChoiceField( + queryset=PowerPortTemplate.objects.all(), + to_field_name='name', + required=False + ) + + class Meta: + model = PowerOutletTemplate + fields = [ + 'device_type', 'name', 'power_port', 'feed_leg', + ] + + +class InterfaceTemplateImportForm(ComponentTemplateImportForm): + + class Meta: + model = InterfaceTemplate + fields = [ + 'device_type', 'name', 'type', 'mgmt_only', + ] + + +class FrontPortTemplateImportForm(ComponentTemplateImportForm): + power_port = forms.ModelChoiceField( + queryset=RearPortTemplate.objects.all(), + to_field_name='name', + required=False + ) + + class Meta: + model = FrontPortTemplate + fields = [ + 'device_type', 'name', 'type', 'rear_port', 'rear_port_position', + ] + + +class RearPortTemplateImportForm(ComponentTemplateImportForm): + + class Meta: + model = RearPortTemplate + fields = [ + 'device_type', 'name', 'type', 'positions', + ] + + +class DeviceBayTemplateImportForm(ComponentTemplateImportForm): + + class Meta: + model = DeviceBayTemplate + fields = [ + 'device_type', 'name', + ] + + # # Device roles # diff --git a/netbox/dcim/tests/test_forms.py b/netbox/dcim/tests/test_forms.py index aed7ee9aa..d9cf10fdb 100644 --- a/netbox/dcim/tests/test_forms.py +++ b/netbox/dcim/tests/test_forms.py @@ -13,22 +13,22 @@ DEVICETYPE_DATA = { 'model': 'TEST-1000', 'slug': 'test-1000', 'u_height': 2, - 'console_ports': [ + 'console-ports': [ {'name': 'Console Port 1'}, {'name': 'Console Port 2'}, {'name': 'Console Port 3'}, ], - 'console_server_ports': [ + 'console-server-ports': [ {'name': 'Console Server Port 1'}, {'name': 'Console Server Port 2'}, {'name': 'Console Server Port 3'}, ], - 'power_ports': [ + 'power-ports': [ {'name': 'Power Port 1'}, {'name': 'Power Port 2'}, {'name': 'Power Port 3'}, ], - 'power_outlets': [ + 'power-outlets': [ {'name': 'Power Outlet 1', 'power_port': 'Power Port 1', 'feed_leg': POWERFEED_LEG_A}, {'name': 'Power Outlet 2', 'power_port': 'Power Port 1', 'feed_leg': POWERFEED_LEG_A}, {'name': 'Power Outlet 3', 'power_port': 'Power Port 1', 'feed_leg': POWERFEED_LEG_A}, @@ -38,16 +38,21 @@ DEVICETYPE_DATA = { {'name': 'Interface 2', 'type': IFACE_TYPE_1GE_FIXED}, {'name': 'Interface 3', 'type': IFACE_TYPE_1GE_FIXED}, ], - 'rear_ports': [ + 'rear-ports': [ {'name': 'Rear Port 1', 'type': PORT_TYPE_8P8C}, {'name': 'Rear Port 2', 'type': PORT_TYPE_8P8C}, {'name': 'Rear Port 3', 'type': PORT_TYPE_8P8C}, ], - 'front_ports': [ + 'front-ports': [ {'name': 'Front Port 1', 'type': PORT_TYPE_8P8C, 'rear_port': 'Rear Port 1'}, {'name': 'Front Port 2', 'type': PORT_TYPE_8P8C, 'rear_port': 'Rear Port 2'}, {'name': 'Front Port 3', 'type': PORT_TYPE_8P8C, 'rear_port': 'Rear Port 3'}, - ] + ], + 'device-bays': [ + {'name': 'Device Bay 1'}, + {'name': 'Device Bay 2'}, + {'name': 'Device Bay 3'}, + ], } @@ -67,6 +72,7 @@ class DeviceTypeImportTestCase(TestCase): dt = DeviceType.objects.get(model='TEST-1000') # Verify all of the components were created + # TODO: The creation of components now occurs in the view rather than the form self.assertEqual(dt.consoleport_templates.count(), 3) cp1 = ConsolePortTemplate.objects.first() self.assertEqual(cp1.name, 'Console Port 1') @@ -101,6 +107,10 @@ class DeviceTypeImportTestCase(TestCase): self.assertEqual(fp1.rear_port, rp1) self.assertEqual(fp1.rear_port_position, 1) + self.assertEqual(dt.devicebay_templates.count(), 3) + db1 = DeviceBayTemplate.objects.first() + self.assertEqual(db1.name, 'Device Bay 1') + class DeviceTestCase(TestCase): diff --git a/netbox/dcim/views.py b/netbox/dcim/views.py index 56119a775..a2e162519 100644 --- a/netbox/dcim/views.py +++ b/netbox/dcim/views.py @@ -1,3 +1,4 @@ +from collections import OrderedDict import re from django.conf import settings @@ -13,7 +14,6 @@ from django.urls import reverse from django.utils.html import escape from django.utils.http import is_safe_url from django.utils.safestring import mark_safe -from django.utils.text import slugify from django.views.generic import View from circuits.models import Circuit @@ -660,6 +660,16 @@ class DeviceTypeImportView(PermissionRequiredMixin, ObjectImportView): permission_required = 'dcim.add_devicetype' model = DeviceType model_form = forms.DeviceTypeImportForm + related_object_forms = OrderedDict(( + ('console-ports', forms.ConsolePortTemplateImportForm), + ('console-server-ports', forms.ConsoleServerPortTemplateImportForm), + ('power-ports', forms.PowerPortTemplateImportForm), + ('power-outlets', forms.PowerOutletTemplateImportForm), + ('interfaces', forms.InterfaceTemplateImportForm), + ('rear-ports', forms.RearPortTemplateImportForm), + ('front-ports', forms.FrontPortTemplateImportForm), + ('device-bays', forms.DeviceBayTemplateImportForm), + )) default_return_url = 'dcim:devicetype_import' diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index 41d577c68..9d9116fbc 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -556,39 +556,6 @@ class SlugField(forms.SlugField): self.widget.attrs['slug-source'] = slug_source -class MultiObjectField(forms.Field): - """ - Use this field to relay data to another form for validation. Useful when importing data via JSON/YAML. - """ - def __init__(self, form, *args, **kwargs): - self.form = form - super().__init__(*args, **kwargs) - - def clean(self, value): - - # Value needs to be an iterable - if value is None: - return list() - - for i, obj_data in enumerate(value, start=1): - - # Bind object data to form - form = self.form(obj_data) - - # Assign default values for required fields that have not been defined - for field_name, field in form.fields.items(): - if field_name not in obj_data and hasattr(field, 'initial'): - form.data[field_name] = field.initial - - if not form.is_valid(): - errors = [ - "Object {} {}: {}".format(i, field, errors) for field, errors in form.errors.items() - ] - raise forms.ValidationError(errors) - - return value - - class FilterChoiceIterator(forms.models.ModelChoiceIterator): def __iter__(self): diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index b46a50ef3..e17da7353 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -26,6 +26,7 @@ from django_tables2 import RequestConfig from extras.models import CustomField, CustomFieldValue, ExportTemplate from extras.querysets import CustomFieldQueryset +from utilities.exceptions import AbortTransaction from utilities.forms import BootstrapMixin, CSVDataField from utilities.utils import csv_format from .error_handlers import handle_protectederror @@ -402,6 +403,7 @@ class ObjectImportView(GetReturnURLMixin, View): """ model = None model_form = None + related_object_forms = dict() template_name = 'utilities/obj_import.html' def create_object(self, data): @@ -436,8 +438,32 @@ class ObjectImportView(GetReturnURLMixin, View): if model_form.is_valid(): - with transaction.atomic(): - obj = model_form.save() + try: + with transaction.atomic(): + + # Save the primary object + obj = model_form.save() + + # Iterate through the related object forms (if any), validating and saving each instance. + for field, related_object_form in self.related_object_forms.items(): + + for i, rel_obj_data in enumerate(data.get(field, list())): + + f = related_object_form(obj, rel_obj_data) + if f.is_valid(): + f.save() + else: + # Replicate errors on the related object form to the primary form for display + for field_name, errors in f.errors.items(): + for err in errors: + err_msg = "{}[{}] {}: {}".format(field, i, field_name, err) + model_form.add_error(None, err_msg) + raise AbortTransaction() + + except AbortTransaction: + pass + + if not model_form.errors: messages.success(request, mark_safe('Imported object: {}'.format( obj.get_absolute_url(), obj From ee4e68b08272ad92149801ed529a48d8f26f41ed Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 1 Oct 2019 16:36:31 -0400 Subject: [PATCH 18/42] Rewrote test for DeviceType import --- netbox/dcim/forms.py | 3 +- netbox/dcim/tests/test_forms.py | 104 ------------------------- netbox/dcim/tests/test_views.py | 133 +++++++++++++++++++++++++++++++- netbox/utilities/views.py | 13 +++- 4 files changed, 140 insertions(+), 113 deletions(-) diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index ea663fbcc..b54a165a0 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -1212,7 +1212,6 @@ class ComponentTemplateImportForm(BootstrapMixin, forms.ModelForm): data.update({ 'device_type': device_type.pk, }) - print(data) super().__init__(data, *args, **kwargs) @@ -1279,7 +1278,7 @@ class InterfaceTemplateImportForm(ComponentTemplateImportForm): class FrontPortTemplateImportForm(ComponentTemplateImportForm): - power_port = forms.ModelChoiceField( + rear_port = forms.ModelChoiceField( queryset=RearPortTemplate.objects.all(), to_field_name='name', required=False diff --git a/netbox/dcim/tests/test_forms.py b/netbox/dcim/tests/test_forms.py index d9cf10fdb..2f333ea69 100644 --- a/netbox/dcim/tests/test_forms.py +++ b/netbox/dcim/tests/test_forms.py @@ -8,110 +8,6 @@ def get_id(model, slug): return model.objects.get(slug=slug).id -DEVICETYPE_DATA = { - 'manufacturer': 'Generic', - 'model': 'TEST-1000', - 'slug': 'test-1000', - 'u_height': 2, - 'console-ports': [ - {'name': 'Console Port 1'}, - {'name': 'Console Port 2'}, - {'name': 'Console Port 3'}, - ], - 'console-server-ports': [ - {'name': 'Console Server Port 1'}, - {'name': 'Console Server Port 2'}, - {'name': 'Console Server Port 3'}, - ], - 'power-ports': [ - {'name': 'Power Port 1'}, - {'name': 'Power Port 2'}, - {'name': 'Power Port 3'}, - ], - 'power-outlets': [ - {'name': 'Power Outlet 1', 'power_port': 'Power Port 1', 'feed_leg': POWERFEED_LEG_A}, - {'name': 'Power Outlet 2', 'power_port': 'Power Port 1', 'feed_leg': POWERFEED_LEG_A}, - {'name': 'Power Outlet 3', 'power_port': 'Power Port 1', 'feed_leg': POWERFEED_LEG_A}, - ], - 'interfaces': [ - {'name': 'Interface 1', 'type': IFACE_TYPE_1GE_FIXED, 'mgmt_only': True}, - {'name': 'Interface 2', 'type': IFACE_TYPE_1GE_FIXED}, - {'name': 'Interface 3', 'type': IFACE_TYPE_1GE_FIXED}, - ], - 'rear-ports': [ - {'name': 'Rear Port 1', 'type': PORT_TYPE_8P8C}, - {'name': 'Rear Port 2', 'type': PORT_TYPE_8P8C}, - {'name': 'Rear Port 3', 'type': PORT_TYPE_8P8C}, - ], - 'front-ports': [ - {'name': 'Front Port 1', 'type': PORT_TYPE_8P8C, 'rear_port': 'Rear Port 1'}, - {'name': 'Front Port 2', 'type': PORT_TYPE_8P8C, 'rear_port': 'Rear Port 2'}, - {'name': 'Front Port 3', 'type': PORT_TYPE_8P8C, 'rear_port': 'Rear Port 3'}, - ], - 'device-bays': [ - {'name': 'Device Bay 1'}, - {'name': 'Device Bay 2'}, - {'name': 'Device Bay 3'}, - ], -} - - -class DeviceTypeImportTestCase(TestCase): - - def setUp(self): - - Manufacturer(name='Generic', slug='generic').save() - - def test_import_devicetype_yaml(self): - - form = DeviceTypeImportForm(DEVICETYPE_DATA) - - self.assertTrue(form.is_valid(), "Form validation failed: {}".format(form.errors)) - - form.save() - dt = DeviceType.objects.get(model='TEST-1000') - - # Verify all of the components were created - # TODO: The creation of components now occurs in the view rather than the form - self.assertEqual(dt.consoleport_templates.count(), 3) - cp1 = ConsolePortTemplate.objects.first() - self.assertEqual(cp1.name, 'Console Port 1') - - self.assertEqual(dt.consoleserverport_templates.count(), 3) - csp1 = ConsoleServerPortTemplate.objects.first() - self.assertEqual(csp1.name, 'Console Server Port 1') - - self.assertEqual(dt.powerport_templates.count(), 3) - pp1 = PowerPortTemplate.objects.first() - self.assertEqual(pp1.name, 'Power Port 1') - - self.assertEqual(dt.poweroutlet_templates.count(), 3) - po1 = PowerOutletTemplate.objects.first() - self.assertEqual(po1.name, 'Power Outlet 1') - self.assertEqual(po1.power_port, pp1) - self.assertEqual(po1.feed_leg, POWERFEED_LEG_A) - - self.assertEqual(dt.interface_templates.count(), 4) - iface1 = Interface.objects.first() - self.assertEqual(iface1.name, 'Interface 1') - self.assertEqual(iface1.type, IFACE_TYPE_1GE_FIXED) - self.assertTrue(iface1.mgmt_only) - - self.assertEqual(dt.rearport_templates.count(), 3) - rp1 = RearPortTemplate.objects.first() - self.assertEqual(rp1.name, 'Rear Port 1') - - self.assertEqual(dt.frontport_templates.count(), 3) - fp1 = FrontPortTemplate.objects.first() - self.assertEqual(fp1.name, 'Front Port 1') - self.assertEqual(fp1.rear_port, rp1) - self.assertEqual(fp1.rear_port_position, 1) - - self.assertEqual(dt.devicebay_templates.count(), 3) - db1 = DeviceBayTemplate.objects.first() - self.assertEqual(db1.name, 'Device Bay 1') - - class DeviceTestCase(TestCase): fixtures = ['dcim', 'ipam'] diff --git a/netbox/dcim/tests/test_views.py b/netbox/dcim/tests/test_views.py index 6e34b8ae9..6af101e4c 100644 --- a/netbox/dcim/tests/test_views.py +++ b/netbox/dcim/tests/test_views.py @@ -3,10 +3,11 @@ import urllib.parse from django.test import Client, TestCase from django.urls import reverse -from dcim.constants import CABLE_TYPE_CAT6, IFACE_TYPE_1GE_FIXED +from dcim.constants import * from dcim.models import ( - Cable, Device, DeviceRole, DeviceType, Interface, InventoryItem, Manufacturer, Platform, Rack, RackGroup, - RackReservation, RackRole, Site, Region, VirtualChassis, + Cable, ConsolePortTemplate, ConsoleServerPortTemplate, Device, DeviceBayTemplate, DeviceRole, DeviceType, + FrontPortTemplate, Interface, InterfaceTemplate, InventoryItem, Manufacturer, Platform, PowerPortTemplate, + PowerOutletTemplate, Rack, RackGroup, RackReservation, RackRole, RearPortTemplate, Site, Region, VirtualChassis, ) from utilities.testing import create_test_user @@ -221,6 +222,132 @@ class DeviceTypeTestCase(TestCase): response = self.client.get(devicetype.get_absolute_url()) self.assertEqual(response.status_code, 200) + def test_devicetype_import(self): + + IMPORT_DATA = """ +manufacturer: Generic +model: TEST-1000 +slug: test-1000 +u_height: 2 +console-ports: + - name: Console Port 1 + - name: Console Port 2 + - name: Console Port 3 +console-server-ports: + - name: Console Server Port 1 + - name: Console Server Port 2 + - name: Console Server Port 3 +power-ports: + - name: Power Port 1 + - name: Power Port 2 + - name: Power Port 3 +power-outlets: + - name: Power Outlet 1 + power_port: Power Port 1 + feed_leg: 1 + - name: Power Outlet 2 + power_port: Power Port 1 + feed_leg: 1 + - name: Power Outlet 3 + power_port: Power Port 1 + feed_leg: 1 +interfaces: + - name: Interface 1 + type: 1000 + mgmt_only: true + - name: Interface 2 + type: 1000 + - name: Interface 3 + type: 1000 +rear-ports: + - name: Rear Port 1 + type: 1000 + - name: Rear Port 2 + type: 1000 + - name: Rear Port 3 + type: 1000 +front-ports: + - name: Front Port 1 + type: 1000 + rear_port: Rear Port 1 + - name: Front Port 2 + type: 1000 + rear_port: Rear Port 2 + - name: Front Port 3 + type: 1000 + rear_port: Rear Port 3 +device-bays: + - name: Device Bay 1 + - name: Device Bay 2 + - name: Device Bay 3 +""" + + # Create the manufacturer + Manufacturer(name='Generic', slug='generic').save() + + # Authenticate as user with necessary permissions + user = create_test_user(username='testuser2', permissions=[ + 'dcim.view_devicetype', + 'dcim.add_devicetype', + 'dcim.add_consoleporttemplate', + 'dcim.add_consoleserverporttemplate', + 'dcim.add_powerporttemplate', + 'dcim.add_poweroutlettemplate', + 'dcim.add_interfacetemplate', + 'dcim.add_frontporttemplate', + 'dcim.add_rearporttemplate', + 'dcim.add_devicebaytemplate', + ]) + self.client.force_login(user) + + form_data = { + 'data': IMPORT_DATA, + 'format': 'yaml' + } + response = self.client.post(reverse('dcim:devicetype_import'), data=form_data, follow=True) + + dt = DeviceType.objects.get(model='TEST-1000') + + # Verify all of the components were created + self.assertEqual(dt.consoleport_templates.count(), 3) + cp1 = ConsolePortTemplate.objects.first() + self.assertEqual(cp1.name, 'Console Port 1') + + self.assertEqual(dt.consoleserverport_templates.count(), 3) + csp1 = ConsoleServerPortTemplate.objects.first() + self.assertEqual(csp1.name, 'Console Server Port 1') + + self.assertEqual(dt.powerport_templates.count(), 3) + pp1 = PowerPortTemplate.objects.first() + self.assertEqual(pp1.name, 'Power Port 1') + + self.assertEqual(dt.poweroutlet_templates.count(), 3) + po1 = PowerOutletTemplate.objects.first() + self.assertEqual(po1.name, 'Power Outlet 1') + self.assertEqual(po1.power_port, pp1) + self.assertEqual(po1.feed_leg, POWERFEED_LEG_A) + + self.assertEqual(dt.interface_templates.count(), 3) + iface1 = InterfaceTemplate.objects.first() + self.assertEqual(iface1.name, 'Interface 1') + self.assertEqual(iface1.type, IFACE_TYPE_1GE_FIXED) + self.assertTrue(iface1.mgmt_only) + + self.assertEqual(dt.rearport_templates.count(), 3) + rp1 = RearPortTemplate.objects.first() + self.assertEqual(rp1.name, 'Rear Port 1') + + self.assertEqual(dt.frontport_templates.count(), 3) + fp1 = FrontPortTemplate.objects.first() + self.assertEqual(fp1.name, 'Front Port 1') + self.assertEqual(fp1.rear_port, rp1) + self.assertEqual(fp1.rear_port_position, 1) + + self.assertEqual(dt.device_bay_templates.count(), 3) + db1 = DeviceBayTemplate.objects.first() + self.assertEqual(db1.name, 'Device Bay 1') + + class DeviceRoleTestCase(TestCase): diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index e17da7353..f53e4bebb 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -445,18 +445,23 @@ class ObjectImportView(GetReturnURLMixin, View): obj = model_form.save() # Iterate through the related object forms (if any), validating and saving each instance. - for field, related_object_form in self.related_object_forms.items(): + for field_name, related_object_form in self.related_object_forms.items(): - for i, rel_obj_data in enumerate(data.get(field, list())): + for i, rel_obj_data in enumerate(data.get(field_name, list())): f = related_object_form(obj, rel_obj_data) + + for subfield_name, field in f.fields.items(): + if subfield_name not in rel_obj_data and hasattr(field, 'initial'): + f.data[subfield_name] = field.initial + if f.is_valid(): f.save() else: # Replicate errors on the related object form to the primary form for display - for field_name, errors in f.errors.items(): + for subfield_name, errors in f.errors.items(): for err in errors: - err_msg = "{}[{}] {}: {}".format(field, i, field_name, err) + err_msg = "{}[{}] {}: {}".format(field_name, i, subfield_name, err) model_form.add_error(None, err_msg) raise AbortTransaction() From 88d61db384f287942d8310ad70091ce166142c40 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 1 Oct 2019 16:39:11 -0400 Subject: [PATCH 19/42] Fix YAMLLoadWarning --- netbox/utilities/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index 9d9116fbc..ee63712a0 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -756,7 +756,7 @@ class ImportForm(BootstrapMixin, forms.Form): }) else: try: - self.cleaned_data['data'] = yaml.load(data) + self.cleaned_data['data'] = yaml.load(data, Loader=yaml.SafeLoader) except yaml.scanner.ScannerError as err: raise forms.ValidationError({ 'data': "Invalid YAML data: {}".format(err) From 6892b79366cb2271d1f648efa2bde502e9f345ac Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 1 Oct 2019 16:54:10 -0400 Subject: [PATCH 20/42] Enforce object creation permissions --- netbox/dcim/tests/test_views.py | 1 + netbox/dcim/views.py | 12 +++++++++++- netbox/utilities/views.py | 3 --- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/netbox/dcim/tests/test_views.py b/netbox/dcim/tests/test_views.py index 6af101e4c..03515b680 100644 --- a/netbox/dcim/tests/test_views.py +++ b/netbox/dcim/tests/test_views.py @@ -305,6 +305,7 @@ device-bays: 'format': 'yaml' } response = self.client.post(reverse('dcim:devicetype_import'), data=form_data, follow=True) + self.assertEqual(response.status_code, 200) dt = DeviceType.objects.get(model='TEST-1000') diff --git a/netbox/dcim/views.py b/netbox/dcim/views.py index a2e162519..7666bafd3 100644 --- a/netbox/dcim/views.py +++ b/netbox/dcim/views.py @@ -657,7 +657,17 @@ class DeviceTypeDeleteView(PermissionRequiredMixin, ObjectDeleteView): class DeviceTypeImportView(PermissionRequiredMixin, ObjectImportView): - permission_required = 'dcim.add_devicetype' + permission_required = [ + 'dcim.add_devicetype', + 'dcim.add_consoleporttemplate', + 'dcim.add_consoleserverporttemplate', + 'dcim.add_powerporttemplate', + 'dcim.add_poweroutlettemplate', + 'dcim.add_interfacetemplate', + 'dcim.add_frontporttemplate', + 'dcim.add_rearporttemplate', + 'dcim.add_devicebaytemplate', + ] model = DeviceType model_form = forms.DeviceTypeImportForm related_object_forms = OrderedDict(( diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index f53e4bebb..48c4f01e6 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -406,9 +406,6 @@ class ObjectImportView(GetReturnURLMixin, View): related_object_forms = dict() template_name = 'utilities/obj_import.html' - def create_object(self, data): - raise NotImplementedError("View must implement object creation logic") - def get(self, request): form = ImportForm() From 807d8496574dc0ada89468a1f91ed1e00c175970 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 1 Oct 2019 17:07:17 -0400 Subject: [PATCH 21/42] PEP8 fix --- netbox/dcim/tests/test_views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/netbox/dcim/tests/test_views.py b/netbox/dcim/tests/test_views.py index 03515b680..754a2dd83 100644 --- a/netbox/dcim/tests/test_views.py +++ b/netbox/dcim/tests/test_views.py @@ -349,7 +349,6 @@ device-bays: self.assertEqual(db1.name, 'Device Bay 1') - class DeviceRoleTestCase(TestCase): def setUp(self): From 32628059381f3ca44595f648759f99390ea0e59b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20L=C3=B6nnegren?= Date: Mon, 7 Oct 2019 08:29:32 +0200 Subject: [PATCH 22/42] Add tenancy to cluster fix pep8 --- netbox/templates/tenancy/tenant.html | 4 +++ netbox/templates/virtualization/cluster.html | 10 +++++++ .../virtualization/cluster_edit.html | 1 + netbox/tenancy/api/serializers.py | 3 ++- netbox/tenancy/views.py | 3 ++- netbox/virtualization/api/serializers.py | 3 ++- netbox/virtualization/api/views.py | 2 +- netbox/virtualization/filters.py | 5 ++++ netbox/virtualization/forms.py | 26 +++++++++++++++++-- .../migrations/0010_cluster_add_tenant.py | 18 +++++++++++++ netbox/virtualization/models.py | 8 ++++++ netbox/virtualization/tables.py | 3 ++- netbox/virtualization/views.py | 2 +- 13 files changed, 80 insertions(+), 8 deletions(-) create mode 100644 netbox/virtualization/migrations/0010_cluster_add_tenant.py diff --git a/netbox/templates/tenancy/tenant.html b/netbox/templates/tenancy/tenant.html index 053c69121..72b17f10c 100644 --- a/netbox/templates/tenancy/tenant.html +++ b/netbox/templates/tenancy/tenant.html @@ -142,6 +142,10 @@

{{ stats.virtualmachine_count }}

Virtual machines

+
+

{{ stats.cluster_count }}

+

Clusters

+
diff --git a/netbox/templates/virtualization/cluster.html b/netbox/templates/virtualization/cluster.html index b543e85b5..264538237 100644 --- a/netbox/templates/virtualization/cluster.html +++ b/netbox/templates/virtualization/cluster.html @@ -83,6 +83,16 @@ {% endif %} + + Tenant + + {% if cluster.tenant %} + {{ cluster.tenant }} + {% else %} + None + {% endif %} + + Site diff --git a/netbox/templates/virtualization/cluster_edit.html b/netbox/templates/virtualization/cluster_edit.html index 629c779ec..0e188d8ab 100644 --- a/netbox/templates/virtualization/cluster_edit.html +++ b/netbox/templates/virtualization/cluster_edit.html @@ -8,6 +8,7 @@ {% render_field form.name %} {% render_field form.type %} {% render_field form.group %} + {% render_field form.tenant %} {% render_field form.site %} diff --git a/netbox/tenancy/api/serializers.py b/netbox/tenancy/api/serializers.py index 28ae04694..7599029c5 100644 --- a/netbox/tenancy/api/serializers.py +++ b/netbox/tenancy/api/serializers.py @@ -31,11 +31,12 @@ class TenantSerializer(TaggitSerializer, CustomFieldModelSerializer): virtualmachine_count = serializers.IntegerField(read_only=True) vlan_count = serializers.IntegerField(read_only=True) vrf_count = serializers.IntegerField(read_only=True) + cluster_count = serializers.IntegerField(read_only=True) class Meta: model = Tenant fields = [ 'id', 'name', 'slug', 'group', 'description', 'comments', 'tags', 'custom_fields', 'created', 'last_updated', 'circuit_count', 'device_count', 'ipaddress_count', 'prefix_count', 'rack_count', - 'site_count', 'virtualmachine_count', 'vlan_count', 'vrf_count', + 'site_count', 'virtualmachine_count', 'vlan_count', 'vrf_count', 'cluster_count', ] diff --git a/netbox/tenancy/views.py b/netbox/tenancy/views.py index 965ae2853..c7690d04b 100644 --- a/netbox/tenancy/views.py +++ b/netbox/tenancy/views.py @@ -9,7 +9,7 @@ from ipam.models import IPAddress, Prefix, VLAN, VRF from utilities.views import ( BulkDeleteView, BulkEditView, BulkImportView, ObjectDeleteView, ObjectEditView, ObjectListView, ) -from virtualization.models import VirtualMachine +from virtualization.models import VirtualMachine, Cluster from . import filters, forms, tables from .models import Tenant, TenantGroup @@ -80,6 +80,7 @@ class TenantView(PermissionRequiredMixin, View): 'vlan_count': VLAN.objects.filter(tenant=tenant).count(), 'circuit_count': Circuit.objects.filter(tenant=tenant).count(), 'virtualmachine_count': VirtualMachine.objects.filter(tenant=tenant).count(), + 'cluster_count': Cluster.objects.filter(tenant=tenant).count(), } return render(request, 'tenancy/tenant.html', { diff --git a/netbox/virtualization/api/serializers.py b/netbox/virtualization/api/serializers.py index 0b98ce44a..75f36fbb6 100644 --- a/netbox/virtualization/api/serializers.py +++ b/netbox/virtualization/api/serializers.py @@ -38,6 +38,7 @@ class ClusterGroupSerializer(ValidatedModelSerializer): class ClusterSerializer(TaggitSerializer, CustomFieldModelSerializer): type = NestedClusterTypeSerializer() group = NestedClusterGroupSerializer(required=False, allow_null=True) + tenant = NestedTenantSerializer(required=False, allow_null=True) site = NestedSiteSerializer(required=False, allow_null=True) tags = TagListSerializerField(required=False) device_count = serializers.IntegerField(read_only=True) @@ -46,7 +47,7 @@ class ClusterSerializer(TaggitSerializer, CustomFieldModelSerializer): class Meta: model = Cluster fields = [ - 'id', 'name', 'type', 'group', 'site', 'comments', 'tags', 'custom_fields', 'created', 'last_updated', + 'id', 'name', 'type', 'group', 'tenant', 'site', 'comments', 'tags', 'custom_fields', 'created', 'last_updated', 'device_count', 'virtualmachine_count', ] diff --git a/netbox/virtualization/api/views.py b/netbox/virtualization/api/views.py index f6d7f1230..94b75d154 100644 --- a/netbox/virtualization/api/views.py +++ b/netbox/virtualization/api/views.py @@ -41,7 +41,7 @@ class ClusterGroupViewSet(ModelViewSet): class ClusterViewSet(CustomFieldModelViewSet): queryset = Cluster.objects.prefetch_related( - 'type', 'group', 'site', 'tags' + 'type', 'group', 'tenant', 'site', 'tags' ).annotate( device_count=get_subquery(Device, 'cluster'), virtualmachine_count=get_subquery(VirtualMachine, 'cluster') diff --git a/netbox/virtualization/filters.py b/netbox/virtualization/filters.py index 44b489d43..4df7c0e9c 100644 --- a/netbox/virtualization/filters.py +++ b/netbox/virtualization/filters.py @@ -4,6 +4,7 @@ from netaddr import EUI from netaddr.core import AddrFormatError from dcim.models import DeviceRole, Interface, Platform, Region, Site +from tenancy.models import Tenant from extras.filters import CustomFieldFilterSet from tenancy.filtersets import TenancyFilterSet from utilities.filters import ( @@ -56,6 +57,10 @@ class ClusterFilter(CustomFieldFilterSet): to_field_name='slug', label='Cluster type (slug)', ) + tenant = django_filters.ModelMultipleChoiceFilter( + queryset=Tenant.objects.all(), + label="Tenant (ID)" + ) site_id = django_filters.ModelMultipleChoiceFilter( queryset=Site.objects.all(), label='Site (ID)', diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index d3fc4b83b..2ad5ea472 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -87,7 +87,7 @@ class ClusterForm(BootstrapMixin, CustomFieldForm): class Meta: model = Cluster fields = [ - 'name', 'type', 'group', 'site', 'comments', 'tags', + 'name', 'type', 'group', 'tenant', 'site', 'comments', 'tags', ] widgets = { 'type': APISelect( @@ -129,6 +129,15 @@ class ClusterCSVForm(forms.ModelForm): 'invalid_choice': 'Invalid site name.', } ) + tenant = forms.ModelChoiceField( + queryset=Tenant.objects.all(), + to_field_name='name', + required=False, + help_text='Name of assigned tenant', + error_messages={ + 'invalid_choice': 'Invalid tenant name' + } + ) class Meta: model = Cluster @@ -154,6 +163,10 @@ class ClusterBulkEditForm(BootstrapMixin, AddRemoveTagsForm, CustomFieldBulkEdit api_url="/api/virtualization/cluster-groups/" ) ) + tenant = forms.ModelChoiceField( + queryset=Tenant.objects.all(), + required=False + ) site = forms.ModelChoiceField( queryset=Site.objects.all(), required=False, @@ -167,7 +180,7 @@ class ClusterBulkEditForm(BootstrapMixin, AddRemoveTagsForm, CustomFieldBulkEdit class Meta: nullable_fields = [ - 'group', 'site', 'comments', + 'group', 'site', 'comments', 'tenant', ] @@ -194,6 +207,15 @@ class ClusterFilterForm(BootstrapMixin, CustomFieldFilterForm): null_option=True, ) ) + tenant = FilterChoiceField( + queryset=Tenant.objects.all(), + null_label='-- None --', + required=False, + widget=APISelectMultiple( + api_url="/api/tenancy/tenants/", + null_option=True, + ) + ) site = FilterChoiceField( queryset=Site.objects.all(), to_field_name='slug', diff --git a/netbox/virtualization/migrations/0010_cluster_add_tenant.py b/netbox/virtualization/migrations/0010_cluster_add_tenant.py new file mode 100644 index 000000000..425b32635 --- /dev/null +++ b/netbox/virtualization/migrations/0010_cluster_add_tenant.py @@ -0,0 +1,18 @@ +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('tenancy', '0001_initial'), + ('virtualization', '0009_custom_tag_models'), + ] + + operations = [ + migrations.AddField( + model_name='cluster', + name='tenant', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='clusters', to='tenancy.Tenant'), + ), + ] diff --git a/netbox/virtualization/models.py b/netbox/virtualization/models.py index 2ef782dfd..7f82bb0bd 100644 --- a/netbox/virtualization/models.py +++ b/netbox/virtualization/models.py @@ -103,6 +103,13 @@ class Cluster(ChangeLoggedModel, CustomFieldModel): blank=True, null=True ) + tenant = models.ForeignKey( + to='tenancy.Tenant', + on_delete=models.PROTECT, + related_name='tenants', + blank=True, + null=True + ) site = models.ForeignKey( to='dcim.Site', on_delete=models.PROTECT, @@ -150,6 +157,7 @@ class Cluster(ChangeLoggedModel, CustomFieldModel): self.type.name, self.group.name if self.group else None, self.site.name if self.site else None, + self.tenant.name if self.tenant else None, self.comments, ) diff --git a/netbox/virtualization/tables.py b/netbox/virtualization/tables.py index 6034dd8dc..ba4554ff5 100644 --- a/netbox/virtualization/tables.py +++ b/netbox/virtualization/tables.py @@ -84,13 +84,14 @@ class ClusterGroupTable(BaseTable): class ClusterTable(BaseTable): pk = ToggleColumn() name = tables.LinkColumn() + tenant = tables.LinkColumn('tenancy:tenant', args=[Accessor('tenant.slug')], verbose_name='Tenant') site = tables.LinkColumn('dcim:site', args=[Accessor('site.slug')]) device_count = tables.Column(accessor=Accessor('devices.count'), orderable=False, verbose_name='Devices') vm_count = tables.Column(accessor=Accessor('virtual_machines.count'), orderable=False, verbose_name='VMs') class Meta(BaseTable.Meta): model = Cluster - fields = ('pk', 'name', 'type', 'group', 'site', 'device_count', 'vm_count') + fields = ('pk', 'name', 'type', 'group', 'tenant', 'site', 'device_count', 'vm_count') # diff --git a/netbox/virtualization/views.py b/netbox/virtualization/views.py index 06a39e651..73eccb4b2 100644 --- a/netbox/virtualization/views.py +++ b/netbox/virtualization/views.py @@ -96,7 +96,7 @@ class ClusterGroupBulkDeleteView(PermissionRequiredMixin, BulkDeleteView): class ClusterListView(PermissionRequiredMixin, ObjectListView): permission_required = 'virtualization.view_cluster' - queryset = Cluster.objects.prefetch_related('type', 'group', 'site') + queryset = Cluster.objects.prefetch_related('type', 'group', 'site', 'tenant') table = tables.ClusterTable filter = filters.ClusterFilter filter_form = forms.ClusterFilterForm From d787c353f3ecbaebbf39f52c801c6cf92143d220 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 10 Oct 2019 23:24:44 -0400 Subject: [PATCH 23/42] Added slug choices for interface and port types --- netbox/dcim/constants.py | 425 ++++++++++++++++++++++++++++++-- netbox/dcim/forms.py | 24 ++ netbox/dcim/tests/test_views.py | 18 +- 3 files changed, 437 insertions(+), 30 deletions(-) diff --git a/netbox/dcim/constants.py b/netbox/dcim/constants.py index 840d55d6b..9d20229a1 100644 --- a/netbox/dcim/constants.py +++ b/netbox/dcim/constants.py @@ -1,4 +1,3 @@ - # Rack types RACK_TYPE_2POST = 100 RACK_TYPE_4POST = 200 @@ -58,7 +57,10 @@ SUBDEVICE_ROLE_CHOICES = ( (SUBDEVICE_ROLE_CHILD, 'Child'), ) -# Interface types +# +# Numeric interface types +# + # Virtual IFACE_TYPE_VIRTUAL = 0 IFACE_TYPE_LAG = 200 @@ -113,15 +115,15 @@ IFACE_TYPE_16GFC_SFP_PLUS = 3160 IFACE_TYPE_32GFC_SFP28 = 3320 IFACE_TYPE_128GFC_QSFP28 = 3400 # InfiniBand -IFACE_FF_INFINIBAND_SDR = 7010 -IFACE_FF_INFINIBAND_DDR = 7020 -IFACE_FF_INFINIBAND_QDR = 7030 -IFACE_FF_INFINIBAND_FDR10 = 7040 -IFACE_FF_INFINIBAND_FDR = 7050 -IFACE_FF_INFINIBAND_EDR = 7060 -IFACE_FF_INFINIBAND_HDR = 7070 -IFACE_FF_INFINIBAND_NDR = 7080 -IFACE_FF_INFINIBAND_XDR = 7090 +IFACE_TYPE_INFINIBAND_SDR = 7010 +IFACE_TYPE_INFINIBAND_DDR = 7020 +IFACE_TYPE_INFINIBAND_QDR = 7030 +IFACE_TYPE_INFINIBAND_FDR10 = 7040 +IFACE_TYPE_INFINIBAND_FDR = 7050 +IFACE_TYPE_INFINIBAND_EDR = 7060 +IFACE_TYPE_INFINIBAND_HDR = 7070 +IFACE_TYPE_INFINIBAND_NDR = 7080 +IFACE_TYPE_INFINIBAND_XDR = 7090 # Serial IFACE_TYPE_T1 = 4000 IFACE_TYPE_E1 = 4010 @@ -227,15 +229,15 @@ IFACE_TYPE_CHOICES = [ [ 'InfiniBand', [ - [IFACE_FF_INFINIBAND_SDR, 'SDR (2 Gbps)'], - [IFACE_FF_INFINIBAND_DDR, 'DDR (4 Gbps)'], - [IFACE_FF_INFINIBAND_QDR, 'QDR (8 Gbps)'], - [IFACE_FF_INFINIBAND_FDR10, 'FDR10 (10 Gbps)'], - [IFACE_FF_INFINIBAND_FDR, 'FDR (13.5 Gbps)'], - [IFACE_FF_INFINIBAND_EDR, 'EDR (25 Gbps)'], - [IFACE_FF_INFINIBAND_HDR, 'HDR (50 Gbps)'], - [IFACE_FF_INFINIBAND_NDR, 'NDR (100 Gbps)'], - [IFACE_FF_INFINIBAND_XDR, 'XDR (250 Gbps)'], + [IFACE_TYPE_INFINIBAND_SDR, 'SDR (2 Gbps)'], + [IFACE_TYPE_INFINIBAND_DDR, 'DDR (4 Gbps)'], + [IFACE_TYPE_INFINIBAND_QDR, 'QDR (8 Gbps)'], + [IFACE_TYPE_INFINIBAND_FDR10, 'FDR10 (10 Gbps)'], + [IFACE_TYPE_INFINIBAND_FDR, 'FDR (13.5 Gbps)'], + [IFACE_TYPE_INFINIBAND_EDR, 'EDR (25 Gbps)'], + [IFACE_TYPE_INFINIBAND_HDR, 'HDR (50 Gbps)'], + [IFACE_TYPE_INFINIBAND_NDR, 'NDR (100 Gbps)'], + [IFACE_TYPE_INFINIBAND_XDR, 'XDR (250 Gbps)'], ] ], [ @@ -382,7 +384,8 @@ CONNECTION_STATUS_CHOICES = [ # Cable endpoint types CABLE_TERMINATION_TYPES = [ - 'consoleport', 'consoleserverport', 'interface', 'poweroutlet', 'powerport', 'frontport', 'rearport', 'circuittermination', + 'consoleport', 'consoleserverport', 'interface', 'poweroutlet', 'powerport', 'frontport', 'rearport', + 'circuittermination', ] # Cable types @@ -510,3 +513,383 @@ POWERFEED_LEG_CHOICES = ( (POWERFEED_LEG_B, 'B'), (POWERFEED_LEG_C, 'C'), ) + + +# +# Interface type values +# + +class InterfaceTypes: + """ + Interface.type slugs + """ + # Virtual + TYPE_VIRTUAL = 'virtual' + TYPE_LAG = 'lag' + + # Ethernet + TYPE_100ME_FIXED = '100base-tx' + TYPE_1GE_FIXED = '1000base-t' + TYPE_1GE_GBIC = '1000base-x-gbic' + TYPE_1GE_SFP = '1000base-x-sfp' + TYPE_2GE_FIXED = '2.5gbase-t' + TYPE_5GE_FIXED = '5gbase-t' + TYPE_10GE_FIXED = '10gbase-t' + TYPE_10GE_CX4 = '10gbase-cx4' + TYPE_10GE_SFP_PLUS = '10gbase-x-sfpp' + TYPE_10GE_XFP = '10gbase-x-xfp' + TYPE_10GE_XENPAK = '10gbase-x-xenpak' + TYPE_10GE_X2 = '10gbase-x-x2' + TYPE_25GE_SFP28 = '25gbase-x-sfp28' + TYPE_40GE_QSFP_PLUS = '40gbase-x-qsfpp' + TYPE_50GE_QSFP28 = '50gbase-x-sfp28' + TYPE_100GE_CFP = '100gbase-x-cfp' + TYPE_100GE_CFP2 = '100gbase-x-cfp2' + TYPE_100GE_CFP4 = '100gbase-x-cfp4' + TYPE_100GE_CPAK = '100gbase-x-cpak' + TYPE_100GE_QSFP28 = '100gbase-x-qsfp28' + TYPE_200GE_CFP2 = '200gbase-x-cfp2' + TYPE_200GE_QSFP56 = '200gbase-x-qsfp56' + TYPE_400GE_QSFP_DD = '400gbase-x-qsfpdd' + + # Wireless + TYPE_80211A = 'ieee802.11a' + TYPE_80211G = 'ieee802.11g' + TYPE_80211N = 'ieee802.11n' + TYPE_80211AC = 'ieee802.11ac' + TYPE_80211AD = 'ieee802.11ad' + + # Cellular + TYPE_GSM = 'gsm' + TYPE_CDMA = 'cdma' + TYPE_LTE = 'lte' + + # SONET + TYPE_SONET_OC3 = 'sonet-oc3' + TYPE_SONET_OC12 = 'sonet-oc12' + TYPE_SONET_OC48 = 'sonet-oc48' + TYPE_SONET_OC192 = 'sonet-oc192' + TYPE_SONET_OC768 = 'sonet-oc768' + TYPE_SONET_OC1920 = 'sonet-oc1920' + TYPE_SONET_OC3840 = 'sonet-oc3840' + + # Fibrechannel + TYPE_1GFC_SFP = '1gfc-sfp' + TYPE_2GFC_SFP = '2gfc-sfp' + TYPE_4GFC_SFP = '4gfc-sfp' + TYPE_8GFC_SFP_PLUS = '8gfc-sfpp' + TYPE_16GFC_SFP_PLUS = '16gfc-sfpp' + TYPE_32GFC_SFP28 = '32gfc-sfp28' + TYPE_128GFC_QSFP28 = '128gfc-sfp28' + + # InfiniBand + TYPE_INFINIBAND_SDR = 'inifiband-sdr' + TYPE_INFINIBAND_DDR = 'inifiband-ddr' + TYPE_INFINIBAND_QDR = 'inifiband-qdr' + TYPE_INFINIBAND_FDR10 = 'inifiband-fdr10' + TYPE_INFINIBAND_FDR = 'inifiband-fdr' + TYPE_INFINIBAND_EDR = 'inifiband-edr' + TYPE_INFINIBAND_HDR = 'inifiband-hdr' + TYPE_INFINIBAND_NDR = 'inifiband-ndr' + TYPE_INFINIBAND_XDR = 'inifiband-xdr' + + # Serial + TYPE_T1 = 't1' + TYPE_E1 = 'e1' + TYPE_T3 = 't3' + TYPE_E3 = 'e3' + + # Stacking + TYPE_STACKWISE = 'cisco-stackwise' + TYPE_STACKWISE_PLUS = 'cisco-stackwise-plus' + TYPE_FLEXSTACK = 'cisco-flexstack' + TYPE_FLEXSTACK_PLUS = 'cisco-flexstack-plus' + TYPE_JUNIPER_VCP = 'juniper-vcp' + TYPE_SUMMITSTACK = 'extreme-summitstack' + TYPE_SUMMITSTACK128 = 'extreme-summitstack-128' + TYPE_SUMMITSTACK256 = 'extreme-summitstack-256' + TYPE_SUMMITSTACK512 = 'extreme-summitstack-512' + + # Other + TYPE_OTHER = 'other' + + @classmethod + def as_choices(cls): + return ( + ( + 'Virtual interfaces', + ( + (cls.TYPE_VIRTUAL, 'Virtual'), + (cls.TYPE_LAG, 'Link Aggregation Group (LAG)'), + ), + ), + ( + 'Ethernet (fixed)', + ( + (cls.TYPE_100ME_FIXED, '100BASE-TX (10/100ME)'), + (cls.TYPE_1GE_FIXED, '1000BASE-T (1GE)'), + (cls.TYPE_2GE_FIXED, '2.5GBASE-T (2.5GE)'), + (cls.TYPE_5GE_FIXED, '5GBASE-T (5GE)'), + (cls.TYPE_10GE_FIXED, '10GBASE-T (10GE)'), + (cls.TYPE_10GE_CX4, '10GBASE-CX4 (10GE)'), + ) + ), + ( + 'Ethernet (modular)', + ( + (cls.TYPE_1GE_GBIC, 'GBIC (1GE)'), + (cls.TYPE_1GE_SFP, 'SFP (1GE)'), + (cls.TYPE_10GE_SFP_PLUS, 'SFP+ (10GE)'), + (cls.TYPE_10GE_XFP, 'XFP (10GE)'), + (cls.TYPE_10GE_XENPAK, 'XENPAK (10GE)'), + (cls.TYPE_10GE_X2, 'X2 (10GE)'), + (cls.TYPE_25GE_SFP28, 'SFP28 (25GE)'), + (cls.TYPE_40GE_QSFP_PLUS, 'QSFP+ (40GE)'), + (cls.TYPE_50GE_QSFP28, 'QSFP28 (50GE)'), + (cls.TYPE_100GE_CFP, 'CFP (100GE)'), + (cls.TYPE_100GE_CFP2, 'CFP2 (100GE)'), + (cls.TYPE_200GE_CFP2, 'CFP2 (200GE)'), + (cls.TYPE_100GE_CFP4, 'CFP4 (100GE)'), + (cls.TYPE_100GE_CPAK, 'Cisco CPAK (100GE)'), + (cls.TYPE_100GE_QSFP28, 'QSFP28 (100GE)'), + (cls.TYPE_200GE_QSFP56, 'QSFP56 (200GE)'), + (cls.TYPE_400GE_QSFP_DD, 'QSFP-DD (400GE)'), + ) + ), + ( + 'Wireless', + ( + (cls.TYPE_80211A, 'IEEE 802.11a'), + (cls.TYPE_80211G, 'IEEE 802.11b/g'), + (cls.TYPE_80211N, 'IEEE 802.11n'), + (cls.TYPE_80211AC, 'IEEE 802.11ac'), + (cls.TYPE_80211AD, 'IEEE 802.11ad'), + ) + ), + ( + 'Cellular', + ( + (cls.TYPE_GSM, 'GSM'), + (cls.TYPE_CDMA, 'CDMA'), + (cls.TYPE_LTE, 'LTE'), + ) + ), + ( + 'SONET', + ( + (cls.TYPE_SONET_OC3, 'OC-3/STM-1'), + (cls.TYPE_SONET_OC12, 'OC-12/STM-4'), + (cls.TYPE_SONET_OC48, 'OC-48/STM-16'), + (cls.TYPE_SONET_OC192, 'OC-192/STM-64'), + (cls.TYPE_SONET_OC768, 'OC-768/STM-256'), + (cls.TYPE_SONET_OC1920, 'OC-1920/STM-640'), + (cls.TYPE_SONET_OC3840, 'OC-3840/STM-1234'), + ) + ), + ( + 'FibreChannel', + ( + (cls.TYPE_1GFC_SFP, 'SFP (1GFC)'), + (cls.TYPE_2GFC_SFP, 'SFP (2GFC)'), + (cls.TYPE_4GFC_SFP, 'SFP (4GFC)'), + (cls.TYPE_8GFC_SFP_PLUS, 'SFP+ (8GFC)'), + (cls.TYPE_16GFC_SFP_PLUS, 'SFP+ (16GFC)'), + (cls.TYPE_32GFC_SFP28, 'SFP28 (32GFC)'), + (cls.TYPE_128GFC_QSFP28, 'QSFP28 (128GFC)'), + ) + ), + ( + 'InfiniBand', + ( + (cls.TYPE_INFINIBAND_SDR, 'SDR (2 Gbps)'), + (cls.TYPE_INFINIBAND_DDR, 'DDR (4 Gbps)'), + (cls.TYPE_INFINIBAND_QDR, 'QDR (8 Gbps)'), + (cls.TYPE_INFINIBAND_FDR10, 'FDR10 (10 Gbps)'), + (cls.TYPE_INFINIBAND_FDR, 'FDR (13.5 Gbps)'), + (cls.TYPE_INFINIBAND_EDR, 'EDR (25 Gbps)'), + (cls.TYPE_INFINIBAND_HDR, 'HDR (50 Gbps)'), + (cls.TYPE_INFINIBAND_NDR, 'NDR (100 Gbps)'), + (cls.TYPE_INFINIBAND_XDR, 'XDR (250 Gbps)'), + ) + ), + ( + 'Serial', + ( + (cls.TYPE_T1, 'T1 (1.544 Mbps)'), + (cls.TYPE_E1, 'E1 (2.048 Mbps)'), + (cls.TYPE_T3, 'T3 (45 Mbps)'), + (cls.TYPE_E3, 'E3 (34 Mbps)'), + ) + ), + ( + 'Stacking', + ( + (cls.TYPE_STACKWISE, 'Cisco StackWise'), + (cls.TYPE_STACKWISE_PLUS, 'Cisco StackWise Plus'), + (cls.TYPE_FLEXSTACK, 'Cisco FlexStack'), + (cls.TYPE_FLEXSTACK_PLUS, 'Cisco FlexStack Plus'), + (cls.TYPE_JUNIPER_VCP, 'Juniper VCP'), + (cls.TYPE_SUMMITSTACK, 'Extreme SummitStack'), + (cls.TYPE_SUMMITSTACK128, 'Extreme SummitStack-128'), + (cls.TYPE_SUMMITSTACK256, 'Extreme SummitStack-256'), + (cls.TYPE_SUMMITSTACK512, 'Extreme SummitStack-512'), + ) + ), + ( + 'Other', + ( + (cls.TYPE_OTHER, 'Other'), + ) + ), + ) + + @classmethod + def slug_to_integer(cls, slug): + """ + Provide backward-compatible mapping of the type slug to integer. + """ + return { + # Slug: integer + cls.TYPE_VIRTUAL: IFACE_TYPE_VIRTUAL, + cls.TYPE_LAG: IFACE_TYPE_LAG, + cls.TYPE_100ME_FIXED: IFACE_TYPE_100ME_FIXED, + cls.TYPE_1GE_FIXED: IFACE_TYPE_1GE_FIXED, + cls.TYPE_1GE_GBIC: IFACE_TYPE_1GE_GBIC, + cls.TYPE_1GE_SFP: IFACE_TYPE_1GE_SFP, + cls.TYPE_2GE_FIXED: IFACE_TYPE_2GE_FIXED, + cls.TYPE_5GE_FIXED: IFACE_TYPE_5GE_FIXED, + cls.TYPE_10GE_FIXED: IFACE_TYPE_10GE_FIXED, + cls.TYPE_10GE_CX4: IFACE_TYPE_10GE_CX4, + cls.TYPE_10GE_SFP_PLUS: IFACE_TYPE_10GE_SFP_PLUS, + cls.TYPE_10GE_XFP: IFACE_TYPE_10GE_XFP, + cls.TYPE_10GE_XENPAK: IFACE_TYPE_10GE_XENPAK, + cls.TYPE_10GE_X2: IFACE_TYPE_10GE_X2, + cls.TYPE_25GE_SFP28: IFACE_TYPE_25GE_SFP28, + cls.TYPE_40GE_QSFP_PLUS: IFACE_TYPE_40GE_QSFP_PLUS, + cls.TYPE_50GE_QSFP28: IFACE_TYPE_50GE_QSFP28, + cls.TYPE_100GE_CFP: IFACE_TYPE_100GE_CFP, + cls.TYPE_100GE_CFP2: IFACE_TYPE_100GE_CFP2, + cls.TYPE_100GE_CFP4: IFACE_TYPE_100GE_CFP4, + cls.TYPE_100GE_CPAK: IFACE_TYPE_100GE_CPAK, + cls.TYPE_100GE_QSFP28: IFACE_TYPE_100GE_QSFP28, + cls.TYPE_200GE_CFP2: IFACE_TYPE_200GE_CFP2, + cls.TYPE_200GE_QSFP56: IFACE_TYPE_200GE_QSFP56, + cls.TYPE_400GE_QSFP_DD: IFACE_TYPE_400GE_QSFP_DD, + cls.TYPE_80211A: IFACE_TYPE_80211A, + cls.TYPE_80211G: IFACE_TYPE_80211G, + cls.TYPE_80211N: IFACE_TYPE_80211N, + cls.TYPE_80211AC: IFACE_TYPE_80211AC, + cls.TYPE_80211AD: IFACE_TYPE_80211AD, + cls.TYPE_GSM: IFACE_TYPE_GSM, + cls.TYPE_CDMA: IFACE_TYPE_CDMA, + cls.TYPE_LTE: IFACE_TYPE_LTE, + cls.TYPE_SONET_OC3: IFACE_TYPE_SONET_OC3, + cls.TYPE_SONET_OC12: IFACE_TYPE_SONET_OC12, + cls.TYPE_SONET_OC48: IFACE_TYPE_SONET_OC48, + cls.TYPE_SONET_OC192: IFACE_TYPE_SONET_OC192, + cls.TYPE_SONET_OC768: IFACE_TYPE_SONET_OC768, + cls.TYPE_SONET_OC1920: IFACE_TYPE_SONET_OC1920, + cls.TYPE_SONET_OC3840: IFACE_TYPE_SONET_OC3840, + cls.TYPE_1GFC_SFP: IFACE_TYPE_1GFC_SFP, + cls.TYPE_2GFC_SFP: IFACE_TYPE_2GFC_SFP, + cls.TYPE_4GFC_SFP: IFACE_TYPE_4GFC_SFP, + cls.TYPE_8GFC_SFP_PLUS: IFACE_TYPE_8GFC_SFP_PLUS, + cls.TYPE_16GFC_SFP_PLUS: IFACE_TYPE_16GFC_SFP_PLUS, + cls.TYPE_32GFC_SFP28: IFACE_TYPE_32GFC_SFP28, + cls.TYPE_128GFC_QSFP28: IFACE_TYPE_128GFC_QSFP28, + cls.TYPE_INFINIBAND_SDR: IFACE_TYPE_INFINIBAND_SDR, + cls.TYPE_INFINIBAND_DDR: IFACE_TYPE_INFINIBAND_DDR, + cls.TYPE_INFINIBAND_QDR: IFACE_TYPE_INFINIBAND_QDR, + cls.TYPE_INFINIBAND_FDR10: IFACE_TYPE_INFINIBAND_FDR10, + cls.TYPE_INFINIBAND_FDR: IFACE_TYPE_INFINIBAND_FDR, + cls.TYPE_INFINIBAND_EDR: IFACE_TYPE_INFINIBAND_EDR, + cls.TYPE_INFINIBAND_HDR: IFACE_TYPE_INFINIBAND_HDR, + cls.TYPE_INFINIBAND_NDR: IFACE_TYPE_INFINIBAND_NDR, + cls.TYPE_INFINIBAND_XDR: IFACE_TYPE_INFINIBAND_XDR, + cls.TYPE_T1: IFACE_TYPE_T1, + cls.TYPE_E1: IFACE_TYPE_E1, + cls.TYPE_T3: IFACE_TYPE_T3, + cls.TYPE_E3: IFACE_TYPE_E3, + cls.TYPE_STACKWISE: IFACE_TYPE_STACKWISE, + cls.TYPE_STACKWISE_PLUS: IFACE_TYPE_STACKWISE_PLUS, + cls.TYPE_FLEXSTACK: IFACE_TYPE_FLEXSTACK, + cls.TYPE_FLEXSTACK_PLUS: IFACE_TYPE_FLEXSTACK_PLUS, + cls.TYPE_JUNIPER_VCP: IFACE_TYPE_JUNIPER_VCP, + cls.TYPE_SUMMITSTACK: IFACE_TYPE_SUMMITSTACK, + cls.TYPE_SUMMITSTACK128: IFACE_TYPE_SUMMITSTACK128, + cls.TYPE_SUMMITSTACK256: IFACE_TYPE_SUMMITSTACK256, + cls.TYPE_SUMMITSTACK512: IFACE_TYPE_SUMMITSTACK512, + }.get(slug) + + +# +# Port type values +# + +class PortTypes: + """ + FrontPort/RearPort.type slugs + """ + TYPE_8P8C = '8p8c' + TYPE_110_PUNCH = '110-punch' + TYPE_BNC = 'bnc' + TYPE_ST = 'st' + TYPE_SC = 'sc' + TYPE_SC_APC = 'sc-apc' + TYPE_FC = 'fc' + TYPE_LC = 'lc' + TYPE_LC_APC = 'lc-apc' + TYPE_MTRJ = 'mtrj' + TYPE_MPO = 'mpo' + TYPE_LSH = 'lsh' + TYPE_LSH_APC = 'lsh-apc' + + @classmethod + def as_choices(cls): + return ( + ( + 'Copper', + ( + (cls.TYPE_8P8C, '8P8C'), + (cls.TYPE_110_PUNCH, '110 Punch'), + (cls.TYPE_BNC, 'BNC'), + ), + ), + ( + 'Fiber Optic', + ( + (cls.TYPE_FC, 'FC'), + (cls.TYPE_LC, 'LC'), + (cls.TYPE_LC_APC, 'LC/APC'), + (cls.TYPE_LSH, 'LSH'), + (cls.TYPE_LSH_APC, 'LSH/APC'), + (cls.TYPE_MPO, 'MPO'), + (cls.TYPE_MTRJ, 'MTRJ'), + (cls.TYPE_SC, 'SC'), + (cls.TYPE_SC_APC, 'SC/APC'), + (cls.TYPE_ST, 'ST'), + ) + ) + ) + + @classmethod + def slug_to_integer(cls, slug): + """ + Provide backward-compatible mapping of the type slug to integer. + """ + return { + # Slug: integer + cls.TYPE_8P8C: PORT_TYPE_8P8C, + cls.TYPE_110_PUNCH: PORT_TYPE_8P8C, + cls.TYPE_BNC: PORT_TYPE_BNC, + cls.TYPE_ST: PORT_TYPE_ST, + cls.TYPE_SC: PORT_TYPE_SC, + cls.TYPE_SC_APC: PORT_TYPE_SC_APC, + cls.TYPE_FC: PORT_TYPE_FC, + cls.TYPE_LC: PORT_TYPE_LC, + cls.TYPE_LC_APC: PORT_TYPE_LC_APC, + cls.TYPE_MTRJ: PORT_TYPE_MTRJ, + cls.TYPE_MPO: PORT_TYPE_MPO, + cls.TYPE_LSH: PORT_TYPE_LSH, + cls.TYPE_LSH_APC: PORT_TYPE_LSH_APC, + }.get(slug) diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 4597a05e3..279b00dd9 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -1289,6 +1289,9 @@ class PowerOutletTemplateImportForm(ComponentTemplateImportForm): class InterfaceTemplateImportForm(ComponentTemplateImportForm): + type = forms.ChoiceField( + choices=InterfaceTypes.as_choices() + ) class Meta: model = InterfaceTemplate @@ -1296,8 +1299,16 @@ class InterfaceTemplateImportForm(ComponentTemplateImportForm): 'device_type', 'name', 'type', 'mgmt_only', ] + def clean_type(self): + # Convert slug value to field integer value + slug = self.cleaned_data['type'] + return InterfaceTypes.slug_to_integer(slug) + class FrontPortTemplateImportForm(ComponentTemplateImportForm): + type = forms.ChoiceField( + choices=PortTypes.as_choices() + ) rear_port = forms.ModelChoiceField( queryset=RearPortTemplate.objects.all(), to_field_name='name', @@ -1310,8 +1321,16 @@ class FrontPortTemplateImportForm(ComponentTemplateImportForm): 'device_type', 'name', 'type', 'rear_port', 'rear_port_position', ] + def clean_type(self): + # Convert slug value to field integer value + slug = self.cleaned_data['type'] + return PortTypes.slug_to_integer(slug) + class RearPortTemplateImportForm(ComponentTemplateImportForm): + type = forms.ChoiceField( + choices=PortTypes.as_choices() + ) class Meta: model = RearPortTemplate @@ -1319,6 +1338,11 @@ class RearPortTemplateImportForm(ComponentTemplateImportForm): 'device_type', 'name', 'type', 'positions', ] + def clean_type(self): + # Convert slug value to field integer value + slug = self.cleaned_data['type'] + return PortTypes.slug_to_integer(slug) + class DeviceBayTemplateImportForm(ComponentTemplateImportForm): diff --git a/netbox/dcim/tests/test_views.py b/netbox/dcim/tests/test_views.py index 754a2dd83..d4572cc39 100644 --- a/netbox/dcim/tests/test_views.py +++ b/netbox/dcim/tests/test_views.py @@ -253,28 +253,28 @@ power-outlets: feed_leg: 1 interfaces: - name: Interface 1 - type: 1000 + type: 1000base-t mgmt_only: true - name: Interface 2 - type: 1000 + type: 1000base-t - name: Interface 3 - type: 1000 + type: 1000base-t rear-ports: - name: Rear Port 1 - type: 1000 + type: 8p8c - name: Rear Port 2 - type: 1000 + type: 8p8c - name: Rear Port 3 - type: 1000 + type: 8p8c front-ports: - name: Front Port 1 - type: 1000 + type: 8p8c rear_port: Rear Port 1 - name: Front Port 2 - type: 1000 + type: 8p8c rear_port: Rear Port 2 - name: Front Port 3 - type: 1000 + type: 8p8c rear_port: Rear Port 3 device-bays: - name: Device Bay 1 From 0d15ac15aeedb71c97e12e325138f1406ff5268f Mon Sep 17 00:00:00 2001 From: John Anderson Date: Sun, 13 Oct 2019 02:49:54 -0400 Subject: [PATCH 24/42] implements #3282 - seperate webhooks and caching redis configs --- docs/configuration/required-settings.md | 44 +++++++++++-------- docs/installation/2-netbox.md | 23 ++++++---- docs/release-notes/version-2.7.md | 48 +++++++++++++++++++++ netbox/extras/apps.py | 10 ++--- netbox/netbox/configuration.example.py | 25 +++++++---- netbox/netbox/settings.py | 56 +++++++++++++++++-------- 6 files changed, 152 insertions(+), 54 deletions(-) diff --git a/docs/configuration/required-settings.md b/docs/configuration/required-settings.md index e4f2aed97..f3ea9245b 100644 --- a/docs/configuration/required-settings.md +++ b/docs/configuration/required-settings.md @@ -24,7 +24,7 @@ NetBox requires access to a PostgreSQL database service to store data. This serv Example: -``` +```python DATABASE = { 'NAME': 'netbox', # Database name 'USER': 'netbox', # PostgreSQL username @@ -40,40 +40,48 @@ DATABASE = { [Redis](https://redis.io/) is an in-memory data store similar to memcached. While Redis has been an optional component of NetBox since the introduction of webhooks in version 2.4, it is required starting in 2.6 to support NetBox's caching -functionality (as well as other planned features). +functionality (as well as other planned features). In 2.7, the connection settings were broken down into two sections for +webhooks and caching, allowing the user to connect to different Redis instances/databases per feature. -Redis is configured using a configuration setting similar to `DATABASE`: +Redis is configured using a configuration setting similar to `DATABASE` and these settings are the same for both of the `webhooks` and `caching` subsections: * `HOST` - Name or IP address of the Redis server (use `localhost` if running locally) * `PORT` - TCP port of the Redis service; leave blank for default port (6379) * `PASSWORD` - Redis password (if set) -* `DATABASE` - Numeric database ID for webhooks -* `CACHE_DATABASE` - Numeric database ID for caching +* `DATABASE` - Numeric database ID * `DEFAULT_TIMEOUT` - Connection timeout in seconds * `SSL` - Use SSL connection to Redis Example: -``` +```python REDIS = { - 'HOST': 'localhost', - 'PORT': 6379, - 'PASSWORD': '', - 'DATABASE': 0, - 'CACHE_DATABASE': 1, - 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'webhooks': { + 'HOST': 'redis.example.com', + 'PORT': 1234, + 'PASSWORD': 'foobar', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'HOST': 'localhost', + 'PORT': 6379, + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } } ``` !!! note: - If you were using these settings in a prior release with webhooks, the `DATABASE` setting remains the same but - an additional `CACHE_DATABASE` setting has been added with a default value of 1 to support the caching backend. The - `DATABASE` setting will be renamed in a future release of NetBox to better relay the meaning of the setting. + If you are upgrading from a version prior to v2.7, please note that the Redis connection configuration settings have + changed. Manual modification to bring the `REDIS` section inline with the above specification is necessary !!! warning: - It is highly recommended to keep the webhook and cache databases seperate. Using the same database number for both may result in webhook - processing data being lost in cache flushing events. + It is highly recommended to keep the webhook and cache databases seperate. Using the same database number on the + same Redis instance for both may result in webhook processing data being lost during cache flushing events. --- diff --git a/docs/installation/2-netbox.md b/docs/installation/2-netbox.md index 4add03300..4f11fee83 100644 --- a/docs/installation/2-netbox.md +++ b/docs/installation/2-netbox.md @@ -138,13 +138,22 @@ Redis is a in-memory key-value store required as part of the NetBox installation ```python REDIS = { - 'HOST': 'localhost', - 'PORT': 6379, - 'PASSWORD': '', - 'DATABASE': 0, - 'CACHE_DATABASE': 1, - 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'webhooks': { + 'HOST': 'redis.example.com', + 'PORT': 1234, + 'PASSWORD': 'foobar', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'HOST': 'localhost', + 'PORT': 6379, + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } } ``` diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index d997aa9c2..3504eccdc 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,5 +1,53 @@ v2.7.0 (FUTURE) +## Changes + +### Redis Configuration ([#3282](https://github.com/netbox-community/netbox/issues/3282)) + +v2.6.0 introduced caching and added the `CACHE_DATABASE` option to the existing `REDIS` database configuration section. +This did not however, allow for using two different Redis connections for the seperate caching and webhooks features. +This change separates the Redis connection configurations in the `REDIS` section into distinct `webhooks` and `caching` subsections. +This requires modification of the `REDIS` section of the `configuration.py` file as follows: + +Old Redis configuration: +```python +REDIS = { + 'HOST': 'localhost', + 'PORT': 6379, + 'PASSWORD': '', + 'DATABASE': 0, + 'CACHE_DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, +} +``` + +New Redis configuration: +```python +REDIS = { + 'webhooks': { + 'HOST': 'redis.example.com', + 'PORT': 1234, + 'PASSWORD': 'foobar', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'HOST': 'localhost', + 'PORT': 6379, + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } +} +``` + +Note that `CACHE_DATABASE` has been removed and the connection settings have been duplicated for both `webhooks` and `caching`. +This allows the user to make use of separate Redis instances and/or databases if desired. +Full connection details are required in both sections, even if they are the same. + ## Enhancements * [#2902](https://github.com/digitalocean/netbox/issues/2902) - Replace supervisord with systemd diff --git a/netbox/extras/apps.py b/netbox/extras/apps.py index 6bb3b9fca..25c7cd5a2 100644 --- a/netbox/extras/apps.py +++ b/netbox/extras/apps.py @@ -21,11 +21,11 @@ class ExtrasConfig(AppConfig): ) try: rs = redis.Redis( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - db=settings.REDIS_DATABASE, - password=settings.REDIS_PASSWORD or None, - ssl=settings.REDIS_SSL, + host=settings.WEBHOOKS_REDIS_HOST, + port=settings.WEBHOOKS_REDIS_PORT, + db=settings.WEBHOOKS_REDIS_DATABASE, + password=settings.WEBHOOKS_REDIS_PASSWORD or None, + ssl=settings.WEBHOOKS_REDIS_SSL, ) rs.ping() except redis.exceptions.ConnectionError: diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index ebc3d4540..d5ac444d2 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -26,14 +26,25 @@ DATABASE = { SECRET_KEY = '' # Redis database settings. The Redis database is used for caching and background processing such as webhooks +# Seperate sections for webhooks and caching allow for connecting to seperate Redis instances/datbases if desired. +# Full connection details are required in both sections, even if they are the same. REDIS = { - 'HOST': 'localhost', - 'PORT': 6379, - 'PASSWORD': '', - 'DATABASE': 0, - 'CACHE_DATABASE': 1, - 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'webhooks': { + 'HOST': 'redis.example.com', + 'PORT': 1234, + 'PASSWORD': 'foobar', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'HOST': 'localhost', + 'PORT': 6379, + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } } diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index c2f51d295..a7c797fce 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -118,13 +118,30 @@ DATABASES = { # Redis # -REDIS_HOST = REDIS.get('HOST', 'localhost') -REDIS_PORT = REDIS.get('PORT', 6379) -REDIS_PASSWORD = REDIS.get('PASSWORD', '') -REDIS_DATABASE = REDIS.get('DATABASE', 0) -REDIS_CACHE_DATABASE = REDIS.get('CACHE_DATABASE', 1) -REDIS_DEFAULT_TIMEOUT = REDIS.get('DEFAULT_TIMEOUT', 300) -REDIS_SSL = REDIS.get('SSL', False) +if 'webhooks' not in REDIS: + raise ImproperlyConfigured( + "REDIS section in configuration.py is missing webhooks subsection." + ) +if 'caching' not in REDIS: + raise ImproperlyConfigured( + "REDIS section in configuration.py is missing caching subsection." + ) + +WEBHOOKS_REDIS = REDIS.get('webhooks', {}) +WEBHOOKS_REDIS_HOST = WEBHOOKS_REDIS.get('HOST', 'localhost') +WEBHOOKS_REDIS_PORT = WEBHOOKS_REDIS.get('PORT', 6379) +WEBHOOKS_REDIS_PASSWORD = WEBHOOKS_REDIS.get('PASSWORD', '') +WEBHOOKS_REDIS_DATABASE = WEBHOOKS_REDIS.get('DATABASE', 0) +WEBHOOKS_REDIS_DEFAULT_TIMEOUT = WEBHOOKS_REDIS.get('DEFAULT_TIMEOUT', 300) +WEBHOOKS_REDIS_SSL = WEBHOOKS_REDIS.get('SSL', False) + +CACHING_REDIS = REDIS.get('caching', {}) +CACHING_REDIS_HOST = WEBHOOKS_REDIS.get('HOST', 'localhost') +CACHING_REDIS_PORT = WEBHOOKS_REDIS.get('PORT', 6379) +CACHING_REDIS_PASSWORD = WEBHOOKS_REDIS.get('PASSWORD', '') +CACHING_REDIS_DATABASE = WEBHOOKS_REDIS.get('DATABASE', 0) +CACHING_REDIS_DEFAULT_TIMEOUT = WEBHOOKS_REDIS.get('DEFAULT_TIMEOUT', 300) +CACHING_REDIS_SSL = WEBHOOKS_REDIS.get('SSL', False) # @@ -341,15 +358,20 @@ if LDAP_CONFIG is not None: # Caching # -if REDIS_SSL: +if CACHING_REDIS_SSL: REDIS_CACHE_CON_STRING = 'rediss://' else: REDIS_CACHE_CON_STRING = 'redis://' -if REDIS_PASSWORD: - REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, REDIS_PASSWORD) +if CACHING_REDIS_PASSWORD: + REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, CACHING_REDIS_PASSWORD) -REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format(REDIS_CACHE_CON_STRING, REDIS_HOST, REDIS_PORT, REDIS_CACHE_DATABASE) +REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format( + REDIS_CACHE_CON_STRING, + CACHING_REDIS_HOST, + CACHING_REDIS_PORT, + CACHING_REDIS_DATABASE +) if not CACHE_TIMEOUT: CACHEOPS_ENABLED = False @@ -467,12 +489,12 @@ SWAGGER_SETTINGS = { RQ_QUEUES = { 'default': { - 'HOST': REDIS_HOST, - 'PORT': REDIS_PORT, - 'DB': REDIS_DATABASE, - 'PASSWORD': REDIS_PASSWORD, - 'DEFAULT_TIMEOUT': REDIS_DEFAULT_TIMEOUT, - 'SSL': REDIS_SSL, + 'HOST': WEBHOOKS_REDIS_HOST, + 'PORT': WEBHOOKS_REDIS_PORT, + 'DB': WEBHOOKS_REDIS_DATABASE, + 'PASSWORD': WEBHOOKS_REDIS_PASSWORD, + 'DEFAULT_TIMEOUT': WEBHOOKS_REDIS_DEFAULT_TIMEOUT, + 'SSL': WEBHOOKS_REDIS_SSL, } } From 18e0bd89374f4d0b2b6a248f8149e62ac69df46c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20L=C3=B6nnegren?= Date: Wed, 16 Oct 2019 08:22:06 +0200 Subject: [PATCH 25/42] Fix typo --- netbox/templates/tenancy/tenant.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/templates/tenancy/tenant.html b/netbox/templates/tenancy/tenant.html index 72b17f10c..a03d60523 100644 --- a/netbox/templates/tenancy/tenant.html +++ b/netbox/templates/tenancy/tenant.html @@ -143,7 +143,7 @@

Virtual machines

-

{{ stats.cluster_count }}

+

{{ stats.cluster_count }}

Clusters

From adc96702db0ae237177941203df3c0193e58308e Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 17 Oct 2019 15:43:49 -0400 Subject: [PATCH 26/42] Deleted errant import of graphviz --- netbox/extras/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/netbox/extras/models.py b/netbox/extras/models.py index 941580137..5a7cf04d0 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -1,7 +1,6 @@ from collections import OrderedDict from datetime import date -import graphviz from django.contrib.auth.models import User from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType From 8ca182571c06dc5d08ab81f3b18e8f6a4c0af86e Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 17 Oct 2019 15:53:10 -0400 Subject: [PATCH 27/42] Rebase schema migrations --- ...026_remove_topology_maps.py => 0028_remove_topology_maps.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename netbox/extras/migrations/{0026_remove_topology_maps.py => 0028_remove_topology_maps.py} (81%) diff --git a/netbox/extras/migrations/0026_remove_topology_maps.py b/netbox/extras/migrations/0028_remove_topology_maps.py similarity index 81% rename from netbox/extras/migrations/0026_remove_topology_maps.py rename to netbox/extras/migrations/0028_remove_topology_maps.py index 40b36d3b5..834586f4f 100644 --- a/netbox/extras/migrations/0026_remove_topology_maps.py +++ b/netbox/extras/migrations/0028_remove_topology_maps.py @@ -6,7 +6,7 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ('extras', '0025_objectchange_time_index'), + ('extras', '0027_webhook_additional_headers'), ] operations = [ From 2ffbced47ed56351c69a7d68a9dea7a7b6325291 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 17 Oct 2019 16:38:31 -0400 Subject: [PATCH 28/42] Rework InterfaceTypes and PortTypes classes --- netbox/dcim/constants.py | 274 +++++++++++++++++++-------------------- netbox/dcim/forms.py | 6 +- 2 files changed, 138 insertions(+), 142 deletions(-) diff --git a/netbox/dcim/constants.py b/netbox/dcim/constants.py index 9d20229a1..02662d9f8 100644 --- a/netbox/dcim/constants.py +++ b/netbox/dcim/constants.py @@ -613,135 +613,133 @@ class InterfaceTypes: # Other TYPE_OTHER = 'other' - @classmethod - def as_choices(cls): - return ( + TYPE_CHOICES = ( + ( + 'Virtual interfaces', ( - 'Virtual interfaces', - ( - (cls.TYPE_VIRTUAL, 'Virtual'), - (cls.TYPE_LAG, 'Link Aggregation Group (LAG)'), - ), + (TYPE_VIRTUAL, 'Virtual'), + (TYPE_LAG, 'Link Aggregation Group (LAG)'), ), + ), + ( + 'Ethernet (fixed)', ( - 'Ethernet (fixed)', - ( - (cls.TYPE_100ME_FIXED, '100BASE-TX (10/100ME)'), - (cls.TYPE_1GE_FIXED, '1000BASE-T (1GE)'), - (cls.TYPE_2GE_FIXED, '2.5GBASE-T (2.5GE)'), - (cls.TYPE_5GE_FIXED, '5GBASE-T (5GE)'), - (cls.TYPE_10GE_FIXED, '10GBASE-T (10GE)'), - (cls.TYPE_10GE_CX4, '10GBASE-CX4 (10GE)'), - ) - ), + (TYPE_100ME_FIXED, '100BASE-TX (10/100ME)'), + (TYPE_1GE_FIXED, '1000BASE-T (1GE)'), + (TYPE_2GE_FIXED, '2.5GBASE-T (2.5GE)'), + (TYPE_5GE_FIXED, '5GBASE-T (5GE)'), + (TYPE_10GE_FIXED, '10GBASE-T (10GE)'), + (TYPE_10GE_CX4, '10GBASE-CX4 (10GE)'), + ) + ), + ( + 'Ethernet (modular)', ( - 'Ethernet (modular)', - ( - (cls.TYPE_1GE_GBIC, 'GBIC (1GE)'), - (cls.TYPE_1GE_SFP, 'SFP (1GE)'), - (cls.TYPE_10GE_SFP_PLUS, 'SFP+ (10GE)'), - (cls.TYPE_10GE_XFP, 'XFP (10GE)'), - (cls.TYPE_10GE_XENPAK, 'XENPAK (10GE)'), - (cls.TYPE_10GE_X2, 'X2 (10GE)'), - (cls.TYPE_25GE_SFP28, 'SFP28 (25GE)'), - (cls.TYPE_40GE_QSFP_PLUS, 'QSFP+ (40GE)'), - (cls.TYPE_50GE_QSFP28, 'QSFP28 (50GE)'), - (cls.TYPE_100GE_CFP, 'CFP (100GE)'), - (cls.TYPE_100GE_CFP2, 'CFP2 (100GE)'), - (cls.TYPE_200GE_CFP2, 'CFP2 (200GE)'), - (cls.TYPE_100GE_CFP4, 'CFP4 (100GE)'), - (cls.TYPE_100GE_CPAK, 'Cisco CPAK (100GE)'), - (cls.TYPE_100GE_QSFP28, 'QSFP28 (100GE)'), - (cls.TYPE_200GE_QSFP56, 'QSFP56 (200GE)'), - (cls.TYPE_400GE_QSFP_DD, 'QSFP-DD (400GE)'), - ) - ), + (TYPE_1GE_GBIC, 'GBIC (1GE)'), + (TYPE_1GE_SFP, 'SFP (1GE)'), + (TYPE_10GE_SFP_PLUS, 'SFP+ (10GE)'), + (TYPE_10GE_XFP, 'XFP (10GE)'), + (TYPE_10GE_XENPAK, 'XENPAK (10GE)'), + (TYPE_10GE_X2, 'X2 (10GE)'), + (TYPE_25GE_SFP28, 'SFP28 (25GE)'), + (TYPE_40GE_QSFP_PLUS, 'QSFP+ (40GE)'), + (TYPE_50GE_QSFP28, 'QSFP28 (50GE)'), + (TYPE_100GE_CFP, 'CFP (100GE)'), + (TYPE_100GE_CFP2, 'CFP2 (100GE)'), + (TYPE_200GE_CFP2, 'CFP2 (200GE)'), + (TYPE_100GE_CFP4, 'CFP4 (100GE)'), + (TYPE_100GE_CPAK, 'Cisco CPAK (100GE)'), + (TYPE_100GE_QSFP28, 'QSFP28 (100GE)'), + (TYPE_200GE_QSFP56, 'QSFP56 (200GE)'), + (TYPE_400GE_QSFP_DD, 'QSFP-DD (400GE)'), + ) + ), + ( + 'Wireless', ( - 'Wireless', - ( - (cls.TYPE_80211A, 'IEEE 802.11a'), - (cls.TYPE_80211G, 'IEEE 802.11b/g'), - (cls.TYPE_80211N, 'IEEE 802.11n'), - (cls.TYPE_80211AC, 'IEEE 802.11ac'), - (cls.TYPE_80211AD, 'IEEE 802.11ad'), - ) - ), + (TYPE_80211A, 'IEEE 802.11a'), + (TYPE_80211G, 'IEEE 802.11b/g'), + (TYPE_80211N, 'IEEE 802.11n'), + (TYPE_80211AC, 'IEEE 802.11ac'), + (TYPE_80211AD, 'IEEE 802.11ad'), + ) + ), + ( + 'Cellular', ( - 'Cellular', - ( - (cls.TYPE_GSM, 'GSM'), - (cls.TYPE_CDMA, 'CDMA'), - (cls.TYPE_LTE, 'LTE'), - ) - ), + (TYPE_GSM, 'GSM'), + (TYPE_CDMA, 'CDMA'), + (TYPE_LTE, 'LTE'), + ) + ), + ( + 'SONET', ( - 'SONET', - ( - (cls.TYPE_SONET_OC3, 'OC-3/STM-1'), - (cls.TYPE_SONET_OC12, 'OC-12/STM-4'), - (cls.TYPE_SONET_OC48, 'OC-48/STM-16'), - (cls.TYPE_SONET_OC192, 'OC-192/STM-64'), - (cls.TYPE_SONET_OC768, 'OC-768/STM-256'), - (cls.TYPE_SONET_OC1920, 'OC-1920/STM-640'), - (cls.TYPE_SONET_OC3840, 'OC-3840/STM-1234'), - ) - ), + (TYPE_SONET_OC3, 'OC-3/STM-1'), + (TYPE_SONET_OC12, 'OC-12/STM-4'), + (TYPE_SONET_OC48, 'OC-48/STM-16'), + (TYPE_SONET_OC192, 'OC-192/STM-64'), + (TYPE_SONET_OC768, 'OC-768/STM-256'), + (TYPE_SONET_OC1920, 'OC-1920/STM-640'), + (TYPE_SONET_OC3840, 'OC-3840/STM-1234'), + ) + ), + ( + 'FibreChannel', ( - 'FibreChannel', - ( - (cls.TYPE_1GFC_SFP, 'SFP (1GFC)'), - (cls.TYPE_2GFC_SFP, 'SFP (2GFC)'), - (cls.TYPE_4GFC_SFP, 'SFP (4GFC)'), - (cls.TYPE_8GFC_SFP_PLUS, 'SFP+ (8GFC)'), - (cls.TYPE_16GFC_SFP_PLUS, 'SFP+ (16GFC)'), - (cls.TYPE_32GFC_SFP28, 'SFP28 (32GFC)'), - (cls.TYPE_128GFC_QSFP28, 'QSFP28 (128GFC)'), - ) - ), + (TYPE_1GFC_SFP, 'SFP (1GFC)'), + (TYPE_2GFC_SFP, 'SFP (2GFC)'), + (TYPE_4GFC_SFP, 'SFP (4GFC)'), + (TYPE_8GFC_SFP_PLUS, 'SFP+ (8GFC)'), + (TYPE_16GFC_SFP_PLUS, 'SFP+ (16GFC)'), + (TYPE_32GFC_SFP28, 'SFP28 (32GFC)'), + (TYPE_128GFC_QSFP28, 'QSFP28 (128GFC)'), + ) + ), + ( + 'InfiniBand', ( - 'InfiniBand', - ( - (cls.TYPE_INFINIBAND_SDR, 'SDR (2 Gbps)'), - (cls.TYPE_INFINIBAND_DDR, 'DDR (4 Gbps)'), - (cls.TYPE_INFINIBAND_QDR, 'QDR (8 Gbps)'), - (cls.TYPE_INFINIBAND_FDR10, 'FDR10 (10 Gbps)'), - (cls.TYPE_INFINIBAND_FDR, 'FDR (13.5 Gbps)'), - (cls.TYPE_INFINIBAND_EDR, 'EDR (25 Gbps)'), - (cls.TYPE_INFINIBAND_HDR, 'HDR (50 Gbps)'), - (cls.TYPE_INFINIBAND_NDR, 'NDR (100 Gbps)'), - (cls.TYPE_INFINIBAND_XDR, 'XDR (250 Gbps)'), - ) - ), + (TYPE_INFINIBAND_SDR, 'SDR (2 Gbps)'), + (TYPE_INFINIBAND_DDR, 'DDR (4 Gbps)'), + (TYPE_INFINIBAND_QDR, 'QDR (8 Gbps)'), + (TYPE_INFINIBAND_FDR10, 'FDR10 (10 Gbps)'), + (TYPE_INFINIBAND_FDR, 'FDR (13.5 Gbps)'), + (TYPE_INFINIBAND_EDR, 'EDR (25 Gbps)'), + (TYPE_INFINIBAND_HDR, 'HDR (50 Gbps)'), + (TYPE_INFINIBAND_NDR, 'NDR (100 Gbps)'), + (TYPE_INFINIBAND_XDR, 'XDR (250 Gbps)'), + ) + ), + ( + 'Serial', ( - 'Serial', - ( - (cls.TYPE_T1, 'T1 (1.544 Mbps)'), - (cls.TYPE_E1, 'E1 (2.048 Mbps)'), - (cls.TYPE_T3, 'T3 (45 Mbps)'), - (cls.TYPE_E3, 'E3 (34 Mbps)'), - ) - ), + (TYPE_T1, 'T1 (1.544 Mbps)'), + (TYPE_E1, 'E1 (2.048 Mbps)'), + (TYPE_T3, 'T3 (45 Mbps)'), + (TYPE_E3, 'E3 (34 Mbps)'), + ) + ), + ( + 'Stacking', ( - 'Stacking', - ( - (cls.TYPE_STACKWISE, 'Cisco StackWise'), - (cls.TYPE_STACKWISE_PLUS, 'Cisco StackWise Plus'), - (cls.TYPE_FLEXSTACK, 'Cisco FlexStack'), - (cls.TYPE_FLEXSTACK_PLUS, 'Cisco FlexStack Plus'), - (cls.TYPE_JUNIPER_VCP, 'Juniper VCP'), - (cls.TYPE_SUMMITSTACK, 'Extreme SummitStack'), - (cls.TYPE_SUMMITSTACK128, 'Extreme SummitStack-128'), - (cls.TYPE_SUMMITSTACK256, 'Extreme SummitStack-256'), - (cls.TYPE_SUMMITSTACK512, 'Extreme SummitStack-512'), - ) - ), + (TYPE_STACKWISE, 'Cisco StackWise'), + (TYPE_STACKWISE_PLUS, 'Cisco StackWise Plus'), + (TYPE_FLEXSTACK, 'Cisco FlexStack'), + (TYPE_FLEXSTACK_PLUS, 'Cisco FlexStack Plus'), + (TYPE_JUNIPER_VCP, 'Juniper VCP'), + (TYPE_SUMMITSTACK, 'Extreme SummitStack'), + (TYPE_SUMMITSTACK128, 'Extreme SummitStack-128'), + (TYPE_SUMMITSTACK256, 'Extreme SummitStack-256'), + (TYPE_SUMMITSTACK512, 'Extreme SummitStack-512'), + ) + ), + ( + 'Other', ( - 'Other', - ( - (cls.TYPE_OTHER, 'Other'), - ) - ), - ) + (TYPE_OTHER, 'Other'), + ) + ), + ) @classmethod def slug_to_integer(cls, slug): @@ -844,33 +842,31 @@ class PortTypes: TYPE_LSH = 'lsh' TYPE_LSH_APC = 'lsh-apc' - @classmethod - def as_choices(cls): - return ( + TYPE_CHOICES = ( + ( + 'Copper', ( - 'Copper', - ( - (cls.TYPE_8P8C, '8P8C'), - (cls.TYPE_110_PUNCH, '110 Punch'), - (cls.TYPE_BNC, 'BNC'), - ), + (TYPE_8P8C, '8P8C'), + (TYPE_110_PUNCH, '110 Punch'), + (TYPE_BNC, 'BNC'), ), + ), + ( + 'Fiber Optic', ( - 'Fiber Optic', - ( - (cls.TYPE_FC, 'FC'), - (cls.TYPE_LC, 'LC'), - (cls.TYPE_LC_APC, 'LC/APC'), - (cls.TYPE_LSH, 'LSH'), - (cls.TYPE_LSH_APC, 'LSH/APC'), - (cls.TYPE_MPO, 'MPO'), - (cls.TYPE_MTRJ, 'MTRJ'), - (cls.TYPE_SC, 'SC'), - (cls.TYPE_SC_APC, 'SC/APC'), - (cls.TYPE_ST, 'ST'), - ) + (TYPE_FC, 'FC'), + (TYPE_LC, 'LC'), + (TYPE_LC_APC, 'LC/APC'), + (TYPE_LSH, 'LSH'), + (TYPE_LSH_APC, 'LSH/APC'), + (TYPE_MPO, 'MPO'), + (TYPE_MTRJ, 'MTRJ'), + (TYPE_SC, 'SC'), + (TYPE_SC_APC, 'SC/APC'), + (TYPE_ST, 'ST'), ) ) + ) @classmethod def slug_to_integer(cls, slug): diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 279b00dd9..6513cfee2 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -1290,7 +1290,7 @@ class PowerOutletTemplateImportForm(ComponentTemplateImportForm): class InterfaceTemplateImportForm(ComponentTemplateImportForm): type = forms.ChoiceField( - choices=InterfaceTypes.as_choices() + choices=InterfaceTypes.TYPE_CHOICES ) class Meta: @@ -1307,7 +1307,7 @@ class InterfaceTemplateImportForm(ComponentTemplateImportForm): class FrontPortTemplateImportForm(ComponentTemplateImportForm): type = forms.ChoiceField( - choices=PortTypes.as_choices() + choices=PortTypes.TYPE_CHOICES ) rear_port = forms.ModelChoiceField( queryset=RearPortTemplate.objects.all(), @@ -1329,7 +1329,7 @@ class FrontPortTemplateImportForm(ComponentTemplateImportForm): class RearPortTemplateImportForm(ComponentTemplateImportForm): type = forms.ChoiceField( - choices=PortTypes.as_choices() + choices=PortTypes.TYPE_CHOICES ) class Meta: From 657a7aef42cfd49bc0a9796ea39682b0cac9f9b5 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 23 Oct 2019 11:55:45 -0400 Subject: [PATCH 29/42] Changelog for #3455 --- docs/release-notes/version-2.7.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 3504eccdc..86404e586 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -51,3 +51,8 @@ Full connection details are required in both sections, even if they are the same ## Enhancements * [#2902](https://github.com/digitalocean/netbox/issues/2902) - Replace supervisord with systemd +* [#3455](https://github.com/digitalocean/netbox/issues/3455) - Add tenant assignment to cluster + +## API Changes + +* virtualization.Cluster: Added field `tenant` From 1cfb8aea23d736546cf61b3c42296fa86170750f Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 28 Oct 2019 15:02:21 -0400 Subject: [PATCH 30/42] Initial work on #3538: script execution API --- netbox/extras/api/serializers.py | 30 +++++++++++++++++++ netbox/extras/api/urls.py | 3 ++ netbox/extras/api/views.py | 49 ++++++++++++++++++++++++++++++++ netbox/extras/scripts.py | 22 +++++++++++--- 4 files changed, 100 insertions(+), 4 deletions(-) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 5789dc7c2..4999f6fab 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -200,6 +200,36 @@ class ReportDetailSerializer(ReportSerializer): result = ReportResultSerializer() +# +# Scripts +# + +class ScriptSerializer(serializers.Serializer): + id = serializers.SerializerMethodField(read_only=True) + name = serializers.SerializerMethodField(read_only=True) + description = serializers.SerializerMethodField(read_only=True) + vars = serializers.SerializerMethodField(read_only=True) + + def get_id(self, instance): + return '{}.{}'.format(instance.__module__, instance.__name__) + + def get_name(self, instance): + return getattr(instance.Meta, 'name', instance.__name__) + + def get_description(self, instance): + return getattr(instance.Meta, 'description', '') + + def get_vars(self, instance): + return { + k: v.__class__.__name__ for k, v in instance._get_vars().items() + } + + +class ScriptInputSerializer(serializers.Serializer): + data = serializers.JSONField() + commit = serializers.BooleanField() + + # # Change logging # diff --git a/netbox/extras/api/urls.py b/netbox/extras/api/urls.py index ddfe2107c..3215439c2 100644 --- a/netbox/extras/api/urls.py +++ b/netbox/extras/api/urls.py @@ -38,6 +38,9 @@ router.register(r'config-contexts', views.ConfigContextViewSet) # Reports router.register(r'reports', views.ReportViewSet, basename='report') +# Scripts +router.register(r'scripts', views.ScriptViewSet, basename='script') + # Change logging router.register(r'object-changes', views.ObjectChangeViewSet) diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index a6b678522..8c8b5de87 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -3,6 +3,7 @@ from collections import OrderedDict from django.contrib.contenttypes.models import ContentType from django.db.models import Count from django.http import Http404 +from rest_framework import status from rest_framework.decorators import action from rest_framework.exceptions import PermissionDenied from rest_framework.response import Response @@ -13,6 +14,7 @@ from extras.models import ( ConfigContext, CustomFieldChoice, ExportTemplate, Graph, ImageAttachment, ObjectChange, ReportResult, Tag, ) from extras.reports import get_report, get_reports +from extras.scripts import get_script, get_scripts from utilities.api import FieldChoicesViewSet, IsAuthenticatedOrLoginNotRequired, ModelViewSet from . import serializers @@ -222,6 +224,53 @@ class ReportViewSet(ViewSet): return Response(serializer.data) +# +# Scripts +# + +class ScriptViewSet(ViewSet): + permission_classes = [IsAuthenticatedOrLoginNotRequired] + _ignore_model_permissions = True + exclude_from_schema = True + lookup_value_regex = '[^/]+' # Allow dots + + def _get_script(self, pk): + module_name, script_name = pk.split('.') + script = get_script(module_name, script_name) + if script is None: + raise Http404 + return script + + def list(self, request): + + flat_list = [] + for script_list in get_scripts().values(): + flat_list.extend(script_list.values()) + + serializer = serializers.ScriptSerializer(flat_list, many=True, context={'request': request}) + + return Response(serializer.data) + + def retrieve(self, request, pk): + script = self._get_script(pk) + serializer = serializers.ScriptSerializer(script, context={'request': request}) + + return Response(serializer.data) + + def post(self, request, pk): + """ + Run a Script identified as ".