From a598f0e632255d1e2777c365c88e2b9707177503 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 31 May 2017 17:40:11 -0400 Subject: [PATCH] Initial work on #655: CSV import headers --- netbox/templates/tenancy/tenant_import.html | 37 --------- netbox/templates/utilities/obj_import.html | 17 +++++ netbox/tenancy/forms.py | 21 +++--- netbox/tenancy/views.py | 6 +- netbox/utilities/forms.py | 56 +++++++++++++- netbox/utilities/views.py | 83 ++++++++++++++++++++- 6 files changed, 168 insertions(+), 52 deletions(-) diff --git a/netbox/templates/tenancy/tenant_import.html b/netbox/templates/tenancy/tenant_import.html index 2e5196a88..4d0e97f48 100644 --- a/netbox/templates/tenancy/tenant_import.html +++ b/netbox/templates/tenancy/tenant_import.html @@ -1,40 +1,3 @@ {% extends 'utilities/obj_import.html' %} {% block title %}Tenant Import{% endblock %} - -{% block instructions %} -

CSV Format

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldDescriptionExample
NameTenant nameWIDG01
SlugURL-friendly namewidg01
GroupTenant group (optional)Customers
DescriptionLong-form name or other text (optional)Widgets Inc.
-

Example

- 
WIDG01,widg01,Customers,Widgets Inc.
-{% endblock %} diff --git a/netbox/templates/utilities/obj_import.html b/netbox/templates/utilities/obj_import.html index 2d69be048..325cfa304 100644 --- a/netbox/templates/utilities/obj_import.html +++ b/netbox/templates/utilities/obj_import.html @@ -28,6 +28,23 @@
{% block instructions %}{% endblock %} + {% if fields %} +

CSV Format

+ + + + + + + {% for name, field in fields.items %} + + + + + + {% endfor %} +
FieldRequiredDescription
{{ name }}{% if field.required %}{% endif %}{{ field.help_text }}
+ {% endif %}
{% endblock %} diff --git a/netbox/tenancy/forms.py b/netbox/tenancy/forms.py index df4f05b95..32e8263c2 100644 --- a/netbox/tenancy/forms.py +++ b/netbox/tenancy/forms.py @@ -5,8 +5,7 @@ from django.db.models import Count from extras.forms import CustomFieldForm, CustomFieldBulkEditForm, CustomFieldFilterForm from utilities.forms import ( - APISelect, BootstrapMixin, BulkImportForm, ChainedFieldsMixin, ChainedModelChoiceField, CommentField, CSVDataField, - FilterChoiceField, SlugField, + APISelect, BootstrapMixin, ChainedFieldsMixin, ChainedModelChoiceField, CommentField, FilterChoiceField, SlugField, ) from .models import Tenant, TenantGroup @@ -36,17 +35,19 @@ class TenantForm(BootstrapMixin, CustomFieldForm): fields = ['name', 'slug', 'group', 'description', 'comments'] -class TenantFromCSVForm(forms.ModelForm): - group = forms.ModelChoiceField(TenantGroup.objects.all(), required=False, to_field_name='name', - error_messages={'invalid_choice': 'Group not found.'}) +class TenantCSVForm(forms.ModelForm): + group = forms.ModelChoiceField( + queryset=TenantGroup.objects.all(), + required=False, + to_field_name='name', + error_messages={ + 'invalid_choice': 'Group not found.' + } + ) class Meta: model = Tenant - fields = ['name', 'slug', 'group', 'description'] - - -class TenantImportForm(BootstrapMixin, BulkImportForm): - csv = CSVDataField(csv_form=TenantFromCSVForm) + fields = ['name', 'slug', 'group', 'description', 'comments'] class TenantBulkEditForm(BootstrapMixin, CustomFieldBulkEditForm): diff --git a/netbox/tenancy/views.py b/netbox/tenancy/views.py index 3b5ad9b37..27afed269 100644 --- a/netbox/tenancy/views.py +++ b/netbox/tenancy/views.py @@ -10,7 +10,7 @@ from circuits.models import Circuit from dcim.models import Site, Rack, Device from ipam.models import IPAddress, Prefix, VLAN, VRF from utilities.views import ( - BulkDeleteView, BulkEditView, BulkImportView, ObjectDeleteView, ObjectEditView, ObjectListView, + BulkDeleteView, BulkEditView, BulkImportView, BulkImportView2, ObjectDeleteView, ObjectEditView, ObjectListView, ) from .models import Tenant, TenantGroup from . import filters, forms, tables @@ -95,9 +95,9 @@ class TenantDeleteView(PermissionRequiredMixin, ObjectDeleteView): default_return_url = 'tenancy:tenant_list' -class TenantBulkImportView(PermissionRequiredMixin, BulkImportView): +class TenantBulkImportView(PermissionRequiredMixin, BulkImportView2): permission_required = 'tenancy.add_tenant' - form = forms.TenantImportForm + model_form = forms.TenantCSVForm table = tables.TenantTable template_name = 'tenancy/tenant_import.html' default_return_url = 'tenancy:tenant_list' diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index 37101a56e..fcdaf2c53 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -256,6 +256,60 @@ class CSVDataField(forms.CharField): return records +class CSVDataField2(forms.CharField): + """ + A CharField (rendered as a Textarea) which accepts CSV-formatted data. It returns a list of dictionaries mapping + column headers to values. Each dictionary represents an individual record. + """ + widget = forms.Textarea + + def __init__(self, fields, required_fields=[], *args, **kwargs): + + self.fields = fields + self.required_fields = required_fields + + super(CSVDataField2, self).__init__(*args, **kwargs) + + self.strip = False + if not self.label: + self.label = 'CSV Data' + if not self.initial: + self.initial = ','.join(required_fields) + '\n' + if not self.help_text: + self.help_text = 'Enter one line per record. Use commas to separate values.' + + def to_python(self, value): + + # Python 2's csv module has problems with Unicode + if not isinstance(value, str): + value = value.encode('utf-8') + + records = [] + reader = csv.reader(value.splitlines()) + + # Consume and valdiate the first line of CSV data as column headers + headers = reader.next() + for f in self.required_fields: + if f not in headers: + raise forms.ValidationError('Required column header "{}" not found.'.format(f)) + for f in headers: + if f not in self.fields: + raise forms.ValidationError('Unexpected column header "{}" found.'.format(f)) + + # Parse CSV data + for i, row in enumerate(reader, start=1): + if row: + if len(row) != len(headers): + raise forms.ValidationError( + "Row {}: Expected {} columns but found {}".format(i, len(headers), len(row)) + ) + row = [col.strip() for col in row] + record = dict(zip(headers, row)) + records.append(record) + + return records + + class ExpandableNameField(forms.CharField): """ A field which allows for numeric range expansion @@ -488,7 +542,7 @@ class BulkEditForm(forms.Form): class BulkImportForm(forms.Form): def clean(self): - records = self.cleaned_data.get('csv') + fields, records = self.cleaned_data.get('csv').split('\n', 1) if not records: return diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index 9c7a4b55e..7791e69b7 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -6,9 +6,10 @@ from django_tables2 import RequestConfig from django.conf import settings from django.contrib import messages from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError from django.db import transaction, IntegrityError from django.db.models import ProtectedError -from django.forms import CharField, ModelMultipleChoiceField, MultipleHiddenInput, TypedChoiceField +from django.forms import CharField, Form, ModelMultipleChoiceField, MultipleHiddenInput, TypedChoiceField from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.template import TemplateSyntaxError @@ -19,6 +20,7 @@ from django.utils.safestring import mark_safe from django.views.generic import View from extras.models import CustomField, CustomFieldValue, ExportTemplate, UserAction +from utilities.forms import BootstrapMixin, CSVDataField2 from .error_handlers import handle_protectederror from .forms import ConfirmationForm from .paginator import EnhancedPaginator @@ -422,6 +424,85 @@ class BulkImportView(View): obj.save() +class BulkImportView2(View): + """ + Import objects in bulk (CSV format). + + model_form: The form used to create each imported object + table: The django-tables2 Table used to render the list of imported objects + template_name: The name of the template + default_return_url: The name of the URL to use for the cancel button + """ + model_form = None + table = None + template_name = None + default_return_url = None + + def _import_form(self, *args, **kwargs): + + fields = self.model_form().fields.keys() + required_fields = [name for name, field in self.model_form().fields.items() if field.required] + + class ImportForm(BootstrapMixin, Form): + csv = CSVDataField2(fields=fields, required_fields=required_fields) + + return ImportForm(*args, **kwargs) + + def get(self, request): + + return render(request, self.template_name, { + 'form': self._import_form(), + 'fields': self.model_form().fields, + 'return_url': self.default_return_url, + }) + + def post(self, request): + + new_objs = [] + form = self._import_form(request.POST) + + if form.is_valid(): + + try: + + # Iterate through CSV data and bind each row to a new model form instance. + with transaction.atomic(): + for row, data in enumerate(form.cleaned_data['csv'], start=1): + obj_form = self.model_form(data) + if obj_form.is_valid(): + obj = obj_form.save() + new_objs.append(obj) + else: + for field, err in obj_form.errors.items(): + form.add_error('csv', "Row {} {}: {}".format(row, field, err[0])) + raise ValidationError("") + + # Compile a table containing the imported objects + obj_table = self.table(new_objs) + + if new_objs: + msg = 'Imported {} {}'.format(len(new_objs), new_objs[0]._meta.verbose_name_plural) + messages.success(request, msg) + UserAction.objects.log_import(request.user, ContentType.objects.get_for_model(new_objs[0]), msg) + + return render(request, "import_success.html", { + 'table': obj_table, + 'return_url': self.default_return_url, + }) + + except ValidationError: + pass + + return render(request, self.template_name, { + 'form': form, + 'fields': self.model_form().fields, + 'return_url': self.default_return_url, + }) + + def save_obj(self, obj): + obj.save() + + class BulkEditView(View): """ Edit objects in bulk.