From e1fe3ca14a3cdfdf0826c5db03e4e5d60c62615e Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Tue, 25 May 2021 11:09:33 -0400 Subject: [PATCH 01/11] CSV Upload as second field in existing form --- netbox/netbox/views/generic.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index 8a2bae4fa..8228f6eff 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -7,7 +7,7 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError from django.db import transaction, IntegrityError from django.db.models import ManyToManyField, ProtectedError -from django.forms import Form, ModelMultipleChoiceField, MultipleHiddenInput, Textarea +from django.forms import Form, ModelMultipleChoiceField, MultipleHiddenInput, Textarea, FileField, CharField from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.utils.html import escape @@ -665,7 +665,10 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): from_form=self.model_form, widget=Textarea(attrs=self.widget_attrs) ) - + Upload_CSV = FileField( + required=False + ) + # this is where the import form is created -- add csv upload here or create new form? return ImportForm(*args, **kwargs) def _save_obj(self, obj_form, request): @@ -690,6 +693,7 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): logger = logging.getLogger('netbox.views.BulkImportView') new_objs = [] form = self._import_form(request.POST) + # this is where the csv data is handled if form.is_valid(): logger.debug("Form validation was successful") @@ -697,6 +701,9 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): try: # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): + # some prints trying to figure out how the form works + print(type(form["Upload_CSV"])) + print(form["Upload_CSV"].data) headers, records = form.cleaned_data['csv'] for row, data in enumerate(records, start=1): obj_form = self.model_form(data, headers=headers) From 0c9376039c84a24a2d38bb316a601bb155d326bc Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Thu, 27 May 2021 10:31:51 -0400 Subject: [PATCH 02/11] working csv upload first draft --- netbox/netbox/views/generic.py | 31 ++++++++++++++++--- .../templates/generic/object_bulk_import.html | 2 +- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index 8228f6eff..c01f69783 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -701,10 +701,33 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): try: # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): - # some prints trying to figure out how the form works - print(type(form["Upload_CSV"])) - print(form["Upload_CSV"].data) - headers, records = form.cleaned_data['csv'] + + if len(request.FILES) != 0: + csv_file = request.FILES["Upload_CSV"] + csv_file.seek(0) + csv_str = csv_file.read().decode('utf-8') + + csv_list = csv_str.split('\n') + header_row = csv_list[0] + csv_list.pop(0) + header_list = header_row.split(',') + headers = {} + for elt in header_list: + headers[elt] = None + records = [] + for row in csv_list: + if row != "": + row_str = (',').join(row) + row_list = row.split(',') + row_dict = {} + for i, elt in enumerate(row_list): + if elt == '': + row_dict[header_list[i]] = None + else: + row_dict[header_list[i]] = elt + records.append(row_dict) + else: + headers, records = form.cleaned_data['csv'] for row, data in enumerate(records, start=1): obj_form = self.model_form(data, headers=headers) restrict_form_fields(obj_form, request.user) diff --git a/netbox/templates/generic/object_bulk_import.html b/netbox/templates/generic/object_bulk_import.html index 170cf3665..e9972e919 100644 --- a/netbox/templates/generic/object_bulk_import.html +++ b/netbox/templates/generic/object_bulk_import.html @@ -20,7 +20,7 @@
-
+ {% csrf_token %} {% render_form form %}
From 2bc68707b5e82c0b868cb65a951ed5d02b691962 Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Wed, 2 Jun 2021 10:12:26 -0400 Subject: [PATCH 03/11] csv parse using python csv library --- netbox/netbox/views/generic.py | 37 ++++++++++++++-------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index c01f69783..c665c3630 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -1,5 +1,6 @@ import logging import re +import csv from copy import deepcopy from django.contrib import messages @@ -7,7 +8,7 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError from django.db import transaction, IntegrityError from django.db.models import ManyToManyField, ProtectedError -from django.forms import Form, ModelMultipleChoiceField, MultipleHiddenInput, Textarea, FileField, CharField +from django.forms import Form, ModelMultipleChoiceField, MultipleHiddenInput, Textarea, FileField from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.utils.html import escape @@ -668,7 +669,6 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): Upload_CSV = FileField( required=False ) - # this is where the import form is created -- add csv upload here or create new form? return ImportForm(*args, **kwargs) def _save_obj(self, obj_form, request): @@ -693,7 +693,6 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): logger = logging.getLogger('netbox.views.BulkImportView') new_objs = [] form = self._import_form(request.POST) - # this is where the csv data is handled if form.is_valid(): logger.debug("Form validation was successful") @@ -701,33 +700,27 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): try: # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): - if len(request.FILES) != 0: csv_file = request.FILES["Upload_CSV"] csv_file.seek(0) csv_str = csv_file.read().decode('utf-8') - - csv_list = csv_str.split('\n') - header_row = csv_list[0] - csv_list.pop(0) - header_list = header_row.split(',') + reader = csv.reader(csv_str.splitlines()) + headers_list = next(reader) headers = {} - for elt in header_list: - headers[elt] = None + for header in headers_list: + headers[header] = None records = [] - for row in csv_list: - if row != "": - row_str = (',').join(row) - row_list = row.split(',') - row_dict = {} - for i, elt in enumerate(row_list): - if elt == '': - row_dict[header_list[i]] = None - else: - row_dict[header_list[i]] = elt - records.append(row_dict) + for row in reader: + row_dict = {} + for i, elt in enumerate(row): + if elt == '': + row_dict[headers_list[i]] = None + else: + row_dict[headers_list[i]] = elt + records.append(row_dict) else: headers, records = form.cleaned_data['csv'] + print("headers:", headers, "records:", records) for row, data in enumerate(records, start=1): obj_form = self.model_form(data, headers=headers) restrict_form_fields(obj_form, request.user) From 6ff5a1db42df4c3232c2c7f7bb6475d22747af1a Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Thu, 3 Jun 2021 15:08:47 -0400 Subject: [PATCH 04/11] cleaned up csv parsing --- netbox/netbox/views/generic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index c665c3630..0d1734c08 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -666,7 +666,7 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): from_form=self.model_form, widget=Textarea(attrs=self.widget_attrs) ) - Upload_CSV = FileField( + upload_csv = FileField( required=False ) return ImportForm(*args, **kwargs) @@ -692,7 +692,7 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): def post(self, request): logger = logging.getLogger('netbox.views.BulkImportView') new_objs = [] - form = self._import_form(request.POST) + form = self._import_form(request.POST, request.FILES) if form.is_valid(): logger.debug("Form validation was successful") @@ -700,8 +700,8 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): try: # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): - if len(request.FILES) != 0: - csv_file = request.FILES["Upload_CSV"] + if request.FILES: + csv_file = request.FILES["upload_csv"] csv_file.seek(0) csv_str = csv_file.read().decode('utf-8') reader = csv.reader(csv_str.splitlines()) From c2b2b059e6ac3dc5660d380f6d92528d73fd71dc Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Fri, 4 Jun 2021 10:27:19 -0400 Subject: [PATCH 05/11] CSV import implemented using CSVFileField --- netbox/netbox/views/generic.py | 27 +++--------- netbox/utilities/forms/fields.py | 72 ++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index 0d1734c08..24d459219 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -1,6 +1,5 @@ import logging import re -import csv from copy import deepcopy from django.contrib import messages @@ -8,7 +7,7 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError from django.db import transaction, IntegrityError from django.db.models import ManyToManyField, ProtectedError -from django.forms import Form, ModelMultipleChoiceField, MultipleHiddenInput, Textarea, FileField +from django.forms import Form, ModelMultipleChoiceField, MultipleHiddenInput, Textarea from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.utils.html import escape @@ -21,7 +20,7 @@ from extras.models import CustomField, ExportTemplate from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortTransaction from utilities.forms import ( - BootstrapMixin, BulkRenameForm, ConfirmationForm, CSVDataField, ImportForm, TableConfigForm, restrict_form_fields, + BootstrapMixin, BulkRenameForm, ConfirmationForm, CSVDataField, ImportForm, TableConfigForm, restrict_form_fields, CSVFileField ) from utilities.permissions import get_permission_for_model from utilities.tables import paginate_table @@ -666,7 +665,8 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): from_form=self.model_form, widget=Textarea(attrs=self.widget_attrs) ) - upload_csv = FileField( + upload_csv = CSVFileField( + from_form=self.model_form, required=False ) return ImportForm(*args, **kwargs) @@ -701,26 +701,9 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): if request.FILES: - csv_file = request.FILES["upload_csv"] - csv_file.seek(0) - csv_str = csv_file.read().decode('utf-8') - reader = csv.reader(csv_str.splitlines()) - headers_list = next(reader) - headers = {} - for header in headers_list: - headers[header] = None - records = [] - for row in reader: - row_dict = {} - for i, elt in enumerate(row): - if elt == '': - row_dict[headers_list[i]] = None - else: - row_dict[headers_list[i]] = elt - records.append(row_dict) + headers, records = form.cleaned_data['upload_csv'] else: headers, records = form.cleaned_data['csv'] - print("headers:", headers, "records:", records) for row, data in enumerate(records, start=1): obj_form = self.model_form(data, headers=headers) restrict_form_fields(obj_form, request.user) diff --git a/netbox/utilities/forms/fields.py b/netbox/utilities/forms/fields.py index 93b9e6c44..1d8e299e0 100644 --- a/netbox/utilities/forms/fields.py +++ b/netbox/utilities/forms/fields.py @@ -26,6 +26,7 @@ __all__ = ( 'CSVChoiceField', 'CSVContentTypeField', 'CSVDataField', + 'CSVFileField', 'CSVModelChoiceField', 'CSVTypedChoiceField', 'DynamicModelChoiceField', @@ -221,6 +222,77 @@ class CSVDataField(forms.CharField): return value +class CSVFileField(forms.FileField): + """ + A CharField (rendered as a Textarea) which accepts CSV-formatted data. It returns data as a two-tuple: The first + item is a dictionary of column headers, mapping field names to the attribute by which they match a related object + (where applicable). The second item is a list of dictionaries, each representing a discrete row of CSV data. + + :param from_form: The form from which the field derives its validation rules. + """ + + def __init__(self, from_form, *args, **kwargs): + + form = from_form() + self.model = form.Meta.model + self.fields = form.fields + self.required_fields = [ + name for name, field in form.fields.items() if field.required + ] + + super().__init__(*args, **kwargs) + + def to_python(self, file): + + records = [] + file.seek(0) + csv_str = file.read().decode('utf-8') + reader = csv.reader(csv_str.splitlines()) + + # Consume the first line of CSV data as column headers. Create a dictionary mapping each header to an optional + # "to" field specifying how the related object is being referenced. For example, importing a Device might use a + # `site.slug` header, to indicate the related site is being referenced by its slug. + + headers = {} + for header in next(reader): + if '.' in header: + field, to_field = header.split('.', 1) + headers[field] = to_field + else: + headers[header] = None + + # Parse CSV rows into a list of dictionaries mapped from the column headers. + for i, row in enumerate(reader, start=1): + if len(row) != len(headers): + raise forms.ValidationError( + f"Row {i}: Expected {len(headers)} columns but found {len(row)}" + ) + row = [col.strip() for col in row] + record = dict(zip(headers.keys(), row)) + records.append(record) + + return headers, records + + def validate(self, value): + headers, records = value + + # Validate provided column headers + for field, to_field in headers.items(): + if field not in self.fields: + raise forms.ValidationError(f'Unexpected column header "{field}" found.') + if to_field and not hasattr(self.fields[field], 'to_field_name'): + raise forms.ValidationError(f'Column "{field}" is not a related object; cannot use dots') + if to_field and not hasattr(self.fields[field].queryset.model, to_field): + raise forms.ValidationError(f'Invalid related object attribute for column "{field}": {to_field}') + + # Validate required fields + for f in self.required_fields: + if f not in headers: + raise forms.ValidationError(f'Required column header "{f}" not found.') + + return value + + class CSVChoiceField(forms.ChoiceField): """ Invert the provided set of choices to take the human-friendly label as input, and return the database value. From ecd84d7c430cf2c75f99440f5535bea7d367bdb1 Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Mon, 7 Jun 2021 11:52:31 -0400 Subject: [PATCH 06/11] edited docstring for CSVFileField --- netbox/utilities/forms/fields.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/netbox/utilities/forms/fields.py b/netbox/utilities/forms/fields.py index 1d8e299e0..a8383de6e 100644 --- a/netbox/utilities/forms/fields.py +++ b/netbox/utilities/forms/fields.py @@ -224,9 +224,10 @@ class CSVDataField(forms.CharField): class CSVFileField(forms.FileField): """ - A CharField (rendered as a Textarea) which accepts CSV-formatted data. It returns data as a two-tuple: The first - item is a dictionary of column headers, mapping field names to the attribute by which they match a related object - (where applicable). The second item is a list of dictionaries, each representing a discrete row of CSV data. + A FileField (rendered as a file input button) which accepts a file containing CSV-formatted data. It returns + data as a two-tuple: The first item is a dictionary of column headers, mapping field names to the attribute + by which they match a related object (where applicable). The second item is a list of dictionaries, each + representing a discrete row of CSV data. :param from_form: The form from which the field derives its validation rules. """ From 3549fc07f6b0a681ba2f74d645e78f2441c4a3e4 Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Mon, 7 Jun 2021 14:29:38 -0400 Subject: [PATCH 07/11] removed unnecessary use of seek() --- netbox/utilities/forms/fields.py | 1 - 1 file changed, 1 deletion(-) diff --git a/netbox/utilities/forms/fields.py b/netbox/utilities/forms/fields.py index a8383de6e..0f5b0b6dd 100644 --- a/netbox/utilities/forms/fields.py +++ b/netbox/utilities/forms/fields.py @@ -246,7 +246,6 @@ class CSVFileField(forms.FileField): def to_python(self, file): records = [] - file.seek(0) csv_str = file.read().decode('utf-8') reader = csv.reader(csv_str.splitlines()) From 55b7cf21ccc834598bdb9d1a4bfd6ac0d3bc80fd Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Thu, 10 Jun 2021 14:41:33 -0400 Subject: [PATCH 08/11] changed name of csv_file variable and started work on ValidationError --- netbox/netbox/views/generic.py | 12 +++++++--- netbox/utilities/forms/fields.py | 38 ++++++++++++++++++-------------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index 24d459219..7ff980750 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -665,10 +665,16 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): from_form=self.model_form, widget=Textarea(attrs=self.widget_attrs) ) - upload_csv = CSVFileField( + csv_file = CSVFileField( + label="CSV file", from_form=self.model_form, required=False ) + def used_both_methods(self): + if self.cleaned_data['csv_file'][1] and self.cleaned_data['csv'][1]: + raise ValidationError('') + return False + return ImportForm(*args, **kwargs) def _save_obj(self, obj_form, request): @@ -694,14 +700,14 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): new_objs = [] form = self._import_form(request.POST, request.FILES) - if form.is_valid(): + if form.is_valid() and not form.used_both_methods(): logger.debug("Form validation was successful") try: # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): if request.FILES: - headers, records = form.cleaned_data['upload_csv'] + headers, records = form.cleaned_data['csv_file'] else: headers, records = form.cleaned_data['csv'] for row, data in enumerate(records, start=1): diff --git a/netbox/utilities/forms/fields.py b/netbox/utilities/forms/fields.py index 0f5b0b6dd..2954f9c73 100644 --- a/netbox/utilities/forms/fields.py +++ b/netbox/utilities/forms/fields.py @@ -246,35 +246,39 @@ class CSVFileField(forms.FileField): def to_python(self, file): records = [] - csv_str = file.read().decode('utf-8') - reader = csv.reader(csv_str.splitlines()) + if file: + csv_str = file.read().decode('utf-8') + reader = csv.reader(csv_str.splitlines()) # Consume the first line of CSV data as column headers. Create a dictionary mapping each header to an optional # "to" field specifying how the related object is being referenced. For example, importing a Device might use a # `site.slug` header, to indicate the related site is being referenced by its slug. headers = {} - for header in next(reader): - if '.' in header: - field, to_field = header.split('.', 1) - headers[field] = to_field - else: - headers[header] = None + if file: + for header in next(reader): + if '.' in header: + field, to_field = header.split('.', 1) + headers[field] = to_field + else: + headers[header] = None - # Parse CSV rows into a list of dictionaries mapped from the column headers. - for i, row in enumerate(reader, start=1): - if len(row) != len(headers): - raise forms.ValidationError( - f"Row {i}: Expected {len(headers)} columns but found {len(row)}" - ) - row = [col.strip() for col in row] - record = dict(zip(headers.keys(), row)) - records.append(record) + # Parse CSV rows into a list of dictionaries mapped from the column headers. + for i, row in enumerate(reader, start=1): + if len(row) != len(headers): + raise forms.ValidationError( + f"Row {i}: Expected {len(headers)} columns but found {len(row)}" + ) + row = [col.strip() for col in row] + record = dict(zip(headers.keys(), row)) + records.append(record) return headers, records def validate(self, value): headers, records = value + if not headers and not records: + return value # Validate provided column headers for field, to_field in headers.items(): From 934543b595f4740397b76d836f79601ba688378f Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Fri, 11 Jun 2021 13:42:26 -0400 Subject: [PATCH 09/11] Caught and handled ValidationError --- netbox/netbox/views/generic.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index 7ff980750..00841f4b9 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -670,9 +670,10 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): from_form=self.model_form, required=False ) - def used_both_methods(self): + + def used_both_csv_fields(self): if self.cleaned_data['csv_file'][1] and self.cleaned_data['csv'][1]: - raise ValidationError('') + return True return False return ImportForm(*args, **kwargs) @@ -700,10 +701,13 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): new_objs = [] form = self._import_form(request.POST, request.FILES) - if form.is_valid() and not form.used_both_methods(): + if form.is_valid(): logger.debug("Form validation was successful") try: + if form.used_both_csv_fields(): + form.add_error('csv_file', "Choose one of two import methods") + raise ValidationError("") # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): if request.FILES: From 0a661596b3ff3d9e18024d8c621099bfa3eac45d Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Mon, 14 Jun 2021 14:07:37 -0400 Subject: [PATCH 10/11] moved duplicated code in CSV Fields into functions in forms/utils.py --- netbox/utilities/forms/fields.py | 80 +++----------------------------- netbox/utilities/forms/utils.py | 53 +++++++++++++++++++++ 2 files changed, 59 insertions(+), 74 deletions(-) diff --git a/netbox/utilities/forms/fields.py b/netbox/utilities/forms/fields.py index 2954f9c73..2b9b57ea3 100644 --- a/netbox/utilities/forms/fields.py +++ b/netbox/utilities/forms/fields.py @@ -17,7 +17,7 @@ from utilities.utils import content_type_name from utilities.validators import EnhancedURLValidator from . import widgets from .constants import * -from .utils import expand_alphanumeric_pattern, expand_ipaddress_pattern +from .utils import expand_alphanumeric_pattern, expand_ipaddress_pattern, parse_csv, validate_csv __all__ = ( 'CommentField', @@ -175,49 +175,13 @@ class CSVDataField(forms.CharField): 'in double quotes.' def to_python(self, value): - - records = [] reader = csv.reader(StringIO(value.strip())) - # Consume the first line of CSV data as column headers. Create a dictionary mapping each header to an optional - # "to" field specifying how the related object is being referenced. For example, importing a Device might use a - # `site.slug` header, to indicate the related site is being referenced by its slug. - headers = {} - for header in next(reader): - if '.' in header: - field, to_field = header.split('.', 1) - headers[field] = to_field - else: - headers[header] = None - - # Parse CSV rows into a list of dictionaries mapped from the column headers. - for i, row in enumerate(reader, start=1): - if len(row) != len(headers): - raise forms.ValidationError( - f"Row {i}: Expected {len(headers)} columns but found {len(row)}" - ) - row = [col.strip() for col in row] - record = dict(zip(headers.keys(), row)) - records.append(record) - - return headers, records + return parse_csv(reader) def validate(self, value): headers, records = value - - # Validate provided column headers - for field, to_field in headers.items(): - if field not in self.fields: - raise forms.ValidationError(f'Unexpected column header "{field}" found.') - if to_field and not hasattr(self.fields[field], 'to_field_name'): - raise forms.ValidationError(f'Column "{field}" is not a related object; cannot use dots') - if to_field and not hasattr(self.fields[field].queryset.model, to_field): - raise forms.ValidationError(f'Invalid related object attribute for column "{field}": {to_field}') - - # Validate required fields - for f in self.required_fields: - if f not in headers: - raise forms.ValidationError(f'Required column header "{f}" not found.') + validate_csv(headers, self.fields, self.required_fields) return value @@ -244,34 +208,14 @@ class CSVFileField(forms.FileField): super().__init__(*args, **kwargs) def to_python(self, file): - - records = [] if file: csv_str = file.read().decode('utf-8') reader = csv.reader(csv_str.splitlines()) - # Consume the first line of CSV data as column headers. Create a dictionary mapping each header to an optional - # "to" field specifying how the related object is being referenced. For example, importing a Device might use a - # `site.slug` header, to indicate the related site is being referenced by its slug. - headers = {} + records = [] if file: - for header in next(reader): - if '.' in header: - field, to_field = header.split('.', 1) - headers[field] = to_field - else: - headers[header] = None - - # Parse CSV rows into a list of dictionaries mapped from the column headers. - for i, row in enumerate(reader, start=1): - if len(row) != len(headers): - raise forms.ValidationError( - f"Row {i}: Expected {len(headers)} columns but found {len(row)}" - ) - row = [col.strip() for col in row] - record = dict(zip(headers.keys(), row)) - records.append(record) + headers, records = parse_csv(reader) return headers, records @@ -280,19 +224,7 @@ class CSVFileField(forms.FileField): if not headers and not records: return value - # Validate provided column headers - for field, to_field in headers.items(): - if field not in self.fields: - raise forms.ValidationError(f'Unexpected column header "{field}" found.') - if to_field and not hasattr(self.fields[field], 'to_field_name'): - raise forms.ValidationError(f'Column "{field}" is not a related object; cannot use dots') - if to_field and not hasattr(self.fields[field].queryset.model, to_field): - raise forms.ValidationError(f'Invalid related object attribute for column "{field}": {to_field}') - - # Validate required fields - for f in self.required_fields: - if f not in headers: - raise forms.ValidationError(f'Required column header "{f}" not found.') + validate_csv(headers, self.fields, self.required_fields) return value diff --git a/netbox/utilities/forms/utils.py b/netbox/utilities/forms/utils.py index dc001be1a..50a6a416e 100644 --- a/netbox/utilities/forms/utils.py +++ b/netbox/utilities/forms/utils.py @@ -14,6 +14,8 @@ __all__ = ( 'parse_alphanumeric_range', 'parse_numeric_range', 'restrict_form_fields', + 'parse_csv', + 'validate_csv', ) @@ -134,3 +136,54 @@ def restrict_form_fields(form, user, action='view'): for field in form.fields.values(): if hasattr(field, 'queryset') and issubclass(field.queryset.__class__, RestrictedQuerySet): field.queryset = field.queryset.restrict(user, action) + + +def parse_csv(reader): + """ + Parse a csv_reader object into a headers dictionary and a list of records dictionaries. Raise an error + if the records are formatted incorrectly. Return headers and records as a tuple. + """ + records = [] + headers = {} + + # Consume the first line of CSV data as column headers. Create a dictionary mapping each header to an optional + # "to" field specifying how the related object is being referenced. For example, importing a Device might use a + # `site.slug` header, to indicate the related site is being referenced by its slug. + + for header in next(reader): + if '.' in header: + field, to_field = header.split('.', 1) + headers[field] = to_field + else: + headers[header] = None + + # Parse CSV rows into a list of dictionaries mapped from the column headers. + for i, row in enumerate(reader, start=1): + if len(row) != len(headers): + raise forms.ValidationError( + f"Row {i}: Expected {len(headers)} columns but found {len(row)}" + ) + row = [col.strip() for col in row] + record = dict(zip(headers.keys(), row)) + records.append(record) + return headers, records + + +def validate_csv(headers, fields, required_fields): + """ + Validate that parsed csv data conforms to the object's available fields. Raise validation errors + if parsed csv data contains invalid headers or does not contain required headers. + """ + # Validate provided column headers + for field, to_field in headers.items(): + if field not in fields: + raise forms.ValidationError(f'Unexpected column header "{field}" found.') + if to_field and not hasattr(fields[field], 'to_field_name'): + raise forms.ValidationError(f'Column "{field}" is not a related object; cannot use dots') + if to_field and not hasattr(fields[field].queryset.model, to_field): + raise forms.ValidationError(f'Invalid related object attribute for column "{field}": {to_field}') + + # Validate required fields + for f in required_fields: + if f not in headers: + raise forms.ValidationError(f'Required column header "{f}" not found.') From 1e7b76005c729489eb5f7d77302298fd3ce0b8cd Mon Sep 17 00:00:00 2001 From: Alyssa Bigley Date: Mon, 14 Jun 2021 15:23:42 -0400 Subject: [PATCH 11/11] cleaned up validation error method --- netbox/netbox/views/generic.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index 00841f4b9..7616a8c06 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -672,9 +672,7 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): ) def used_both_csv_fields(self): - if self.cleaned_data['csv_file'][1] and self.cleaned_data['csv'][1]: - return True - return False + return self.cleaned_data['csv_file'][1] and self.cleaned_data['csv'][1] return ImportForm(*args, **kwargs)