- {% render_field form.csv %}
+
+ {# Data Import Form #}
+
+ {% block content %}
+
+
+ {% endblock content %}
+
+
+ {# File Upload Form #}
+
{% endblock content-wrapper %}
diff --git a/netbox/utilities/forms/choices.py b/netbox/utilities/forms/choices.py
new file mode 100644
index 000000000..bf0ea5f94
--- /dev/null
+++ b/netbox/utilities/forms/choices.py
@@ -0,0 +1,17 @@
+from utilities.choices import ChoiceSet
+
+
+#
+# Import Choices
+#
+
+class ImportFormatChoices(ChoiceSet):
+ CSV = 'csv'
+ JSON = 'json'
+ YAML = 'yaml'
+
+ CHOICES = [
+ (CSV, 'CSV'),
+ (JSON, 'JSON'),
+ (YAML, 'YAML'),
+ ]
diff --git a/netbox/utilities/forms/forms.py b/netbox/utilities/forms/forms.py
index 8ad6f103b..0569853b8 100644
--- a/netbox/utilities/forms/forms.py
+++ b/netbox/utilities/forms/forms.py
@@ -1,12 +1,15 @@
+import csv
import json
import re
+from io import StringIO
import yaml
from django import forms
+from utilities.forms.utils import parse_csv
+from .choices import ImportFormatChoices
from .widgets import APISelect, APISelectMultiple, ClearableFileInput, StaticSelect
-
__all__ = (
'BootstrapMixin',
'BulkEditForm',
@@ -120,64 +123,94 @@ class CSVModelForm(forms.ModelForm):
"""
ModelForm used for the import of objects in CSV format.
"""
-
- def __init__(self, *args, headers=None, **kwargs):
+ def __init__(self, *args, headers=None, fields=None, **kwargs):
+ headers = headers or {}
+ fields = fields or []
super().__init__(*args, **kwargs)
# Modify the model form to accommodate any customized to_field_name properties
- if headers:
- for field, to_field in headers.items():
- if to_field is not None:
- self.fields[field].to_field_name = to_field
+ for field, to_field in headers.items():
+ if to_field is not None:
+ self.fields[field].to_field_name = to_field
+
+ # Omit any fields not specified (e.g. because the form is being used to
+ # updated rather than create objects)
+ if fields:
+ for field in list(self.fields.keys()):
+ if field not in fields:
+ del self.fields[field]
class ImportForm(BootstrapMixin, forms.Form):
- """
- Generic form for creating an object from JSON/YAML data
- """
data = forms.CharField(
+ required=False,
widget=forms.Textarea(attrs={'class': 'font-monospace'}),
- help_text="Enter object data in JSON or YAML format. Note: Only a single object/document is supported."
+ help_text="Enter object data in CSV, JSON or YAML format."
)
+ data_file = forms.FileField(
+ label="Data file",
+ required=False
+ )
+ # TODO: Enable auto-detection of format
format = forms.ChoiceField(
- choices=(
- ('json', 'JSON'),
- ('yaml', 'YAML')
- ),
- initial='yaml'
+ choices=ImportFormatChoices,
+ initial=ImportFormatChoices.CSV,
+ widget=StaticSelect()
)
+ data_field = 'data'
+
def clean(self):
super().clean()
-
- data = self.cleaned_data['data']
format = self.cleaned_data['format']
- # Process JSON/YAML data
- if format == 'json':
- try:
- self.cleaned_data['data'] = json.loads(data)
- # Check for multiple JSON objects
- if type(self.cleaned_data['data']) is not dict:
- raise forms.ValidationError({
- 'data': "Import is limited to one object at a time."
- })
- except json.decoder.JSONDecodeError as err:
- raise forms.ValidationError({
- 'data': "Invalid JSON data: {}".format(err)
- })
+ # Determine whether we're reading from form data or an uploaded file
+ if self.cleaned_data['data'] and self.cleaned_data['data_file']:
+ raise forms.ValidationError("Form data must be empty when uploading a file.")
+ if 'data_file' in self.files:
+ self.data_field = 'data_file'
+ file = self.files.get('data_file')
+ data = file.read().decode('utf-8')
else:
- # Check for multiple YAML documents
- if '\n---' in data:
- raise forms.ValidationError({
- 'data': "Import is limited to one object at a time."
- })
- try:
- self.cleaned_data['data'] = yaml.load(data, Loader=yaml.SafeLoader)
- except yaml.error.YAMLError as err:
- raise forms.ValidationError({
- 'data': "Invalid YAML data: {}".format(err)
- })
+ data = self.cleaned_data['data']
+
+ # Process data according to the selected format
+ if format == ImportFormatChoices.CSV:
+ self.cleaned_data['data'] = self._clean_csv(data)
+ elif format == ImportFormatChoices.JSON:
+ self.cleaned_data['data'] = self._clean_json(data)
+ elif format == ImportFormatChoices.YAML:
+ self.cleaned_data['data'] = self._clean_yaml(data)
+
+ def _clean_csv(self, data):
+ stream = StringIO(data.strip())
+ reader = csv.reader(stream)
+ headers, records = parse_csv(reader)
+
+ # Set CSV headers for reference by the model form
+ self._csv_headers = headers
+
+ return records
+
+ def _clean_json(self, data):
+ try:
+ data = json.loads(data)
+ # Accommodate for users entering single objects
+ if type(data) is not list:
+ data = [data]
+ return data
+ except json.decoder.JSONDecodeError as err:
+ raise forms.ValidationError({
+ self.data_field: f"Invalid JSON data: {err}"
+ })
+
+ def _clean_yaml(self, data):
+ try:
+ return yaml.load_all(data, Loader=yaml.SafeLoader)
+ except yaml.error.YAMLError as err:
+ raise forms.ValidationError({
+ self.data_field: f"Invalid YAML data: {err}"
+ })
class FilterForm(BootstrapMixin, forms.Form):
diff --git a/netbox/utilities/testing/views.py b/netbox/utilities/testing/views.py
index f51893f74..5e1e207cc 100644
--- a/netbox/utilities/testing/views.py
+++ b/netbox/utilities/testing/views.py
@@ -9,6 +9,7 @@ from django.urls import reverse
from extras.choices import ObjectChangeActionChoices
from extras.models import ObjectChange
from users.models import ObjectPermission
+from utilities.forms.choices import ImportFormatChoices
from .base import ModelTestCase
from .utils import disable_warnings, post_data
@@ -555,7 +556,8 @@ class ViewTestCases:
def test_bulk_import_objects_without_permission(self):
data = {
- 'csv': self._get_csv_data(),
+ 'data': self._get_csv_data(),
+ 'format': 'csv',
}
# Test GET without permission
@@ -571,7 +573,8 @@ class ViewTestCases:
def test_bulk_import_objects_with_permission(self):
initial_count = self._get_queryset().count()
data = {
- 'csv': self._get_csv_data(),
+ 'data': self._get_csv_data(),
+ 'format': 'csv',
}
# Assign model-level permission
@@ -598,7 +601,8 @@ class ViewTestCases:
initial_count = self._get_queryset().count()
array, csv_data = self._get_update_csv_data()
data = {
- 'csv': csv_data,
+ 'format': ImportFormatChoices.CSV,
+ 'data': csv_data,
}
# Assign model-level permission
@@ -630,7 +634,8 @@ class ViewTestCases:
def test_bulk_import_objects_with_constrained_permission(self):
initial_count = self._get_queryset().count()
data = {
- 'csv': self._get_csv_data(),
+ 'data': self._get_csv_data(),
+ 'format': 'csv',
}
# Assign constrained permission