mirror of
https://github.com/netbox-community/netbox.git
synced 2025-08-24 08:25:17 -06:00
4347 initial code for json import
This commit is contained in:
parent
0ce7f84ec1
commit
ead26d09d3
@ -936,7 +936,7 @@ class DeviceTypeDeleteView(generic.ObjectDeleteView):
|
|||||||
queryset = DeviceType.objects.all()
|
queryset = DeviceType.objects.all()
|
||||||
|
|
||||||
|
|
||||||
class DeviceTypeImportView(generic.ObjectImportView):
|
class DeviceTypeImportView(generic.BulkImportView):
|
||||||
additional_permissions = [
|
additional_permissions = [
|
||||||
'dcim.add_devicetype',
|
'dcim.add_devicetype',
|
||||||
'dcim.add_consoleporttemplate',
|
'dcim.add_consoleporttemplate',
|
||||||
@ -952,6 +952,7 @@ class DeviceTypeImportView(generic.ObjectImportView):
|
|||||||
]
|
]
|
||||||
queryset = DeviceType.objects.all()
|
queryset = DeviceType.objects.all()
|
||||||
model_form = forms.DeviceTypeImportForm
|
model_form = forms.DeviceTypeImportForm
|
||||||
|
table = tables.DeviceTypeTable
|
||||||
related_object_forms = {
|
related_object_forms = {
|
||||||
'console-ports': forms.ConsolePortTemplateImportForm,
|
'console-ports': forms.ConsolePortTemplateImportForm,
|
||||||
'console-server-ports': forms.ConsoleServerPortTemplateImportForm,
|
'console-server-ports': forms.ConsoleServerPortTemplateImportForm,
|
||||||
@ -1069,7 +1070,7 @@ class ModuleTypeDeleteView(generic.ObjectDeleteView):
|
|||||||
queryset = ModuleType.objects.all()
|
queryset = ModuleType.objects.all()
|
||||||
|
|
||||||
|
|
||||||
class ModuleTypeImportView(generic.ObjectImportView):
|
class ModuleTypeImportView(generic.BulkImportView):
|
||||||
additional_permissions = [
|
additional_permissions = [
|
||||||
'dcim.add_moduletype',
|
'dcim.add_moduletype',
|
||||||
'dcim.add_consoleporttemplate',
|
'dcim.add_consoleporttemplate',
|
||||||
@ -1082,6 +1083,7 @@ class ModuleTypeImportView(generic.ObjectImportView):
|
|||||||
]
|
]
|
||||||
queryset = ModuleType.objects.all()
|
queryset = ModuleType.objects.all()
|
||||||
model_form = forms.ModuleTypeImportForm
|
model_form = forms.ModuleTypeImportForm
|
||||||
|
table = tables.ModuleTypeTable
|
||||||
related_object_forms = {
|
related_object_forms = {
|
||||||
'console-ports': forms.ConsolePortTemplateImportForm,
|
'console-ports': forms.ConsolePortTemplateImportForm,
|
||||||
'console-server-ports': forms.ConsoleServerPortTemplateImportForm,
|
'console-server-ports': forms.ConsoleServerPortTemplateImportForm,
|
||||||
|
@ -887,7 +887,7 @@ class CustomFieldImportTest(TestCase):
|
|||||||
)
|
)
|
||||||
csv_data = '\n'.join(','.join(row) for row in data)
|
csv_data = '\n'.join(','.join(row) for row in data)
|
||||||
|
|
||||||
response = self.client.post(reverse('dcim:site_import'), {'csv': csv_data})
|
response = self.client.post(reverse('dcim:site_import'), {'data': csv_data, 'format': 'csv'})
|
||||||
self.assertEqual(response.status_code, 200)
|
self.assertEqual(response.status_code, 200)
|
||||||
self.assertEqual(Site.objects.count(), 3)
|
self.assertEqual(Site.objects.count(), 3)
|
||||||
|
|
||||||
|
@ -26,6 +26,7 @@ from utilities.forms import (
|
|||||||
from utilities.htmx import is_htmx
|
from utilities.htmx import is_htmx
|
||||||
from utilities.permissions import get_permission_for_model
|
from utilities.permissions import get_permission_for_model
|
||||||
from utilities.views import GetReturnURLMixin
|
from utilities.views import GetReturnURLMixin
|
||||||
|
from utilities.forms.choices import ImportFormatChoices
|
||||||
from .base import BaseMultiObjectView
|
from .base import BaseMultiObjectView
|
||||||
from .mixins import ActionsMixin, TableMixin
|
from .mixins import ActionsMixin, TableMixin
|
||||||
from .utils import get_prerequisite_model
|
from .utils import get_prerequisite_model
|
||||||
@ -288,134 +289,6 @@ class BulkCreateView(GetReturnURLMixin, BaseMultiObjectView):
|
|||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
class OldBulkImportView(GetReturnURLMixin, BaseMultiObjectView):
|
|
||||||
"""
|
|
||||||
Import objects in bulk (CSV format).
|
|
||||||
|
|
||||||
Attributes:
|
|
||||||
model_form: The form used to create each imported object
|
|
||||||
"""
|
|
||||||
template_name = 'generic/bulk_import.html'
|
|
||||||
model_form = None
|
|
||||||
|
|
||||||
def _import_form(self, *args, **kwargs):
|
|
||||||
|
|
||||||
class ImportForm(BootstrapMixin, Form):
|
|
||||||
csv = CSVDataField(
|
|
||||||
from_form=self.model_form
|
|
||||||
)
|
|
||||||
csv_file = CSVFileField(
|
|
||||||
label="CSV file",
|
|
||||||
from_form=self.model_form,
|
|
||||||
required=False
|
|
||||||
)
|
|
||||||
|
|
||||||
def clean(self):
|
|
||||||
csv_rows = self.cleaned_data['csv'][1] if 'csv' in self.cleaned_data else None
|
|
||||||
csv_file = self.files.get('csv_file')
|
|
||||||
|
|
||||||
# Check that the user has not submitted both text data and a file
|
|
||||||
if csv_rows and csv_file:
|
|
||||||
raise ValidationError(
|
|
||||||
"Cannot process CSV text and file attachment simultaneously. Please choose only one import "
|
|
||||||
"method."
|
|
||||||
)
|
|
||||||
|
|
||||||
return ImportForm(*args, **kwargs)
|
|
||||||
|
|
||||||
def _create_objects(self, form, request):
|
|
||||||
new_objs = []
|
|
||||||
if request.FILES:
|
|
||||||
headers, records = form.cleaned_data['csv_file']
|
|
||||||
else:
|
|
||||||
headers, records = form.cleaned_data['csv']
|
|
||||||
|
|
||||||
for row, data in enumerate(records, start=1):
|
|
||||||
obj_form = self.model_form(data, headers=headers)
|
|
||||||
restrict_form_fields(obj_form, request.user)
|
|
||||||
|
|
||||||
if obj_form.is_valid():
|
|
||||||
obj = self._save_obj(obj_form, request)
|
|
||||||
new_objs.append(obj)
|
|
||||||
else:
|
|
||||||
for field, err in obj_form.errors.items():
|
|
||||||
form.add_error('csv', f'Row {row} {field}: {err[0]}')
|
|
||||||
raise ValidationError("")
|
|
||||||
|
|
||||||
return new_objs
|
|
||||||
|
|
||||||
def _save_obj(self, obj_form, request):
|
|
||||||
"""
|
|
||||||
Provide a hook to modify the object immediately before saving it (e.g. to encrypt secret data).
|
|
||||||
"""
|
|
||||||
return obj_form.save()
|
|
||||||
|
|
||||||
def get_required_permission(self):
|
|
||||||
return get_permission_for_model(self.queryset.model, 'add')
|
|
||||||
|
|
||||||
#
|
|
||||||
# Request handlers
|
|
||||||
#
|
|
||||||
|
|
||||||
def get(self, request):
|
|
||||||
|
|
||||||
return render(request, self.template_name, {
|
|
||||||
'model': self.model_form._meta.model,
|
|
||||||
'form': self._import_form(),
|
|
||||||
'fields': self.model_form().fields,
|
|
||||||
'return_url': self.get_return_url(request),
|
|
||||||
**self.get_extra_context(request),
|
|
||||||
})
|
|
||||||
|
|
||||||
def post(self, request):
|
|
||||||
logger = logging.getLogger('netbox.views.BulkImportView')
|
|
||||||
form = self._import_form(request.POST, request.FILES)
|
|
||||||
|
|
||||||
if form.is_valid():
|
|
||||||
logger.debug("Form validation was successful")
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Iterate through CSV data and bind each row to a new model form instance.
|
|
||||||
with transaction.atomic():
|
|
||||||
new_objs = self._create_objects(form, request)
|
|
||||||
|
|
||||||
# Enforce object-level permissions
|
|
||||||
if self.queryset.filter(pk__in=[obj.pk for obj in new_objs]).count() != len(new_objs):
|
|
||||||
raise PermissionsViolation
|
|
||||||
|
|
||||||
# Compile a table containing the imported objects
|
|
||||||
obj_table = self.table(new_objs)
|
|
||||||
|
|
||||||
if new_objs:
|
|
||||||
msg = 'Imported {} {}'.format(len(new_objs), new_objs[0]._meta.verbose_name_plural)
|
|
||||||
logger.info(msg)
|
|
||||||
messages.success(request, msg)
|
|
||||||
|
|
||||||
return render(request, "import_success.html", {
|
|
||||||
'table': obj_table,
|
|
||||||
'return_url': self.get_return_url(request),
|
|
||||||
})
|
|
||||||
|
|
||||||
except ValidationError:
|
|
||||||
clear_webhooks.send(sender=self)
|
|
||||||
|
|
||||||
except (AbortRequest, PermissionsViolation) as e:
|
|
||||||
logger.debug(e.message)
|
|
||||||
form.add_error(None, e.message)
|
|
||||||
clear_webhooks.send(sender=self)
|
|
||||||
|
|
||||||
else:
|
|
||||||
logger.debug("Form validation failed")
|
|
||||||
|
|
||||||
return render(request, self.template_name, {
|
|
||||||
'model': self.model_form._meta.model,
|
|
||||||
'form': form,
|
|
||||||
'fields': self.model_form().fields,
|
|
||||||
'return_url': self.get_return_url(request),
|
|
||||||
**self.get_extra_context(request),
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
|
class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
|
||||||
"""
|
"""
|
||||||
Import objects in bulk (CSV format).
|
Import objects in bulk (CSV format).
|
||||||
@ -425,19 +298,14 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
|
|||||||
"""
|
"""
|
||||||
template_name = 'generic/bulk_import.html'
|
template_name = 'generic/bulk_import.html'
|
||||||
model_form = None
|
model_form = None
|
||||||
|
related_object_forms = dict()
|
||||||
|
|
||||||
'''
|
def prep_related_object_data(self, parent, data):
|
||||||
supported_formats = [
|
"""
|
||||||
{
|
Hook to modify the data for related objects before it's passed to the related object form (for example, to
|
||||||
'name': 'CSV',
|
assign a parent object).
|
||||||
'help_text': 'Enter the list of column headers followed by one line per record to be imported, using ' \
|
"""
|
||||||
'commas to separate values. Multi-line data and values containing commas may be wrapped ' \
|
return data
|
||||||
'in double quotes.'
|
|
||||||
},
|
|
||||||
{'name': 'JSON', },
|
|
||||||
{'name': 'YAML', },
|
|
||||||
]
|
|
||||||
'''
|
|
||||||
|
|
||||||
def _create_object(self, request, model_form):
|
def _create_object(self, request, model_form):
|
||||||
|
|
||||||
@ -478,16 +346,16 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
|
|||||||
|
|
||||||
return obj
|
return obj
|
||||||
|
|
||||||
def _create_objects(self, form, request):
|
def _create_objects(self, form, format, data, request):
|
||||||
new_objs = []
|
new_objs = []
|
||||||
for row_num, record in enumerate(data['data'], start=1):
|
for row_num, record in enumerate(data['data'], start=1):
|
||||||
if format == 'csv':
|
if format == ImportFormatChoices.CSV:
|
||||||
model_form = self.model_form(record, headers=headers)
|
model_form = self.model_form(record, headers=data['headers'])
|
||||||
else:
|
else:
|
||||||
model_form = self.model_form(record)
|
model_form = self.model_form(record)
|
||||||
restrict_form_fields(model_form, request.user)
|
restrict_form_fields(model_form, request.user)
|
||||||
|
|
||||||
if format == 'json' or format == 'yaml':
|
if format == ImportFormatChoices.JSON or format == ImportFormatChoices.YAML:
|
||||||
# Assign default values for any fields which were not specified.
|
# Assign default values for any fields which were not specified.
|
||||||
# We have to do this manually because passing 'initial=' to the form
|
# We have to do this manually because passing 'initial=' to the form
|
||||||
# on initialization merely sets default values for the widgets.
|
# on initialization merely sets default values for the widgets.
|
||||||
@ -505,8 +373,8 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
|
|||||||
# Replicate model form errors for display
|
# Replicate model form errors for display
|
||||||
for field, errors in model_form.errors.items():
|
for field, errors in model_form.errors.items():
|
||||||
for err in errors:
|
for err in errors:
|
||||||
if format == 'csv':
|
if format == ImportFormatChoices.CSV:
|
||||||
form.add_error('csv', f'Row {row} {field}: {err[0]}')
|
form.add_error(None, f'Row {row_num} {field}: {err}')
|
||||||
else:
|
else:
|
||||||
if field == '__all__':
|
if field == '__all__':
|
||||||
form.add_error(None, err)
|
form.add_error(None, err)
|
||||||
@ -530,10 +398,13 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
|
|||||||
# Request handlers
|
# Request handlers
|
||||||
#
|
#
|
||||||
|
|
||||||
def get_context(self, request, data_form, file_form):
|
def get_context(self, request, data_form, file_form, form=None):
|
||||||
|
# small hack - need to return 'form' set to either the file or data form
|
||||||
|
# as the bulk_import base view relies on it for error reporting.
|
||||||
return {
|
return {
|
||||||
'model': self.model_form._meta.model,
|
'model': self.model_form._meta.model,
|
||||||
'data_form': data_form,
|
'data_form': data_form,
|
||||||
|
'form': form,
|
||||||
'file_form': file_form,
|
'file_form': file_form,
|
||||||
'fields': self.model_form().fields,
|
'fields': self.model_form().fields,
|
||||||
'return_url': self.get_return_url(request),
|
'return_url': self.get_return_url(request),
|
||||||
@ -541,34 +412,36 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
|
|||||||
}
|
}
|
||||||
|
|
||||||
def get(self, request):
|
def get(self, request):
|
||||||
data_form = ImportForm()
|
data_form = ImportForm(related=self.related_object_forms)
|
||||||
file_form = FileUploadImportForm()
|
file_form = FileUploadImportForm(related=self.related_object_forms)
|
||||||
|
|
||||||
return render(request, self.template_name, self.get_context(request, data_form, file_form))
|
return render(request, self.template_name, self.get_context(request, data_form, file_form))
|
||||||
|
|
||||||
def post(self, request):
|
def post(self, request):
|
||||||
logger = logging.getLogger('netbox.views.BulkImportView')
|
logger = logging.getLogger('netbox.views.BulkImportView')
|
||||||
data_form = ImportForm(request.POST)
|
data_form = ImportForm(request.POST, related=self.related_object_forms)
|
||||||
file_form = FileUploadImportForm(request.POST, request.FILES)
|
file_form = FileUploadImportForm(request.POST, request.FILES, related=self.related_object_forms)
|
||||||
|
|
||||||
data = None
|
data = None
|
||||||
if 'data_submit' in request.POST:
|
form = None
|
||||||
if data_form.is_valid():
|
if 'file_submit' in request.POST:
|
||||||
logger.debug("Data Import form validation was successful")
|
form = file_form
|
||||||
data = data_form.cleaned_data
|
|
||||||
elif 'file_submit' in request.POST:
|
|
||||||
if file_form.is_valid():
|
if file_form.is_valid():
|
||||||
logger.debug("File Import form validation was successful")
|
logger.debug("File Import form validation was successful")
|
||||||
data = file_form.cleaned_data
|
data = file_form.cleaned_data
|
||||||
|
else: # data_submit
|
||||||
|
form = data_form
|
||||||
|
if data_form.is_valid():
|
||||||
|
logger.debug("Data Import form validation was successful")
|
||||||
|
data = data_form.cleaned_data
|
||||||
|
|
||||||
if data:
|
if data:
|
||||||
format = data['format']
|
format = data['format']
|
||||||
headers = data['headers'] if format == 'csv' else None
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Iterate through data and bind each row to a new model form instance.
|
# Iterate through data and bind each row to a new model form instance.
|
||||||
with transaction.atomic():
|
with transaction.atomic():
|
||||||
new_objs = self._create_objects(form, request)
|
new_objs = self._create_objects(form, format, data, request)
|
||||||
|
|
||||||
# Enforce object-level permissions
|
# Enforce object-level permissions
|
||||||
if self.queryset.filter(pk__in=[obj.pk for obj in new_objs]).count() != len(new_objs):
|
if self.queryset.filter(pk__in=[obj.pk for obj in new_objs]).count() != len(new_objs):
|
||||||
@ -598,7 +471,7 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
|
|||||||
else:
|
else:
|
||||||
logger.debug("Form validation failed")
|
logger.debug("Form validation failed")
|
||||||
|
|
||||||
return render(request, self.template_name, self.get_context(request, data_form, file_form))
|
return render(request, self.template_name, self.get_context(request, data_form, file_form, form))
|
||||||
|
|
||||||
|
|
||||||
class BulkEditView(GetReturnURLMixin, BaseMultiObjectView):
|
class BulkEditView(GetReturnURLMixin, BaseMultiObjectView):
|
||||||
|
@ -28,7 +28,6 @@ __all__ = (
|
|||||||
'ObjectChildrenView',
|
'ObjectChildrenView',
|
||||||
'ObjectDeleteView',
|
'ObjectDeleteView',
|
||||||
'ObjectEditView',
|
'ObjectEditView',
|
||||||
'ObjectImportView',
|
|
||||||
'ObjectView',
|
'ObjectView',
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -145,149 +144,6 @@ class ObjectChildrenView(ObjectView, ActionsMixin, TableMixin):
|
|||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
class ObjectImportView(GetReturnURLMixin, BaseObjectView):
|
|
||||||
"""
|
|
||||||
Import a single object (YAML or JSON format).
|
|
||||||
|
|
||||||
Attributes:
|
|
||||||
model_form: The ModelForm used to create individual objects
|
|
||||||
related_object_forms: A dictionary mapping of forms to be used for the creation of related (child) objects
|
|
||||||
"""
|
|
||||||
template_name = 'generic/object_import.html'
|
|
||||||
model_form = None
|
|
||||||
related_object_forms = dict()
|
|
||||||
|
|
||||||
def get_required_permission(self):
|
|
||||||
return get_permission_for_model(self.queryset.model, 'add')
|
|
||||||
|
|
||||||
def prep_related_object_data(self, parent, data):
|
|
||||||
"""
|
|
||||||
Hook to modify the data for related objects before it's passed to the related object form (for example, to
|
|
||||||
assign a parent object).
|
|
||||||
"""
|
|
||||||
return data
|
|
||||||
|
|
||||||
def _create_object(self, model_form):
|
|
||||||
|
|
||||||
# Save the primary object
|
|
||||||
obj = model_form.save()
|
|
||||||
|
|
||||||
# Enforce object-level permissions
|
|
||||||
if not self.queryset.filter(pk=obj.pk).first():
|
|
||||||
raise PermissionsViolation()
|
|
||||||
|
|
||||||
# Iterate through the related object forms (if any), validating and saving each instance.
|
|
||||||
for field_name, related_object_form in self.related_object_forms.items():
|
|
||||||
|
|
||||||
related_obj_pks = []
|
|
||||||
for i, rel_obj_data in enumerate(model_form.data.get(field_name, list())):
|
|
||||||
rel_obj_data = self.prep_related_object_data(obj, rel_obj_data)
|
|
||||||
f = related_object_form(rel_obj_data)
|
|
||||||
|
|
||||||
for subfield_name, field in f.fields.items():
|
|
||||||
if subfield_name not in rel_obj_data and hasattr(field, 'initial'):
|
|
||||||
f.data[subfield_name] = field.initial
|
|
||||||
|
|
||||||
if f.is_valid():
|
|
||||||
related_obj = f.save()
|
|
||||||
related_obj_pks.append(related_obj.pk)
|
|
||||||
else:
|
|
||||||
# Replicate errors on the related object form to the primary form for display
|
|
||||||
for subfield_name, errors in f.errors.items():
|
|
||||||
for err in errors:
|
|
||||||
err_msg = "{}[{}] {}: {}".format(field_name, i, subfield_name, err)
|
|
||||||
model_form.add_error(None, err_msg)
|
|
||||||
raise AbortTransaction()
|
|
||||||
|
|
||||||
# Enforce object-level permissions on related objects
|
|
||||||
model = related_object_form.Meta.model
|
|
||||||
if model.objects.filter(pk__in=related_obj_pks).count() != len(related_obj_pks):
|
|
||||||
raise ObjectDoesNotExist
|
|
||||||
|
|
||||||
return obj
|
|
||||||
|
|
||||||
#
|
|
||||||
# Request handlers
|
|
||||||
#
|
|
||||||
|
|
||||||
def get(self, request):
|
|
||||||
form = ImportForm()
|
|
||||||
model = self.queryset.model
|
|
||||||
|
|
||||||
return render(request, self.template_name, {
|
|
||||||
'model': model,
|
|
||||||
'form': form,
|
|
||||||
'obj_type': self.queryset.model._meta.verbose_name,
|
|
||||||
'return_url': self.get_return_url(request),
|
|
||||||
})
|
|
||||||
|
|
||||||
def post(self, request):
|
|
||||||
logger = logging.getLogger('netbox.views.ObjectImportView')
|
|
||||||
form = ImportForm(request.POST)
|
|
||||||
|
|
||||||
if form.is_valid():
|
|
||||||
logger.debug("Import form validation was successful")
|
|
||||||
|
|
||||||
# Initialize model form
|
|
||||||
data = form.cleaned_data['data']
|
|
||||||
model_form = self.model_form(data)
|
|
||||||
restrict_form_fields(model_form, request.user)
|
|
||||||
|
|
||||||
# Assign default values for any fields which were not specified. We have to do this manually because passing
|
|
||||||
# 'initial=' to the form on initialization merely sets default values for the widgets. Since widgets are not
|
|
||||||
# used for YAML/JSON import, we first bind the imported data normally, then update the form's data with the
|
|
||||||
# applicable field defaults as needed prior to form validation.
|
|
||||||
for field_name, field in model_form.fields.items():
|
|
||||||
if field_name not in data and hasattr(field, 'initial'):
|
|
||||||
model_form.data[field_name] = field.initial
|
|
||||||
|
|
||||||
if model_form.is_valid():
|
|
||||||
|
|
||||||
try:
|
|
||||||
with transaction.atomic():
|
|
||||||
obj = self._create_object(model_form)
|
|
||||||
|
|
||||||
except AbortTransaction:
|
|
||||||
clear_webhooks.send(sender=self)
|
|
||||||
|
|
||||||
except (AbortRequest, PermissionsViolation) as e:
|
|
||||||
logger.debug(e.message)
|
|
||||||
form.add_error(None, e.message)
|
|
||||||
clear_webhooks.send(sender=self)
|
|
||||||
|
|
||||||
if not model_form.errors:
|
|
||||||
logger.info(f"Import object {obj} (PK: {obj.pk})")
|
|
||||||
msg = f'Imported object: <a href="{obj.get_absolute_url()}">{obj}</a>'
|
|
||||||
messages.success(request, mark_safe(msg))
|
|
||||||
|
|
||||||
if '_addanother' in request.POST:
|
|
||||||
return redirect(request.get_full_path())
|
|
||||||
|
|
||||||
self.get_return_url(request, obj)
|
|
||||||
return redirect(self.get_return_url(request, obj))
|
|
||||||
|
|
||||||
else:
|
|
||||||
logger.debug("Model form validation failed")
|
|
||||||
|
|
||||||
# Replicate model form errors for display
|
|
||||||
for field, errors in model_form.errors.items():
|
|
||||||
for err in errors:
|
|
||||||
if field == '__all__':
|
|
||||||
form.add_error(None, err)
|
|
||||||
else:
|
|
||||||
form.add_error(None, "{}: {}".format(field, err))
|
|
||||||
|
|
||||||
else:
|
|
||||||
logger.debug("Import form validation failed")
|
|
||||||
|
|
||||||
return render(request, self.template_name, {
|
|
||||||
'model': model,
|
|
||||||
'form': form,
|
|
||||||
'obj_type': self.queryset.model._meta.verbose_name,
|
|
||||||
'return_url': self.get_return_url(request),
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
class ObjectEditView(GetReturnURLMixin, BaseObjectView):
|
class ObjectEditView(GetReturnURLMixin, BaseObjectView):
|
||||||
"""
|
"""
|
||||||
Create or edit a single object.
|
Create or edit a single object.
|
||||||
|
@ -170,152 +170,3 @@ Context:
|
|||||||
|
|
||||||
</div>
|
</div>
|
||||||
{% endblock content-wrapper %}
|
{% endblock content-wrapper %}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
{% comment %}
|
|
||||||
{% block tabs %}
|
|
||||||
<ul class="nav nav-tabs px-3">
|
|
||||||
<li class="nav-item" role="presentation">
|
|
||||||
<a class ="nav-link active" href="#">Bulk Import</a>
|
|
||||||
</li>
|
|
||||||
</ul>
|
|
||||||
{% endblock tabs %}
|
|
||||||
|
|
||||||
{% block content-wrapper %}
|
|
||||||
<div class="tab-content">
|
|
||||||
{% block content %}
|
|
||||||
<div class="row">
|
|
||||||
<div class="col col-md-12 col-lg-10 offset-lg-1">
|
|
||||||
<ul class="nav nav-pills px-3" role="tablist">
|
|
||||||
<li class="nav-item" role="presentation">
|
|
||||||
<button class="nav-link active" role="tab" type="button" data-bs-target="#csv" data-bs-toggle="tab">CSV Data</button>
|
|
||||||
</li>
|
|
||||||
<li class="nav-item" role="presentation">
|
|
||||||
<button class="nav-link" role="tab" type="button" data-bs-target="#csv-file" data-bs-toggle="tab">CSV File Upload</button>
|
|
||||||
</li>
|
|
||||||
</ul>
|
|
||||||
<form action="" method="post" enctype="multipart/form-data" class="form">
|
|
||||||
{% csrf_token %}
|
|
||||||
<div class="tab-content border-0">
|
|
||||||
<div role="tabpanel" class="tab-pane active" id="csv">
|
|
||||||
{% render_field form.csv %}
|
|
||||||
</div>
|
|
||||||
<div role="tabpanel" class="tab-pane" id="csv-file">
|
|
||||||
{% render_field form.csv_file %}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div class="form-group">
|
|
||||||
<div class="col col-md-12 text-end">
|
|
||||||
<button type="submit" class="btn btn-primary">Submit</button>
|
|
||||||
{% if return_url %}
|
|
||||||
<a href="{{ return_url }}" class="btn btn-outline-danger">Cancel</a>
|
|
||||||
{% endif %}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</form>
|
|
||||||
{% if fields %}
|
|
||||||
<div class="row my-3">
|
|
||||||
<div class="col col-md-12">
|
|
||||||
<div class="card">
|
|
||||||
<h5 class="card-header">
|
|
||||||
CSV Field Options
|
|
||||||
</h5>
|
|
||||||
<div class="card-body">
|
|
||||||
<table class="table">
|
|
||||||
<tr>
|
|
||||||
<th>Field</th>
|
|
||||||
<th>Required</th>
|
|
||||||
<th>Accessor</th>
|
|
||||||
<th>Description</th>
|
|
||||||
</tr>
|
|
||||||
{% for name, field in fields.items %}
|
|
||||||
<tr>
|
|
||||||
<td>
|
|
||||||
<code>{{ name }}</code>
|
|
||||||
</td>
|
|
||||||
<td>
|
|
||||||
{% if field.required %}
|
|
||||||
{% checkmark True true="Required" %}
|
|
||||||
{% else %}
|
|
||||||
{{ ''|placeholder }}
|
|
||||||
{% endif %}
|
|
||||||
</td>
|
|
||||||
<td>
|
|
||||||
{% if field.to_field_name %}
|
|
||||||
<code>{{ field.to_field_name }}</code>
|
|
||||||
{% else %}
|
|
||||||
{{ ''|placeholder }}
|
|
||||||
{% endif %}
|
|
||||||
</td>
|
|
||||||
<td>
|
|
||||||
{% if field.STATIC_CHOICES %}
|
|
||||||
<button type="button" class="btn btn-link btn-sm float-end" data-bs-toggle="modal" data-bs-target="#{{ name }}_choices">
|
|
||||||
<i class="mdi mdi-help-circle"></i>
|
|
||||||
</button>
|
|
||||||
<div class="modal fade" id="{{ name }}_choices" tabindex="-1" role="dialog">
|
|
||||||
<div class="modal-dialog" role="document">
|
|
||||||
<div class="modal-content">
|
|
||||||
<div class="modal-header">
|
|
||||||
<h5 class="modal-title"><code>{{ name }}</code> Choices</h5>
|
|
||||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
|
||||||
</div>
|
|
||||||
<div class="modal-body">
|
|
||||||
<table class="table table-striped">
|
|
||||||
<tr>
|
|
||||||
<th>Import Value</th>
|
|
||||||
<th>Label</th>
|
|
||||||
</tr>
|
|
||||||
{% for value, label in field.choices %}
|
|
||||||
{% if value %}
|
|
||||||
<tr>
|
|
||||||
<td>
|
|
||||||
<samp>{{ value }}</samp>
|
|
||||||
</td>
|
|
||||||
<td>
|
|
||||||
{{ label }}
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
{% endif %}
|
|
||||||
{% endfor %}
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{% endif %}
|
|
||||||
{% if field.help_text %}
|
|
||||||
{{ field.help_text }}<br />
|
|
||||||
{% elif field.label %}
|
|
||||||
{{ field.label }}<br />
|
|
||||||
{% endif %}
|
|
||||||
{% if field|widget_type == 'dateinput' %}
|
|
||||||
<small class="text-muted">Format: YYYY-MM-DD</small>
|
|
||||||
{% elif field|widget_type == 'checkboxinput' %}
|
|
||||||
<small class="text-muted">Specify "true" or "false"</small>
|
|
||||||
{% endif %}
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
{% endfor %}
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<p class="small text-muted">
|
|
||||||
<i class="mdi mdi-check-bold text-success"></i> Required fields <strong>must</strong> be specified for all
|
|
||||||
objects.
|
|
||||||
</p>
|
|
||||||
<p class="small text-muted">
|
|
||||||
<i class="mdi mdi-information-outline"></i> Related objects may be referenced by any unique attribute.
|
|
||||||
For example, <code>vrf.rd</code> would identify a VRF by its route distinguisher.
|
|
||||||
</p>
|
|
||||||
{% endif %}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{% endblock content %}
|
|
||||||
</div>
|
|
||||||
{% endblock content-wrapper %}
|
|
||||||
{% endcomment %}
|
|
||||||
|
27
netbox/utilities/forms/choices.py
Normal file
27
netbox/utilities/forms/choices.py
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
from utilities.choices import ChoiceSet
|
||||||
|
|
||||||
|
|
||||||
|
#
|
||||||
|
# Import Choices
|
||||||
|
#
|
||||||
|
|
||||||
|
class ImportFormatChoices(ChoiceSet):
|
||||||
|
CSV = 'csv'
|
||||||
|
JSON = 'json'
|
||||||
|
YAML = 'yaml'
|
||||||
|
|
||||||
|
CHOICES = [
|
||||||
|
(CSV, 'CSV'),
|
||||||
|
(JSON, 'JSON'),
|
||||||
|
(YAML, 'YAML'),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class ImportFormatChoicesRelated(ChoiceSet):
|
||||||
|
JSON = 'json'
|
||||||
|
YAML = 'yaml'
|
||||||
|
|
||||||
|
CHOICES = [
|
||||||
|
(JSON, 'JSON'),
|
||||||
|
(YAML, 'YAML'),
|
||||||
|
]
|
@ -1,3 +1,4 @@
|
|||||||
|
import csv
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
from io import StringIO
|
from io import StringIO
|
||||||
@ -6,6 +7,7 @@ import yaml
|
|||||||
from django import forms
|
from django import forms
|
||||||
from utilities.forms.utils import parse_csv, validate_csv
|
from utilities.forms.utils import parse_csv, validate_csv
|
||||||
|
|
||||||
|
from .choices import ImportFormatChoices, ImportFormatChoicesRelated
|
||||||
from .widgets import APISelect, APISelectMultiple, ClearableFileInput, StaticSelect
|
from .widgets import APISelect, APISelectMultiple, ClearableFileInput, StaticSelect
|
||||||
|
|
||||||
__all__ = (
|
__all__ = (
|
||||||
@ -133,95 +135,46 @@ class CSVModelForm(forms.ModelForm):
|
|||||||
self.fields[field].to_field_name = to_field
|
self.fields[field].to_field_name = to_field
|
||||||
|
|
||||||
|
|
||||||
class OldImportForm(BootstrapMixin, forms.Form):
|
|
||||||
"""
|
|
||||||
Generic form for creating an object from JSON/YAML data
|
|
||||||
"""
|
|
||||||
data = forms.CharField(
|
|
||||||
widget=forms.Textarea(attrs={'class': 'font-monospace'}),
|
|
||||||
help_text="Enter object data in JSON or YAML format. Note: Only a single object/document is supported."
|
|
||||||
)
|
|
||||||
format = forms.ChoiceField(
|
|
||||||
choices=(
|
|
||||||
('json', 'JSON'),
|
|
||||||
('yaml', 'YAML')
|
|
||||||
),
|
|
||||||
initial='yaml'
|
|
||||||
)
|
|
||||||
|
|
||||||
def clean(self):
|
|
||||||
super().clean()
|
|
||||||
|
|
||||||
data = self.cleaned_data['data']
|
|
||||||
format = self.cleaned_data['format']
|
|
||||||
|
|
||||||
# Process JSON/YAML data
|
|
||||||
if format == 'json':
|
|
||||||
try:
|
|
||||||
self.cleaned_data['data'] = json.loads(data)
|
|
||||||
# Check for multiple JSON objects
|
|
||||||
if type(self.cleaned_data['data']) is not dict:
|
|
||||||
raise forms.ValidationError({
|
|
||||||
'data': "Import is limited to one object at a time."
|
|
||||||
})
|
|
||||||
except json.decoder.JSONDecodeError as err:
|
|
||||||
raise forms.ValidationError({
|
|
||||||
'data': "Invalid JSON data: {}".format(err)
|
|
||||||
})
|
|
||||||
else:
|
|
||||||
# Check for multiple YAML documents
|
|
||||||
if '\n---' in data:
|
|
||||||
raise forms.ValidationError({
|
|
||||||
'data': "Import is limited to one object at a time."
|
|
||||||
})
|
|
||||||
try:
|
|
||||||
self.cleaned_data['data'] = yaml.load(data, Loader=yaml.SafeLoader)
|
|
||||||
except yaml.error.YAMLError as err:
|
|
||||||
raise forms.ValidationError({
|
|
||||||
'data': "Invalid YAML data: {}".format(err)
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
class BaseImportForm(BootstrapMixin, forms.Form):
|
class BaseImportForm(BootstrapMixin, forms.Form):
|
||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
|
related = kwargs.pop("related", False)
|
||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
self.fields['format'].choices = self.get_supported_formats()
|
if related:
|
||||||
|
self.fields['format'].choices = ImportFormatChoicesRelated.CHOICES
|
||||||
|
self.fields['format'].initial = ImportFormatChoicesRelated.YAML
|
||||||
|
|
||||||
def get_supported_formats(self):
|
@property
|
||||||
return (
|
def data_field(self):
|
||||||
('csv', 'CSV'),
|
return 'data'
|
||||||
('json', 'JSON'),
|
|
||||||
('yaml', 'YAML')
|
|
||||||
)
|
|
||||||
|
|
||||||
def convert_data(self, data):
|
def convert_data(self, data):
|
||||||
format = self.cleaned_data['format']
|
format = self.cleaned_data['format']
|
||||||
stream = StringIO(data.strip())
|
stream = StringIO(data.strip())
|
||||||
|
|
||||||
# Process data
|
# Process data
|
||||||
if format == 'csv':
|
if format == ImportFormatChoices.CSV:
|
||||||
reader = csv.reader(stream)
|
reader = csv.reader(stream)
|
||||||
headers, records = parse_csv(reader)
|
headers, records = parse_csv(reader)
|
||||||
self.cleaned_data['data'] = records
|
self.cleaned_data['data'] = records
|
||||||
self.cleaned_data['headers'] = headers
|
self.cleaned_data['headers'] = headers
|
||||||
elif format == 'json':
|
elif format == ImportFormatChoices.JSON:
|
||||||
try:
|
try:
|
||||||
self.cleaned_data['data'] = json.loads(data)
|
self.cleaned_data['data'] = json.loads(data)
|
||||||
except json.decoder.JSONDecodeError as err:
|
except json.decoder.JSONDecodeError as err:
|
||||||
raise forms.ValidationError({
|
raise forms.ValidationError({
|
||||||
'data': f"Invalid JSON data: {err}"
|
self.data_field: f"Invalid JSON data: {err}"
|
||||||
})
|
})
|
||||||
elif format == 'yaml':
|
elif format == ImportFormatChoices.YAML:
|
||||||
try:
|
try:
|
||||||
self.cleaned_data['data'] = yaml.load_all(data, Loader=yaml.SafeLoader)
|
self.cleaned_data['data'] = yaml.load_all(data, Loader=yaml.SafeLoader)
|
||||||
except yaml.error.YAMLError as err:
|
except yaml.error.YAMLError as err:
|
||||||
raise forms.ValidationError({
|
raise forms.ValidationError({
|
||||||
'data': f"Invalid YAML data: {err}"
|
self.data_field: f"Invalid YAML data: {err}"
|
||||||
})
|
})
|
||||||
else:
|
else:
|
||||||
raise forms.ValidationError({
|
raise forms.ValidationError({
|
||||||
'data': f"Invalid file format: {format}"
|
self.data_field: f"Invalid file format: {format}"
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
@ -234,8 +187,8 @@ class ImportForm(BaseImportForm):
|
|||||||
help_text="Enter object data in CSV, JSON or YAML format."
|
help_text="Enter object data in CSV, JSON or YAML format."
|
||||||
)
|
)
|
||||||
format = forms.ChoiceField(
|
format = forms.ChoiceField(
|
||||||
choices=(),
|
choices=ImportFormatChoices.CHOICES,
|
||||||
initial='csv'
|
initial=ImportFormatChoices.CSV
|
||||||
)
|
)
|
||||||
|
|
||||||
def clean(self):
|
def clean(self):
|
||||||
@ -253,19 +206,22 @@ class FileUploadImportForm(BaseImportForm):
|
|||||||
label="data file",
|
label="data file",
|
||||||
required=False
|
required=False
|
||||||
)
|
)
|
||||||
|
|
||||||
format = forms.ChoiceField(
|
format = forms.ChoiceField(
|
||||||
choices=(),
|
choices=ImportFormatChoices.CHOICES,
|
||||||
initial='csv'
|
initial=ImportFormatChoices.CSV
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def data_field(self):
|
||||||
|
return 'data_file'
|
||||||
|
|
||||||
def clean(self):
|
def clean(self):
|
||||||
super().clean()
|
super().clean()
|
||||||
|
|
||||||
file = self.files.get('data_file')
|
file = self.files.get('data_file')
|
||||||
|
|
||||||
data = file.read().decode('utf-8')
|
data = file.read().decode('utf-8')
|
||||||
self.convert_data()
|
self.convert_data(data)
|
||||||
|
|
||||||
|
|
||||||
class FilterForm(BootstrapMixin, forms.Form):
|
class FilterForm(BootstrapMixin, forms.Form):
|
||||||
|
@ -548,7 +548,8 @@ class ViewTestCases:
|
|||||||
|
|
||||||
def test_bulk_import_objects_without_permission(self):
|
def test_bulk_import_objects_without_permission(self):
|
||||||
data = {
|
data = {
|
||||||
'csv': self._get_csv_data(),
|
'data': self._get_csv_data(),
|
||||||
|
'format': 'csv',
|
||||||
}
|
}
|
||||||
|
|
||||||
# Test GET without permission
|
# Test GET without permission
|
||||||
@ -564,7 +565,8 @@ class ViewTestCases:
|
|||||||
def test_bulk_import_objects_with_permission(self):
|
def test_bulk_import_objects_with_permission(self):
|
||||||
initial_count = self._get_queryset().count()
|
initial_count = self._get_queryset().count()
|
||||||
data = {
|
data = {
|
||||||
'csv': self._get_csv_data(),
|
'data': self._get_csv_data(),
|
||||||
|
'format': 'csv',
|
||||||
}
|
}
|
||||||
|
|
||||||
# Assign model-level permission
|
# Assign model-level permission
|
||||||
@ -587,7 +589,8 @@ class ViewTestCases:
|
|||||||
def test_bulk_import_objects_with_constrained_permission(self):
|
def test_bulk_import_objects_with_constrained_permission(self):
|
||||||
initial_count = self._get_queryset().count()
|
initial_count = self._get_queryset().count()
|
||||||
data = {
|
data = {
|
||||||
'csv': self._get_csv_data(),
|
'data': self._get_csv_data(),
|
||||||
|
'format': 'csv',
|
||||||
}
|
}
|
||||||
|
|
||||||
# Assign constrained permission
|
# Assign constrained permission
|
||||||
|
Loading…
Reference in New Issue
Block a user