mirror of
https://github.com/netbox-community/netbox.git
synced 2025-08-24 08:25:17 -06:00
4347 initial code for json import
This commit is contained in:
Arthur
jeremystretch
parent
d486fa8452
commit
0ce7f84ec1
@ -19,8 +19,10 @@ from extras.signals import clear_webhooks
|
||||
from utilities.error_handlers import handle_protectederror
|
||||
from utilities.exceptions import AbortRequest, PermissionsViolation
|
||||
from utilities.forms import (
|
||||
BootstrapMixin, BulkRenameForm, ConfirmationForm, CSVDataField, CSVFileField, restrict_form_fields,
|
||||
BootstrapMixin, BulkRenameForm, ConfirmationForm, CSVDataField, CSVFileField,
|
||||
ImportForm, FileUploadImportForm, restrict_form_fields,
|
||||
)
|
||||
|
||||
from utilities.htmx import is_htmx
|
||||
from utilities.permissions import get_permission_for_model
|
||||
from utilities.views import GetReturnURLMixin
|
||||
@ -286,7 +288,7 @@ class BulkCreateView(GetReturnURLMixin, BaseMultiObjectView):
|
||||
})
|
||||
|
||||
|
||||
class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
|
||||
class OldBulkImportView(GetReturnURLMixin, BaseMultiObjectView):
|
||||
"""
|
||||
Import objects in bulk (CSV format).
|
||||
|
||||
@ -414,6 +416,191 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
|
||||
})
|
||||
|
||||
|
||||
class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
|
||||
"""
|
||||
Import objects in bulk (CSV format).
|
||||
|
||||
Attributes:
|
||||
model_form: The form used to create each imported object
|
||||
"""
|
||||
template_name = 'generic/bulk_import.html'
|
||||
model_form = None
|
||||
|
||||
'''
|
||||
supported_formats = [
|
||||
{
|
||||
'name': 'CSV',
|
||||
'help_text': 'Enter the list of column headers followed by one line per record to be imported, using ' \
|
||||
'commas to separate values. Multi-line data and values containing commas may be wrapped ' \
|
||||
'in double quotes.'
|
||||
},
|
||||
{'name': 'JSON', },
|
||||
{'name': 'YAML', },
|
||||
]
|
||||
'''
|
||||
|
||||
def _create_object(self, request, model_form):
|
||||
|
||||
# Save the primary object
|
||||
obj = self._save_obj(model_form, request)
|
||||
|
||||
# Enforce object-level permissions
|
||||
if not self.queryset.filter(pk=obj.pk).first():
|
||||
raise PermissionsViolation()
|
||||
|
||||
# Iterate through the related object forms (if any), validating and saving each instance.
|
||||
for field_name, related_object_form in self.related_object_forms.items():
|
||||
|
||||
related_obj_pks = []
|
||||
for i, rel_obj_data in enumerate(model_form.data.get(field_name, list())):
|
||||
rel_obj_data = self.prep_related_object_data(obj, rel_obj_data)
|
||||
f = related_object_form(rel_obj_data)
|
||||
|
||||
for subfield_name, field in f.fields.items():
|
||||
if subfield_name not in rel_obj_data and hasattr(field, 'initial'):
|
||||
f.data[subfield_name] = field.initial
|
||||
|
||||
if f.is_valid():
|
||||
related_obj = f.save()
|
||||
related_obj_pks.append(related_obj.pk)
|
||||
else:
|
||||
# Replicate errors on the related object form to the primary form for display
|
||||
for subfield_name, errors in f.errors.items():
|
||||
for err in errors:
|
||||
err_msg = "{}[{}] {}: {}".format(field_name, i, subfield_name, err)
|
||||
model_form.add_error(None, err_msg)
|
||||
raise AbortTransaction()
|
||||
|
||||
# Enforce object-level permissions on related objects
|
||||
model = related_object_form.Meta.model
|
||||
if model.objects.filter(pk__in=related_obj_pks).count() != len(related_obj_pks):
|
||||
raise ObjectDoesNotExist
|
||||
|
||||
return obj
|
||||
|
||||
def _create_objects(self, form, request):
|
||||
new_objs = []
|
||||
for row_num, record in enumerate(data['data'], start=1):
|
||||
if format == 'csv':
|
||||
model_form = self.model_form(record, headers=headers)
|
||||
else:
|
||||
model_form = self.model_form(record)
|
||||
restrict_form_fields(model_form, request.user)
|
||||
|
||||
if format == 'json' or format == 'yaml':
|
||||
# Assign default values for any fields which were not specified.
|
||||
# We have to do this manually because passing 'initial=' to the form
|
||||
# on initialization merely sets default values for the widgets.
|
||||
# Since widgets are not used for YAML/JSON import, we first bind the
|
||||
# imported data normally, then update the form's data with the applicable
|
||||
# field defaults as needed prior to form validation.
|
||||
for field_name, field in model_form.fields.items():
|
||||
if field_name not in record and hasattr(field, 'initial'):
|
||||
model_form.data[field_name] = field.initial
|
||||
|
||||
if model_form.is_valid():
|
||||
obj = self._create_object(request, model_form)
|
||||
new_objs.append(obj)
|
||||
else:
|
||||
# Replicate model form errors for display
|
||||
for field, errors in model_form.errors.items():
|
||||
for err in errors:
|
||||
if format == 'csv':
|
||||
form.add_error('csv', f'Row {row} {field}: {err[0]}')
|
||||
else:
|
||||
if field == '__all__':
|
||||
form.add_error(None, err)
|
||||
else:
|
||||
form.add_error(None, "{}: {}".format(field, err))
|
||||
|
||||
raise ValidationError("")
|
||||
|
||||
return new_objs
|
||||
|
||||
def _save_obj(self, obj_form, request):
|
||||
"""
|
||||
Provide a hook to modify the object immediately before saving it (e.g. to encrypt secret data).
|
||||
"""
|
||||
return obj_form.save()
|
||||
|
||||
def get_required_permission(self):
|
||||
return get_permission_for_model(self.queryset.model, 'add')
|
||||
|
||||
#
|
||||
# Request handlers
|
||||
#
|
||||
|
||||
def get_context(self, request, data_form, file_form):
|
||||
return {
|
||||
'model': self.model_form._meta.model,
|
||||
'data_form': data_form,
|
||||
'file_form': file_form,
|
||||
'fields': self.model_form().fields,
|
||||
'return_url': self.get_return_url(request),
|
||||
**self.get_extra_context(request),
|
||||
}
|
||||
|
||||
def get(self, request):
|
||||
data_form = ImportForm()
|
||||
file_form = FileUploadImportForm()
|
||||
|
||||
return render(request, self.template_name, self.get_context(request, data_form, file_form))
|
||||
|
||||
def post(self, request):
|
||||
logger = logging.getLogger('netbox.views.BulkImportView')
|
||||
data_form = ImportForm(request.POST)
|
||||
file_form = FileUploadImportForm(request.POST, request.FILES)
|
||||
|
||||
data = None
|
||||
if 'data_submit' in request.POST:
|
||||
if data_form.is_valid():
|
||||
logger.debug("Data Import form validation was successful")
|
||||
data = data_form.cleaned_data
|
||||
elif 'file_submit' in request.POST:
|
||||
if file_form.is_valid():
|
||||
logger.debug("File Import form validation was successful")
|
||||
data = file_form.cleaned_data
|
||||
|
||||
if data:
|
||||
format = data['format']
|
||||
headers = data['headers'] if format == 'csv' else None
|
||||
|
||||
try:
|
||||
# Iterate through data and bind each row to a new model form instance.
|
||||
with transaction.atomic():
|
||||
new_objs = self._create_objects(form, request)
|
||||
|
||||
# Enforce object-level permissions
|
||||
if self.queryset.filter(pk__in=[obj.pk for obj in new_objs]).count() != len(new_objs):
|
||||
raise PermissionsViolation
|
||||
|
||||
# Compile a table containing the imported objects
|
||||
obj_table = self.table(new_objs)
|
||||
|
||||
if new_objs:
|
||||
msg = 'Imported {} {}'.format(len(new_objs), new_objs[0]._meta.verbose_name_plural)
|
||||
logger.info(msg)
|
||||
messages.success(request, msg)
|
||||
|
||||
return render(request, "import_success.html", {
|
||||
'table': obj_table,
|
||||
'return_url': self.get_return_url(request),
|
||||
})
|
||||
|
||||
except ValidationError:
|
||||
clear_webhooks.send(sender=self)
|
||||
|
||||
except (AbortRequest, PermissionsViolation) as e:
|
||||
logger.debug(e.message)
|
||||
form.add_error(None, e.message)
|
||||
clear_webhooks.send(sender=self)
|
||||
|
||||
else:
|
||||
logger.debug("Form validation failed")
|
||||
|
||||
return render(request, self.template_name, self.get_context(request, data_form, file_form))
|
||||
|
||||
|
||||
class BulkEditView(GetReturnURLMixin, BaseMultiObjectView):
|
||||
"""
|
||||
Edit objects in bulk.
|
||||
|
||||
@ -212,8 +212,10 @@ class ObjectImportView(GetReturnURLMixin, BaseObjectView):
|
||||
|
||||
def get(self, request):
|
||||
form = ImportForm()
|
||||
model = self.queryset.model
|
||||
|
||||
return render(request, self.template_name, {
|
||||
'model': model,
|
||||
'form': form,
|
||||
'obj_type': self.queryset.model._meta.verbose_name,
|
||||
'return_url': self.get_return_url(request),
|
||||
@ -279,6 +281,7 @@ class ObjectImportView(GetReturnURLMixin, BaseObjectView):
|
||||
logger.debug("Import form validation failed")
|
||||
|
||||
return render(request, self.template_name, {
|
||||
'model': model,
|
||||
'form': form,
|
||||
'obj_type': self.queryset.model._meta.verbose_name,
|
||||
'return_url': self.get_return_url(request),
|
||||
|
||||
@ -12,6 +12,170 @@ Context:
|
||||
|
||||
{% block title %}{{ model|meta:"verbose_name"|bettertitle }} Bulk Import{% endblock %}
|
||||
|
||||
{% block tabs %}
|
||||
<ul class="nav nav-tabs px-3">
|
||||
<li class="nav-item" role="presentation">
|
||||
<button class="nav-link active" id="data-import-tab" data-bs-toggle="tab" data-bs-target="#data-import-form" type="button" role="tab" aria-controls="data-import-form" aria-selected="true">
|
||||
Data Import
|
||||
</button>
|
||||
</li>
|
||||
<li class="nav-item" role="presentation">
|
||||
<button class="nav-link" id="file-upload-tab" data-bs-toggle="tab" data-bs-target="#file-upload-form" type="button" role="tab" aria-controls="file-upload-form" aria-selected="false">
|
||||
Upload File
|
||||
</button>
|
||||
</li>
|
||||
</ul>
|
||||
{% endblock tabs %}
|
||||
|
||||
{% block content-wrapper %}
|
||||
<div class="tab-content">
|
||||
|
||||
{# Data Import Form #}
|
||||
<div class="tab-pane show active" id="data-import-form" role="tabpanel" aria-labelledby="data-import-tab">
|
||||
{% block content %}
|
||||
<div class="row">
|
||||
<div class="col col-md-12 col-lg-10">
|
||||
<form action="" method="post" enctype="multipart/form-data" class="form">
|
||||
{% csrf_token %}
|
||||
{% render_form data_form %}
|
||||
<div class="form-group">
|
||||
<div class="col col-md-12 text-end">
|
||||
<button type="submit" name="data_submit" class="btn btn-primary">Submit</button>
|
||||
{% if return_url %}
|
||||
<a href="{{ return_url }}" class="btn btn-outline-danger">Cancel</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock content %}
|
||||
</div>
|
||||
|
||||
{# File Upload Form #}
|
||||
<div class="tab-pane show" id="file-upload-form" role="tabpanel" aria-labelledby="file-upload-tab">
|
||||
<div class="col col-md-12 col-lg-10">
|
||||
<form action="" method="post" enctype="multipart/form-data" class="form">
|
||||
{% csrf_token %}
|
||||
{% render_form file_form %}
|
||||
<div class="form-group">
|
||||
<div class="col col-md-12 text-end">
|
||||
<button type="submit" name="file_submit" class="btn btn-primary">Submit</button>
|
||||
{% if return_url %}
|
||||
<a href="{{ return_url }}" class="btn btn-outline-danger">Cancel</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if fields %}
|
||||
<div class="row my-3">
|
||||
<div class="col col-md-12">
|
||||
<div class="card">
|
||||
<h5 class="card-header">
|
||||
Field Options
|
||||
</h5>
|
||||
<div class="card-body">
|
||||
<table class="table">
|
||||
<tr>
|
||||
<th>Field</th>
|
||||
<th>Required</th>
|
||||
<th>Accessor</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
{% for name, field in fields.items %}
|
||||
<tr>
|
||||
<td>
|
||||
<code>{% if field.required %}<strong>{% endif %}{{ name }}{% if field.required %}</strong>{% endif %}</code>
|
||||
</td>
|
||||
<td>
|
||||
{% if field.required %}
|
||||
{% checkmark True true="Required" %}
|
||||
{% else %}
|
||||
{{ ''|placeholder }}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if field.to_field_name %}
|
||||
<code>{{ field.to_field_name }}</code>
|
||||
{% else %}
|
||||
{{ ''|placeholder }}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if field.STATIC_CHOICES %}
|
||||
<button type="button" class="btn btn-link btn-sm float-end" data-bs-toggle="modal" data-bs-target="#{{ name }}_choices">
|
||||
<i class="mdi mdi-help-circle"></i>
|
||||
</button>
|
||||
<div class="modal fade" id="{{ name }}_choices" tabindex="-1" role="dialog">
|
||||
<div class="modal-dialog" role="document">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title"><code>{{ name }}</code> Choices</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<table class="table table-striped">
|
||||
<tr>
|
||||
<th>Import Value</th>
|
||||
<th>Label</th>
|
||||
</tr>
|
||||
{% for value, label in field.choices %}
|
||||
{% if value %}
|
||||
<tr>
|
||||
<td>
|
||||
<samp>{{ value }}</samp>
|
||||
</td>
|
||||
<td>
|
||||
{{ label }}
|
||||
</td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if field.help_text %}
|
||||
{{ field.help_text }}<br />
|
||||
{% elif field.label %}
|
||||
{{ field.label }}<br />
|
||||
{% endif %}
|
||||
{% if field|widget_type == 'dateinput' %}
|
||||
<small class="text-muted">Format: YYYY-MM-DD</small>
|
||||
{% elif field|widget_type == 'checkboxinput' %}
|
||||
<small class="text-muted">Specify "true" or "false"</small>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<p class="small text-muted">
|
||||
<i class="mdi mdi-check-bold text-success"></i> Required fields <strong>must</strong> be specified for all
|
||||
objects.
|
||||
</p>
|
||||
<p class="small text-muted">
|
||||
<i class="mdi mdi-information-outline"></i> Related objects may be referenced by any unique attribute.
|
||||
For example, <code>vrf.rd</code> would identify a VRF by its route distinguisher.
|
||||
</p>
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
{% endblock content-wrapper %}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
{% comment %}
|
||||
{% block tabs %}
|
||||
<ul class="nav nav-tabs px-3">
|
||||
<li class="nav-item" role="presentation">
|
||||
@ -154,3 +318,4 @@ Context:
|
||||
{% endblock content %}
|
||||
</div>
|
||||
{% endblock content-wrapper %}
|
||||
{% endcomment %}
|
||||
|
||||
@ -1,12 +1,13 @@
|
||||
import json
|
||||
import re
|
||||
from io import StringIO
|
||||
|
||||
import yaml
|
||||
from django import forms
|
||||
from utilities.forms.utils import parse_csv, validate_csv
|
||||
|
||||
from .widgets import APISelect, APISelectMultiple, ClearableFileInput, StaticSelect
|
||||
|
||||
|
||||
__all__ = (
|
||||
'BootstrapMixin',
|
||||
'BulkEditForm',
|
||||
@ -15,6 +16,7 @@ __all__ = (
|
||||
'CSVModelForm',
|
||||
'FilterForm',
|
||||
'ImportForm',
|
||||
'FileUploadImportForm',
|
||||
'ReturnURLForm',
|
||||
'TableConfigForm',
|
||||
)
|
||||
@ -131,7 +133,7 @@ class CSVModelForm(forms.ModelForm):
|
||||
self.fields[field].to_field_name = to_field
|
||||
|
||||
|
||||
class ImportForm(BootstrapMixin, forms.Form):
|
||||
class OldImportForm(BootstrapMixin, forms.Form):
|
||||
"""
|
||||
Generic form for creating an object from JSON/YAML data
|
||||
"""
|
||||
@ -180,6 +182,92 @@ class ImportForm(BootstrapMixin, forms.Form):
|
||||
})
|
||||
|
||||
|
||||
class BaseImportForm(BootstrapMixin, forms.Form):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields['format'].choices = self.get_supported_formats()
|
||||
|
||||
def get_supported_formats(self):
|
||||
return (
|
||||
('csv', 'CSV'),
|
||||
('json', 'JSON'),
|
||||
('yaml', 'YAML')
|
||||
)
|
||||
|
||||
def convert_data(self, data):
|
||||
format = self.cleaned_data['format']
|
||||
stream = StringIO(data.strip())
|
||||
|
||||
# Process data
|
||||
if format == 'csv':
|
||||
reader = csv.reader(stream)
|
||||
headers, records = parse_csv(reader)
|
||||
self.cleaned_data['data'] = records
|
||||
self.cleaned_data['headers'] = headers
|
||||
elif format == 'json':
|
||||
try:
|
||||
self.cleaned_data['data'] = json.loads(data)
|
||||
except json.decoder.JSONDecodeError as err:
|
||||
raise forms.ValidationError({
|
||||
'data': f"Invalid JSON data: {err}"
|
||||
})
|
||||
elif format == 'yaml':
|
||||
try:
|
||||
self.cleaned_data['data'] = yaml.load_all(data, Loader=yaml.SafeLoader)
|
||||
except yaml.error.YAMLError as err:
|
||||
raise forms.ValidationError({
|
||||
'data': f"Invalid YAML data: {err}"
|
||||
})
|
||||
else:
|
||||
raise forms.ValidationError({
|
||||
'data': f"Invalid file format: {format}"
|
||||
})
|
||||
|
||||
|
||||
class ImportForm(BaseImportForm):
|
||||
"""
|
||||
Generic form for creating an object from JSON/YAML data
|
||||
"""
|
||||
data = forms.CharField(
|
||||
widget=forms.Textarea(attrs={'class': 'font-monospace'}),
|
||||
help_text="Enter object data in CSV, JSON or YAML format."
|
||||
)
|
||||
format = forms.ChoiceField(
|
||||
choices=(),
|
||||
initial='csv'
|
||||
)
|
||||
|
||||
def clean(self):
|
||||
super().clean()
|
||||
|
||||
data = self.cleaned_data['data'] if 'data' in self.cleaned_data else None
|
||||
self.convert_data(data)
|
||||
|
||||
|
||||
class FileUploadImportForm(BaseImportForm):
|
||||
"""
|
||||
Generic form for creating an object from JSON/YAML data
|
||||
"""
|
||||
data_file = forms.FileField(
|
||||
label="data file",
|
||||
required=False
|
||||
)
|
||||
|
||||
format = forms.ChoiceField(
|
||||
choices=(),
|
||||
initial='csv'
|
||||
)
|
||||
|
||||
def clean(self):
|
||||
super().clean()
|
||||
|
||||
file = self.files.get('data_file')
|
||||
|
||||
data = file.read().decode('utf-8')
|
||||
self.convert_data()
|
||||
|
||||
|
||||
class FilterForm(BootstrapMixin, forms.Form):
|
||||
"""
|
||||
Base Form class for FilterSet forms.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user