4347 initial code for json import

netbox/netbox/views/generic/bulk_views.py

@@ -19,8 +19,10 @@ from extras.signals import clear_webhooks
 from utilities.error_handlers import handle_protectederror
 from utilities.exceptions import AbortRequest, PermissionsViolation
 from utilities.forms import (
-    BootstrapMixin, BulkRenameForm, ConfirmationForm, CSVDataField, CSVFileField, restrict_form_fields,
+    BootstrapMixin, BulkRenameForm, ConfirmationForm, CSVDataField, CSVFileField,
+    ImportForm, FileUploadImportForm, restrict_form_fields,
 )
+
 from utilities.htmx import is_htmx
 from utilities.permissions import get_permission_for_model
 from utilities.views import GetReturnURLMixin
@@ -286,7 +288,7 @@ class BulkCreateView(GetReturnURLMixin, BaseMultiObjectView):
         })
 
 
-class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
+class OldBulkImportView(GetReturnURLMixin, BaseMultiObjectView):
     """
     Import objects in bulk (CSV format).
 
@@ -414,6 +416,191 @@ class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
         })
 
 
+class BulkImportView(GetReturnURLMixin, BaseMultiObjectView):
+    """
+    Import objects in bulk (CSV format).
+
+    Attributes:
+        model_form: The form used to create each imported object
+    """
+    template_name = 'generic/bulk_import.html'
+    model_form = None
+
+    '''
+    supported_formats = [
+        {
+            'name': 'CSV',
+            'help_text': 'Enter the list of column headers followed by one line per record to be imported, using ' \
+                         'commas to separate values. Multi-line data and values containing commas may be wrapped ' \
+                         'in double quotes.'
+        },
+        {'name': 'JSON', },
+        {'name': 'YAML', },
+    ]
+    '''
+
+    def _create_object(self, request, model_form):
+
+        # Save the primary object
+        obj = self._save_obj(model_form, request)
+
+        # Enforce object-level permissions
+        if not self.queryset.filter(pk=obj.pk).first():
+            raise PermissionsViolation()
+
+        # Iterate through the related object forms (if any), validating and saving each instance.
+        for field_name, related_object_form in self.related_object_forms.items():
+
+            related_obj_pks = []
+            for i, rel_obj_data in enumerate(model_form.data.get(field_name, list())):
+                rel_obj_data = self.prep_related_object_data(obj, rel_obj_data)
+                f = related_object_form(rel_obj_data)
+
+                for subfield_name, field in f.fields.items():
+                    if subfield_name not in rel_obj_data and hasattr(field, 'initial'):
+                        f.data[subfield_name] = field.initial
+
+                if f.is_valid():
+                    related_obj = f.save()
+                    related_obj_pks.append(related_obj.pk)
+                else:
+                    # Replicate errors on the related object form to the primary form for display
+                    for subfield_name, errors in f.errors.items():
+                        for err in errors:
+                            err_msg = "{}[{}] {}: {}".format(field_name, i, subfield_name, err)
+                            model_form.add_error(None, err_msg)
+                    raise AbortTransaction()
+
+            # Enforce object-level permissions on related objects
+            model = related_object_form.Meta.model
+            if model.objects.filter(pk__in=related_obj_pks).count() != len(related_obj_pks):
+                raise ObjectDoesNotExist
+
+        return obj
+
+    def _create_objects(self, form, request):
+        new_objs = []
+        for row_num, record in enumerate(data['data'], start=1):
+            if format == 'csv':
+                model_form = self.model_form(record, headers=headers)
+            else:
+                model_form = self.model_form(record)
+            restrict_form_fields(model_form, request.user)
+
+            if format == 'json' or format == 'yaml':
+                # Assign default values for any fields which were not specified.
+                # We have to do this manually because passing 'initial=' to the form
+                # on initialization merely sets default values for the widgets.
+                # Since widgets are not used for YAML/JSON import, we first bind the
+                # imported data normally, then update the form's data with the applicable
+                # field defaults as needed prior to form validation.
+                for field_name, field in model_form.fields.items():
+                    if field_name not in record and hasattr(field, 'initial'):
+                        model_form.data[field_name] = field.initial
+
+            if model_form.is_valid():
+                obj = self._create_object(request, model_form)
+                new_objs.append(obj)
+            else:
+                # Replicate model form errors for display
+                for field, errors in model_form.errors.items():
+                    for err in errors:
+                        if format == 'csv':
+                            form.add_error('csv', f'Row {row} {field}: {err[0]}')
+                        else:
+                            if field == '__all__':
+                                form.add_error(None, err)
+                            else:
+                                form.add_error(None, "{}: {}".format(field, err))
+
+                raise ValidationError("")
+
+        return new_objs
+
+    def _save_obj(self, obj_form, request):
+        """
+        Provide a hook to modify the object immediately before saving it (e.g. to encrypt secret data).
+        """
+        return obj_form.save()
+
+    def get_required_permission(self):
+        return get_permission_for_model(self.queryset.model, 'add')
+
+    #
+    # Request handlers
+    #
+
+    def get_context(self, request, data_form, file_form):
+        return {
+            'model': self.model_form._meta.model,
+            'data_form': data_form,
+            'file_form': file_form,
+            'fields': self.model_form().fields,
+            'return_url': self.get_return_url(request),
+            **self.get_extra_context(request),
+        }
+
+    def get(self, request):
+        data_form = ImportForm()
+        file_form = FileUploadImportForm()
+
+        return render(request, self.template_name, self.get_context(request, data_form, file_form))
+
+    def post(self, request):
+        logger = logging.getLogger('netbox.views.BulkImportView')
+        data_form = ImportForm(request.POST)
+        file_form = FileUploadImportForm(request.POST, request.FILES)
+
+        data = None
+        if 'data_submit' in request.POST:
+            if data_form.is_valid():
+                logger.debug("Data Import form validation was successful")
+                data = data_form.cleaned_data
+        elif 'file_submit' in request.POST:
+            if file_form.is_valid():
+                logger.debug("File Import form validation was successful")
+                data = file_form.cleaned_data
+
+        if data:
+            format = data['format']
+            headers = data['headers'] if format == 'csv' else None
+
+            try:
+                # Iterate through data and bind each row to a new model form instance.
+                with transaction.atomic():
+                    new_objs = self._create_objects(form, request)
+
+                    # Enforce object-level permissions
+                    if self.queryset.filter(pk__in=[obj.pk for obj in new_objs]).count() != len(new_objs):
+                        raise PermissionsViolation
+
+                # Compile a table containing the imported objects
+                obj_table = self.table(new_objs)
+
+                if new_objs:
+                    msg = 'Imported {} {}'.format(len(new_objs), new_objs[0]._meta.verbose_name_plural)
+                    logger.info(msg)
+                    messages.success(request, msg)
+
+                    return render(request, "import_success.html", {
+                        'table': obj_table,
+                        'return_url': self.get_return_url(request),
+                    })
+
+            except ValidationError:
+                clear_webhooks.send(sender=self)
+
+            except (AbortRequest, PermissionsViolation) as e:
+                logger.debug(e.message)
+                form.add_error(None, e.message)
+                clear_webhooks.send(sender=self)
+
+        else:
+            logger.debug("Form validation failed")
+
+        return render(request, self.template_name, self.get_context(request, data_form, file_form))
+
+
 class BulkEditView(GetReturnURLMixin, BaseMultiObjectView):
     """
     Edit objects in bulk.

netbox/netbox/views/generic/object_views.py

@@ -212,8 +212,10 @@ class ObjectImportView(GetReturnURLMixin, BaseObjectView):
 
     def get(self, request):
         form = ImportForm()
+        model = self.queryset.model
 
         return render(request, self.template_name, {
+            'model': model,
             'form': form,
             'obj_type': self.queryset.model._meta.verbose_name,
             'return_url': self.get_return_url(request),
@@ -279,6 +281,7 @@ class ObjectImportView(GetReturnURLMixin, BaseObjectView):
             logger.debug("Import form validation failed")
 
         return render(request, self.template_name, {
+            'model': model,
             'form': form,
             'obj_type': self.queryset.model._meta.verbose_name,
             'return_url': self.get_return_url(request),

netbox/templates/generic/bulk_import.html

@@ -12,6 +12,170 @@ Context:
 
 {% block title %}{{ model|meta:"verbose_name"|bettertitle }} Bulk Import{% endblock %}
 
+{% block tabs %}
+  <ul class="nav nav-tabs px-3">
+    <li class="nav-item" role="presentation">
+      <button class="nav-link active" id="data-import-tab" data-bs-toggle="tab" data-bs-target="#data-import-form" type="button" role="tab" aria-controls="data-import-form" aria-selected="true">
+        Data Import
+      </button>
+    </li>
+    <li class="nav-item" role="presentation">
+      <button class="nav-link" id="file-upload-tab" data-bs-toggle="tab" data-bs-target="#file-upload-form" type="button" role="tab" aria-controls="file-upload-form" aria-selected="false">
+        Upload File
+      </button>
+    </li>
+  </ul>
+{% endblock tabs %}
+
+{% block content-wrapper %}
+  <div class="tab-content">
+
+    {# Data Import Form #}
+    <div class="tab-pane show active" id="data-import-form" role="tabpanel" aria-labelledby="data-import-tab">
+      {% block content %}
+          <div class="row">
+              <div class="col col-md-12 col-lg-10">
+                  <form action="" method="post" enctype="multipart/form-data" class="form">
+                    {% csrf_token %}
+                    {% render_form data_form %}
+                    <div class="form-group">
+                      <div class="col col-md-12 text-end">
+                        <button type="submit" name="data_submit" class="btn btn-primary">Submit</button>
+                        {% if return_url %}
+                          <a href="{{ return_url }}" class="btn btn-outline-danger">Cancel</a>
+                        {% endif %}
+                      </div>
+                    </div>
+                  </form>
+              </div>
+          </div>
+      {% endblock content %}
+    </div>
+
+    {# File Upload Form #}
+    <div class="tab-pane show" id="file-upload-form" role="tabpanel" aria-labelledby="file-upload-tab">
+              <div class="col col-md-12 col-lg-10">
+                  <form action="" method="post" enctype="multipart/form-data" class="form">
+                    {% csrf_token %}
+                    {% render_form file_form %}
+                    <div class="form-group">
+                      <div class="col col-md-12 text-end">
+                        <button type="submit" name="file_submit" class="btn btn-primary">Submit</button>
+                        {% if return_url %}
+                          <a href="{{ return_url }}" class="btn btn-outline-danger">Cancel</a>
+                        {% endif %}
+                      </div>
+                    </div>
+                  </form>
+              </div>
+    </div>
+
+    {% if fields %}
+    <div class="row my-3">
+        <div class="col col-md-12">
+            <div class="card">
+                <h5 class="card-header">
+                    Field Options
+                </h5>
+                <div class="card-body">
+                    <table class="table">
+                        <tr>
+                            <th>Field</th>
+                            <th>Required</th>
+                            <th>Accessor</th>
+                            <th>Description</th>
+                        </tr>
+                        {% for name, field in fields.items %}
+                            <tr>
+                                <td>
+                                    <code>{% if field.required %}<strong>{% endif %}{{ name }}{% if field.required %}</strong>{% endif %}</code>
+                                </td>
+                                <td>
+                                    {% if field.required %}
+                                        {% checkmark True true="Required" %}
+                                    {% else %}
+                                        {{ ''|placeholder }}
+                                    {% endif %}
+                                </td>
+                                <td>
+                                    {% if field.to_field_name %}
+                                        <code>{{ field.to_field_name }}</code>
+                                    {% else %}
+                                        {{ ''|placeholder }}
+                                    {% endif %}
+                                </td>
+                                <td>
+                                    {% if field.STATIC_CHOICES %}
+                                        <button type="button" class="btn btn-link btn-sm float-end" data-bs-toggle="modal" data-bs-target="#{{ name }}_choices">
+                                            <i class="mdi mdi-help-circle"></i>
+                                        </button>
+                                        <div class="modal fade" id="{{ name }}_choices" tabindex="-1" role="dialog">
+                                            <div class="modal-dialog" role="document">
+                                                <div class="modal-content">
+                                                    <div class="modal-header">
+                                                        <h5 class="modal-title"><code>{{ name }}</code> Choices</h5>
+                                                        <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+                                                    </div>
+                                                    <div class="modal-body">
+                                                        <table class="table table-striped">
+                                                            <tr>
+                                                                <th>Import Value</th>
+                                                                <th>Label</th>
+                                                            </tr>
+                                                            {% for value, label in field.choices %}
+                                                            {% if value %}
+                                                            <tr>
+                                                                <td>
+                                                                    <samp>{{ value }}</samp>
+                                                                </td>
+                                                                <td>
+                                                                    {{ label }}
+                                                                </td>
+                                                            </tr>
+                                                            {% endif %}
+                                                            {% endfor %}
+                                                        </table>
+                                                    </div>
+                                                </div>
+                                            </div>
+                                        </div>
+                                    {% endif %}
+                                    {% if field.help_text %}
+                                        {{ field.help_text }}<br />
+                                    {% elif field.label %}
+                                        {{ field.label }}<br />
+                                    {% endif %}
+                                    {% if field|widget_type == 'dateinput' %}
+                                        <small class="text-muted">Format: YYYY-MM-DD</small>
+                                    {% elif field|widget_type == 'checkboxinput' %}
+                                        <small class="text-muted">Specify "true" or "false"</small>
+                                    {% endif %}
+                                </td>
+                            </tr>
+                        {% endfor %}
+                    </table>
+                </div>
+            </div>
+        </div>
+    </div>
+    <p class="small text-muted">
+        <i class="mdi mdi-check-bold text-success"></i> Required fields <strong>must</strong> be specified for all
+        objects.
+    </p>
+    <p class="small text-muted">
+        <i class="mdi mdi-information-outline"></i> Related objects may be referenced by any unique attribute.
+        For example, <code>vrf.rd</code> would identify a VRF by its route distinguisher.
+    </p>
+    {% endif %}
+
+  </div>
+{% endblock content-wrapper %}
+
+
+
+
+
+{% comment %}
 {% block tabs %}
   <ul class="nav nav-tabs px-3">
     <li class="nav-item" role="presentation">
@@ -154,3 +318,4 @@ Context:
     {% endblock content %}
   </div>
 {% endblock content-wrapper %}
+{% endcomment %}

netbox/utilities/forms/forms.py

@@ -1,12 +1,13 @@
 import json
 import re
+from io import StringIO
 
 import yaml
 from django import forms
+from utilities.forms.utils import parse_csv, validate_csv
 
 from .widgets import APISelect, APISelectMultiple, ClearableFileInput, StaticSelect
 
-
 __all__ = (
     'BootstrapMixin',
     'BulkEditForm',
@@ -15,6 +16,7 @@ __all__ = (
     'CSVModelForm',
     'FilterForm',
     'ImportForm',
+    'FileUploadImportForm',
     'ReturnURLForm',
     'TableConfigForm',
 )
@@ -131,7 +133,7 @@ class CSVModelForm(forms.ModelForm):
                     self.fields[field].to_field_name = to_field
 
 
-class ImportForm(BootstrapMixin, forms.Form):
+class OldImportForm(BootstrapMixin, forms.Form):
     """
     Generic form for creating an object from JSON/YAML data
     """
@@ -180,6 +182,92 @@ class ImportForm(BootstrapMixin, forms.Form):
                 })
 
 
+class BaseImportForm(BootstrapMixin, forms.Form):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields['format'].choices = self.get_supported_formats()
+
+    def get_supported_formats(self):
+        return (
+            ('csv', 'CSV'),
+            ('json', 'JSON'),
+            ('yaml', 'YAML')
+        )
+
+    def convert_data(self, data):
+        format = self.cleaned_data['format']
+        stream = StringIO(data.strip())
+
+        # Process data
+        if format == 'csv':
+            reader = csv.reader(stream)
+            headers, records = parse_csv(reader)
+            self.cleaned_data['data'] = records
+            self.cleaned_data['headers'] = headers
+        elif format == 'json':
+            try:
+                self.cleaned_data['data'] = json.loads(data)
+            except json.decoder.JSONDecodeError as err:
+                raise forms.ValidationError({
+                    'data': f"Invalid JSON data: {err}"
+                })
+        elif format == 'yaml':
+            try:
+                self.cleaned_data['data'] = yaml.load_all(data, Loader=yaml.SafeLoader)
+            except yaml.error.YAMLError as err:
+                raise forms.ValidationError({
+                    'data': f"Invalid YAML data: {err}"
+                })
+        else:
+            raise forms.ValidationError({
+                'data': f"Invalid file format: {format}"
+            })
+
+
+class ImportForm(BaseImportForm):
+    """
+    Generic form for creating an object from JSON/YAML data
+    """
+    data = forms.CharField(
+        widget=forms.Textarea(attrs={'class': 'font-monospace'}),
+        help_text="Enter object data in CSV, JSON or YAML format."
+    )
+    format = forms.ChoiceField(
+        choices=(),
+        initial='csv'
+    )
+
+    def clean(self):
+        super().clean()
+
+        data = self.cleaned_data['data'] if 'data' in self.cleaned_data else None
+        self.convert_data(data)
+
+
+class FileUploadImportForm(BaseImportForm):
+    """
+    Generic form for creating an object from JSON/YAML data
+    """
+    data_file = forms.FileField(
+        label="data file",
+        required=False
+    )
+
+    format = forms.ChoiceField(
+        choices=(),
+        initial='csv'
+    )
+
+    def clean(self):
+        super().clean()
+
+        file = self.files.get('data_file')
+
+        data = file.read().decode('utf-8')
+        self.convert_data()
+
+
 class FilterForm(BootstrapMixin, forms.Form):
     """
     Base Form class for FilterSet forms.