diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index bfaea7605..a767f2b1e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -17,7 +17,7 @@ body: What version of NetBox are you currently running? (If you don't have access to the most recent NetBox release, consider testing on our [demo instance](https://demo.netbox.dev/) before opening a bug report to see if your issue has already been addressed.) - placeholder: v2.11.9 + placeholder: v2.11.10 validations: required: true - type: dropdown diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index bfb0cc7aa..319538cda 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v2.11.9 + placeholder: v2.11.10 validations: required: true - type: dropdown diff --git a/.gitignore b/.gitignore index e8865f5b2..b594efe4b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ yarn-error.log* !/netbox/project-static/docs/.info /netbox/netbox/configuration.py /netbox/netbox/ldap_config.py +/netbox/project-static/.cache +/netbox/project-static/node_modules /netbox/reports/* !/netbox/reports/__init__.py /netbox/scripts/* diff --git a/contrib/nginx.conf b/contrib/nginx.conf index 1230f3ce4..34821cd52 100644 --- a/contrib/nginx.conf +++ b/contrib/nginx.conf @@ -1,5 +1,5 @@ server { - listen 443 ssl; + listen [::]:443 ssl ipv6only=off; # CHANGE THIS TO YOUR SERVER'S NAME server_name netbox.example.com; @@ -23,7 +23,7 @@ server { server { # Redirect HTTP traffic to HTTPS - listen 80; + listen [::]:80 ipv6only=off; server_name _; return 301 https://$host$request_uri; } diff --git a/docs/development/adding-models.md b/docs/development/adding-models.md new file mode 100644 index 000000000..6b778d886 --- /dev/null +++ b/docs/development/adding-models.md @@ -0,0 +1,85 @@ +# Adding Models + +## 1. Define the model class + +Models within each app are stored in either `models.py` or within a submodule under the `models/` directory. When creating a model, be sure to subclass the [appropriate base model](models.md) from `netbox.models`. This will typically be PrimaryModel or OrganizationalModel. Remember to add the model class to the `__all__` listing for the module. + +Each model should define, at a minimum: + +* A `__str__()` method returning a user-friendly string representation of the instance +* A `get_absolute_url()` method returning an instance's direct URL (using `reverse()`) +* A `Meta` class specifying a deterministic ordering (if ordered by fields other than the primary ID) + +## 2. Define field choices + +If the model has one or more fields with static choices, define those choices in `choices.py` by subclassing `utilities.choices.ChoiceSet`. + +## 3. Generate database migrations + +Once your model definition is complete, generate database migrations by running `manage.py -n $NAME --no-header`. Always specify a short unique name when generating migrations. + +!!! info + Set `DEVELOPER = True` in your NetBox configuration to enable the creation of new migrations. + +## 4. Add all standard views + +Most models will need view classes created in `views.py` to serve the following operations: + +* List view +* Detail view +* Edit view +* Delete view +* Bulk import +* Bulk edit +* Bulk delete + +## 5. Add URL paths + +Add the relevant URL path for each view created in the previous step to `urls.py`. + +## 6. Create the FilterSet + +Each model should have a corresponding FilterSet class defined. This is used to filter UI and API queries. Subclass the appropriate class from `netbox.filtersets` that matches the model's parent class. + +Every model FilterSet should define a `q` filter to support general search queries. + +## 7. Create the table + +Create a table class for the model in `tables.py` by subclassing `utilities.tables.BaseTable`. Under the table's `Meta` class, be sure to list both the fields and default columns. + +## 8. Create the object template + +Create the HTML template for the object view. (The other views each typically employ a generic template.) This template should extend `generic/object.html`. + +## 9. Add the model to the navigation menu + +For NetBox releases prior to v3.0, add the relevant link(s) to the navigation menu template. For later releases, add the relevant items in `netbox/netbox/navigation_menu.py`. + +## 10. REST API components + +Create the following for each model: + +* Detailed (full) model serializer in `api/serializers.py` +* Nested serializer in `api/nested_serializers.py` +* API view in `api/views.py` +* Endpoint route in `api/urls.py` + +## 11. GraphQL API components (v3.0+) + +Create a Graphene object type for the model in `graphql/types.py` by subclassing the appropriate class from `netbox.graphql.types`. + +Also extend the schema class defined in `graphql/schema.py` with the individual object and object list fields per the established convention. + +## 12. Add tests + +Add tests for the following: + +* UI views +* API views +* Filter sets + +## 13. Documentation + +Create a new documentation page for the model in `docs/models//.md`. Include this file under the "features" documentation where appropriate. + +Also add your model to the index in `docs/development/models.md`. diff --git a/docs/index.md b/docs/index.md index 0f47a6eb8..ad28a708c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,4 @@ -![NetBox](netbox_logo.svg "NetBox logo") +![NetBox](netbox_logo.svg "NetBox logo"){style="height: 100px; margin-bottom: 3em"} # What is NetBox? diff --git a/docs/installation/3-netbox.md b/docs/installation/3-netbox.md index 379496987..24d46006b 100644 --- a/docs/installation/3-netbox.md +++ b/docs/installation/3-netbox.md @@ -288,6 +288,13 @@ Quit the server with CONTROL-C. Next, connect to the name or IP of the server (as defined in `ALLOWED_HOSTS`) on port 8000; for example, . You should be greeted with the NetBox home page. Try logging in using the username and password specified when creating a superuser. +!!! note + By default RHEL based distros will likely block your testing attempts with firewalld. The development server port can be opened with `firewall-cmd` (add `--permanent` if you want the rule to survive server restarts): + + ```no-highlight + firewall-cmd --zone=public --add-port=8000/tcp + ``` + !!! danger The development server is for development and testing purposes only. It is neither performant nor secure enough for production use. **Do not use it in production.** diff --git a/docs/installation/6-ldap.md b/docs/installation/6-ldap.md index 27bdb0b40..86114dfb0 100644 --- a/docs/installation/6-ldap.md +++ b/docs/installation/6-ldap.md @@ -74,7 +74,7 @@ STARTTLS can be configured by setting `AUTH_LDAP_START_TLS = True` and using the ### User Authentication !!! info - When using Windows Server 2012, `AUTH_LDAP_USER_DN_TEMPLATE` should be set to None. + When using Windows Server 2012+, `AUTH_LDAP_USER_DN_TEMPLATE` should be set to None. ```python from django_auth_ldap.config import LDAPSearch diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 953f42d5c..736807e2c 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -1,14 +1,31 @@ # NetBox v2.11 -## v2.11.10 (FUTURE) +## v2.11.10 (2021-07-28) + +### Enhancements + +* [#6560](https://github.com/netbox-community/netbox/issues/6560) - Enable CSV import via uploaded file +* [#6644](https://github.com/netbox-community/netbox/issues/6644) - Add 6P/4P pass-through port types +* [#6771](https://github.com/netbox-community/netbox/issues/6771) - Add count of inventory items to manufacturer view +* [#6785](https://github.com/netbox-community/netbox/issues/6785) - Add "hardwired" type for power port types ### Bug Fixes * [#5442](https://github.com/netbox-community/netbox/issues/5442) - Fix assignment of permissions based on LDAP groups +* [#5627](https://github.com/netbox-community/netbox/issues/5627) - Fix filtering of interface connections list +* [#6759](https://github.com/netbox-community/netbox/issues/6759) - Fix assignment of parent interfaces for bulk import * [#6773](https://github.com/netbox-community/netbox/issues/6773) - Add missing `display` field to rack unit serializer +* [#6774](https://github.com/netbox-community/netbox/issues/6774) - Fix A/Z assignment when swapping circuit terminations * [#6777](https://github.com/netbox-community/netbox/issues/6777) - Fix default value validation for custom text fields * [#6778](https://github.com/netbox-community/netbox/issues/6778) - Rack reservation should display rack's location * [#6780](https://github.com/netbox-community/netbox/issues/6780) - Include rack location in navigation breadcrumbs +* [#6794](https://github.com/netbox-community/netbox/issues/6794) - Fix device name display on device status view +* [#6812](https://github.com/netbox-community/netbox/issues/6812) - Limit reported prefix utilization to 100% +* [#6822](https://github.com/netbox-community/netbox/issues/6822) - Use consistent maximum value for interface MTU + +### Other Changes + +* [#6781](https://github.com/netbox-community/netbox/issues/6781) - Database query caching is now disabled by default --- diff --git a/mkdocs.yml b/mkdocs.yml index 29c60caf2..2dcf24ba3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,12 +1,15 @@ site_name: NetBox Documentation site_dir: netbox/project-static/docs site_url: https://netbox.readthedocs.io/ +repo_name: netbox-community/netbox repo_url: https://github.com/netbox-community/netbox python: install: - requirements: docs/requirements.txt theme: name: material + icon: + repo: fontawesome/brands/github palette: - scheme: default toggle: @@ -26,6 +29,7 @@ extra_css: - extra.css markdown_extensions: - admonition + - attr_list - markdown_include.include: headingOffset: 1 - pymdownx.emoji: @@ -94,6 +98,7 @@ nav: - Getting Started: 'development/getting-started.md' - Style Guide: 'development/style-guide.md' - Models: 'development/models.md' + - Adding Models: 'development/adding-models.md' - Extending Models: 'development/extending-models.md' - Signals: 'development/signals.md' - Application Registry: 'development/application-registry.md' diff --git a/netbox/circuits/views.py b/netbox/circuits/views.py index b4bb0155e..dfbfe68a4 100644 --- a/netbox/circuits/views.py +++ b/netbox/circuits/views.py @@ -287,6 +287,10 @@ class CircuitSwapTerminations(generic.ObjectEditView): termination_z.save() termination_a.term_side = 'Z' termination_a.save() + circuit.refresh_from_db() + circuit.termination_a = termination_z + circuit.termination_z = termination_a + circuit.save() elif termination_a: termination_a.term_side = 'Z' termination_a.save() @@ -300,9 +304,6 @@ class CircuitSwapTerminations(generic.ObjectEditView): circuit.termination_z = None circuit.save() - print(f'term A: {circuit.termination_a}') - print(f'term Z: {circuit.termination_z}') - messages.success(request, f"Swapped terminations for circuit {circuit}.") return redirect('circuits:circuit', pk=circuit.pk) diff --git a/netbox/dcim/choices.py b/netbox/dcim/choices.py index c3e0a1a75..ede397dc9 100644 --- a/netbox/dcim/choices.py +++ b/netbox/dcim/choices.py @@ -341,6 +341,8 @@ class PowerPortTypeChoices(ChoiceSet): TYPE_DC = 'dc-terminal' # Proprietary TYPE_SAF_D_GRID = 'saf-d-grid' + # Other + TYPE_HARDWIRED = 'hardwired' CHOICES = ( ('IEC 60320', ( @@ -447,6 +449,9 @@ class PowerPortTypeChoices(ChoiceSet): ('Proprietary', ( (TYPE_SAF_D_GRID, 'Saf-D-Grid'), )), + ('Other', ( + (TYPE_HARDWIRED, 'Hardwired'), + )), ) @@ -917,6 +922,11 @@ class PortTypeChoices(ChoiceSet): TYPE_8P6C = '8p6c' TYPE_8P4C = '8p4c' TYPE_8P2C = '8p2c' + TYPE_6P6C = '6p6c' + TYPE_6P4C = '6p4c' + TYPE_6P2C = '6p2c' + TYPE_4P4C = '4p4c' + TYPE_4P2C = '4p2c' TYPE_GG45 = 'gg45' TYPE_TERA4P = 'tera-4p' TYPE_TERA2P = 'tera-2p' @@ -948,6 +958,11 @@ class PortTypeChoices(ChoiceSet): (TYPE_8P6C, '8P6C'), (TYPE_8P4C, '8P4C'), (TYPE_8P2C, '8P2C'), + (TYPE_6P6C, '6P6C'), + (TYPE_6P4C, '6P4C'), + (TYPE_6P2C, '6P2C'), + (TYPE_4P4C, '4P4C'), + (TYPE_4P2C, '4P2C'), (TYPE_GG45, 'GG45'), (TYPE_TERA4P, 'TERA 4P'), (TYPE_TERA2P, 'TERA 2P'), diff --git a/netbox/dcim/constants.py b/netbox/dcim/constants.py index 42ed7c7d0..2a4d368f4 100644 --- a/netbox/dcim/constants.py +++ b/netbox/dcim/constants.py @@ -29,7 +29,7 @@ REARPORT_POSITIONS_MAX = 1024 # INTERFACE_MTU_MIN = 1 -INTERFACE_MTU_MAX = 32767 # Max value of a signed 16-bit integer +INTERFACE_MTU_MAX = 65536 VIRTUAL_IFACE_TYPES = [ InterfaceTypeChoices.TYPE_VIRTUAL, diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 9ddc8ae80..0eb86035e 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -102,6 +102,12 @@ class InterfaceCommonForm(forms.Form): required=False, label='MAC address' ) + mtu = forms.IntegerField( + required=False, + min_value=INTERFACE_MTU_MIN, + max_value=INTERFACE_MTU_MAX, + label='MTU' + ) def clean(self): super().clean() @@ -3224,12 +3230,6 @@ class InterfaceCreateForm(ComponentCreateForm, InterfaceCommonForm): 'type': 'lag', } ) - mtu = forms.IntegerField( - required=False, - min_value=INTERFACE_MTU_MIN, - max_value=INTERFACE_MTU_MAX, - label='MTU' - ) mac_address = forms.CharField( required=False, label='MAC Address' @@ -3432,13 +3432,18 @@ class InterfaceCSVForm(CustomFieldModelCSVForm): Q(device=device) | Q(device__virtual_chassis=device.virtual_chassis), type=InterfaceTypeChoices.TYPE_LAG ) + self.fields['parent'].queryset = Interface.objects.filter( + Q(device=device) | Q(device__virtual_chassis=device.virtual_chassis) + ) elif device: self.fields['lag'].queryset = Interface.objects.filter( device=device, type=InterfaceTypeChoices.TYPE_LAG ) + self.fields['parent'].queryset = Interface.objects.filter(device=device) else: self.fields['lag'].queryset = Interface.objects.none() + self.fields['parent'].queryset = Interface.objects.none() def clean_enabled(self): # Make sure enabled is True when it's not included in the uploaded data diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index ff42a6e80..bd1100870 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -435,7 +435,10 @@ class BaseInterface(models.Model): mtu = models.PositiveIntegerField( blank=True, null=True, - validators=[MinValueValidator(1), MaxValueValidator(65536)], + validators=[ + MinValueValidator(INTERFACE_MTU_MIN), + MaxValueValidator(INTERFACE_MTU_MAX) + ], verbose_name='MTU' ) mode = models.CharField( diff --git a/netbox/dcim/tests/test_views.py b/netbox/dcim/tests/test_views.py index ca6418e58..15f5a9a9a 100644 --- a/netbox/dcim/tests/test_views.py +++ b/netbox/dcim/tests/test_views.py @@ -1469,7 +1469,7 @@ class InterfaceTestCase(ViewTestCases.DeviceComponentViewTestCase): 'enabled': False, 'lag': interfaces[3].pk, 'mac_address': EUI('01:02:03:04:05:06'), - 'mtu': 2000, + 'mtu': 65000, 'mgmt_only': True, 'description': 'A front port', 'mode': InterfaceModeChoices.MODE_TAGGED, diff --git a/netbox/dcim/views.py b/netbox/dcim/views.py index d38d95aee..a489fdbfa 100644 --- a/netbox/dcim/views.py +++ b/netbox/dcim/views.py @@ -696,6 +696,9 @@ class ManufacturerView(generic.ObjectView): ).annotate( instance_count=count_related(Device, 'device_type') ) + inventory_items = InventoryItem.objects.restrict(request.user, 'view').filter( + manufacturer=instance + ) devicetypes_table = tables.DeviceTypeTable(devicetypes) devicetypes_table.columns.hide('manufacturer') @@ -703,6 +706,7 @@ class ManufacturerView(generic.ObjectView): return { 'devicetypes_table': devicetypes_table, + 'inventory_item_count': inventory_items.count(), } @@ -2558,11 +2562,7 @@ class PowerConnectionsListView(generic.ObjectListView): class InterfaceConnectionsListView(generic.ObjectListView): - queryset = Interface.objects.filter( - # Avoid duplicate connections by only selecting the lower PK in a connected pair - _path__isnull=False, - pk__lt=F('_path__destination_id') - ).order_by('device') + queryset = Interface.objects.filter(_path__isnull=False).order_by('device') filterset = filtersets.InterfaceConnectionFilterSet filterset_form = forms.InterfaceConnectionFilterForm table = tables.InterfaceConnectionTable diff --git a/netbox/ipam/models/ip.py b/netbox/ipam/models/ip.py index 2ccdfbd98..2ab691c1b 100644 --- a/netbox/ipam/models/ip.py +++ b/netbox/ipam/models/ip.py @@ -163,7 +163,9 @@ class Aggregate(PrimaryModel): """ queryset = Prefix.objects.filter(prefix__net_contained_or_equal=str(self.prefix)) child_prefixes = netaddr.IPSet([p.prefix for p in queryset]) - return int(float(child_prefixes.size) / self.prefix.size * 100) + utilization = int(float(child_prefixes.size) / self.prefix.size * 100) + + return min(utilization, 100) @extras_features('custom_fields', 'custom_links', 'export_templates', 'webhooks') @@ -469,14 +471,16 @@ class Prefix(PrimaryModel): vrf=self.vrf ) child_prefixes = netaddr.IPSet([p.prefix for p in queryset]) - return int(float(child_prefixes.size) / self.prefix.size * 100) + utilization = int(float(child_prefixes.size) / self.prefix.size * 100) else: # Compile an IPSet to avoid counting duplicate IPs child_count = netaddr.IPSet([ip.address.ip for ip in self.get_child_ips()]).size prefix_size = self.prefix.size if self.prefix.version == 4 and self.prefix.prefixlen < 31 and not self.is_pool: prefix_size -= 2 - return int(float(child_count) / prefix_size * 100) + utilization = int(float(child_count) / prefix_size * 100) + + return min(utilization, 100) @extras_features('custom_fields', 'custom_links', 'export_templates', 'tags', 'webhooks') diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 84209ada1..24402fb95 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -16,7 +16,7 @@ from django.core.validators import URLValidator # Environment setup # -VERSION = '3.0-beta1' +VERSION = '3.0-dev' # Hostname HOSTNAME = platform.node() diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index 9a78062fb..d5d5198e0 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -20,7 +20,8 @@ from extras.models import ExportTemplate from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortTransaction, PermissionsViolation from utilities.forms import ( - BootstrapMixin, BulkRenameForm, ConfirmationForm, CSVDataField, ImportForm, TableConfigForm, restrict_form_fields, + BootstrapMixin, BulkRenameForm, ConfirmationForm, CSVDataField, CSVFileField, ImportForm, TableConfigForm, + restrict_form_fields, ) from utilities.permissions import get_permission_for_model from utilities.tables import paginate_table @@ -644,6 +645,22 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): from_form=self.model_form, widget=Textarea(attrs=self.widget_attrs) ) + csv_file = CSVFileField( + label="CSV file", + from_form=self.model_form, + required=False + ) + + def clean(self): + csv_rows = self.cleaned_data['csv'][1] + csv_file = self.files.get('csv_file') + + # Check that the user has not submitted both text data and a file + if csv_rows and csv_file: + raise ValidationError( + "Cannot process CSV text and file attachment simultaneously. Please choose only one import " + "method." + ) return ImportForm(*args, **kwargs) @@ -668,7 +685,7 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): def post(self, request): logger = logging.getLogger('netbox.views.BulkImportView') new_objs = [] - form = self._import_form(request.POST) + form = self._import_form(request.POST, request.FILES) if form.is_valid(): logger.debug("Form validation was successful") @@ -676,7 +693,10 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): try: # Iterate through CSV data and bind each row to a new model form instance. with transaction.atomic(): - headers, records = form.cleaned_data['csv'] + if request.FILES: + headers, records = form.cleaned_data['csv_file'] + else: + headers, records = form.cleaned_data['csv'] for row, data in enumerate(records, start=1): obj_form = self.model_form(data, headers=headers) restrict_form_fields(obj_form, request.user) diff --git a/netbox/templates/dcim/manufacturer.html b/netbox/templates/dcim/manufacturer.html index e814a778e..3c3699c91 100644 --- a/netbox/templates/dcim/manufacturer.html +++ b/netbox/templates/dcim/manufacturer.html @@ -25,6 +25,12 @@ {{ devicetypes_table.rows|length }} + + Inventory Items + + {{ inventory_item_count }} + + diff --git a/netbox/utilities/forms/fields.py b/netbox/utilities/forms/fields.py index ac7f5ee3e..b1db79ecc 100644 --- a/netbox/utilities/forms/fields.py +++ b/netbox/utilities/forms/fields.py @@ -18,7 +18,7 @@ from utilities.utils import content_type_name from utilities.validators import EnhancedURLValidator from . import widgets from .constants import * -from .utils import expand_alphanumeric_pattern, expand_ipaddress_pattern +from .utils import expand_alphanumeric_pattern, expand_ipaddress_pattern, parse_csv, validate_csv __all__ = ( 'ColorField', @@ -28,6 +28,7 @@ __all__ = ( 'CSVChoiceField', 'CSVContentTypeField', 'CSVDataField', + 'CSVFileField', 'CSVModelChoiceField', 'CSVMultipleContentTypeField', 'CSVTypedChoiceField', @@ -184,49 +185,54 @@ class CSVDataField(forms.CharField): 'in double quotes.' def to_python(self, value): - - records = [] reader = csv.reader(StringIO(value.strip())) - # Consume the first line of CSV data as column headers. Create a dictionary mapping each header to an optional - # "to" field specifying how the related object is being referenced. For example, importing a Device might use a - # `site.slug` header, to indicate the related site is being referenced by its slug. - headers = {} - for header in next(reader): - if '.' in header: - field, to_field = header.split('.', 1) - headers[field] = to_field - else: - headers[header] = None + return parse_csv(reader) - # Parse CSV rows into a list of dictionaries mapped from the column headers. - for i, row in enumerate(reader, start=1): - if len(row) != len(headers): - raise forms.ValidationError( - f"Row {i}: Expected {len(headers)} columns but found {len(row)}" - ) - row = [col.strip() for col in row] - record = dict(zip(headers.keys(), row)) - records.append(record) + def validate(self, value): + headers, records = value + validate_csv(headers, self.fields, self.required_fields) + + return value + + +class CSVFileField(forms.FileField): + """ + A FileField (rendered as a file input button) which accepts a file containing CSV-formatted data. It returns + data as a two-tuple: The first item is a dictionary of column headers, mapping field names to the attribute + by which they match a related object (where applicable). The second item is a list of dictionaries, each + representing a discrete row of CSV data. + + :param from_form: The form from which the field derives its validation rules. + """ + + def __init__(self, from_form, *args, **kwargs): + + form = from_form() + self.model = form.Meta.model + self.fields = form.fields + self.required_fields = [ + name for name, field in form.fields.items() if field.required + ] + + super().__init__(*args, **kwargs) + + def to_python(self, file): + if file is None: + return None + + csv_str = file.read().decode('utf-8').strip() + reader = csv.reader(csv_str.splitlines()) + headers, records = parse_csv(reader) return headers, records def validate(self, value): + if value is None: + return None + headers, records = value - - # Validate provided column headers - for field, to_field in headers.items(): - if field not in self.fields: - raise forms.ValidationError(f'Unexpected column header "{field}" found.') - if to_field and not hasattr(self.fields[field], 'to_field_name'): - raise forms.ValidationError(f'Column "{field}" is not a related object; cannot use dots') - if to_field and not hasattr(self.fields[field].queryset.model, to_field): - raise forms.ValidationError(f'Invalid related object attribute for column "{field}": {to_field}') - - # Validate required fields - for f in self.required_fields: - if f not in headers: - raise forms.ValidationError(f'Required column header "{f}" not found.') + validate_csv(headers, self.fields, self.required_fields) return value diff --git a/netbox/utilities/forms/utils.py b/netbox/utilities/forms/utils.py index dc001be1a..503a2e8a0 100644 --- a/netbox/utilities/forms/utils.py +++ b/netbox/utilities/forms/utils.py @@ -14,6 +14,8 @@ __all__ = ( 'parse_alphanumeric_range', 'parse_numeric_range', 'restrict_form_fields', + 'parse_csv', + 'validate_csv', ) @@ -134,3 +136,55 @@ def restrict_form_fields(form, user, action='view'): for field in form.fields.values(): if hasattr(field, 'queryset') and issubclass(field.queryset.__class__, RestrictedQuerySet): field.queryset = field.queryset.restrict(user, action) + + +def parse_csv(reader): + """ + Parse a csv_reader object into a headers dictionary and a list of records dictionaries. Raise an error + if the records are formatted incorrectly. Return headers and records as a tuple. + """ + records = [] + headers = {} + + # Consume the first line of CSV data as column headers. Create a dictionary mapping each header to an optional + # "to" field specifying how the related object is being referenced. For example, importing a Device might use a + # `site.slug` header, to indicate the related site is being referenced by its slug. + + for header in next(reader): + if '.' in header: + field, to_field = header.split('.', 1) + headers[field] = to_field + else: + headers[header] = None + + # Parse CSV rows into a list of dictionaries mapped from the column headers. + for i, row in enumerate(reader, start=1): + if len(row) != len(headers): + raise forms.ValidationError( + f"Row {i}: Expected {len(headers)} columns but found {len(row)}" + ) + row = [col.strip() for col in row] + record = dict(zip(headers.keys(), row)) + records.append(record) + + return headers, records + + +def validate_csv(headers, fields, required_fields): + """ + Validate that parsed csv data conforms to the object's available fields. Raise validation errors + if parsed csv data contains invalid headers or does not contain required headers. + """ + # Validate provided column headers + for field, to_field in headers.items(): + if field not in fields: + raise forms.ValidationError(f'Unexpected column header "{field}" found.') + if to_field and not hasattr(fields[field], 'to_field_name'): + raise forms.ValidationError(f'Column "{field}" is not a related object; cannot use dots') + if to_field and not hasattr(fields[field].queryset.model, to_field): + raise forms.ValidationError(f'Invalid related object attribute for column "{field}": {to_field}') + + # Validate required fields + for f in required_fields: + if f not in headers: + raise forms.ValidationError(f'Required column header "{f}" not found.') diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index 06af5828e..332bebe8e 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -687,12 +687,6 @@ class VMInterfaceCreateForm(BootstrapMixin, CustomFieldsMixin, InterfaceCommonFo 'virtual_machine_id': '$virtual_machine', } ) - mtu = forms.IntegerField( - required=False, - min_value=INTERFACE_MTU_MIN, - max_value=INTERFACE_MTU_MAX, - label='MTU' - ) mac_address = forms.CharField( required=False, label='MAC Address' diff --git a/netbox/virtualization/tests/test_views.py b/netbox/virtualization/tests/test_views.py index 56c9cf280..86be5159f 100644 --- a/netbox/virtualization/tests/test_views.py +++ b/netbox/virtualization/tests/test_views.py @@ -263,7 +263,7 @@ class VMInterfaceTestCase(ViewTestCases.DeviceComponentViewTestCase): 'name': 'Interface X', 'enabled': False, 'mac_address': EUI('01-02-03-04-05-06'), - 'mtu': 2000, + 'mtu': 65000, 'description': 'New description', 'mode': InterfaceModeChoices.MODE_TAGGED, 'untagged_vlan': vlans[0].pk,