From f1d5e28f13d3c8f7fe694860004b7b1eb169a717 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Fri, 10 Jan 2020 14:26:39 +0000 Subject: [PATCH 01/27] CSV import/export custom fields --- netbox/circuits/forms.py | 6 ++--- netbox/dcim/forms.py | 28 ++++++++++++------------ netbox/ipam/forms.py | 16 +++++++------- netbox/secrets/forms.py | 4 ++-- netbox/tenancy/forms.py | 4 ++-- netbox/utilities/templatetags/helpers.py | 3 +++ netbox/utilities/views.py | 17 ++++++++++++-- netbox/virtualization/forms.py | 8 +++---- 8 files changed, 51 insertions(+), 35 deletions(-) diff --git a/netbox/circuits/forms.py b/netbox/circuits/forms.py index 4a5c06a6e..c5decc9e7 100644 --- a/netbox/circuits/forms.py +++ b/netbox/circuits/forms.py @@ -46,7 +46,7 @@ class ProviderForm(BootstrapMixin, CustomFieldForm): } -class ProviderCSVForm(forms.ModelForm): +class ProviderCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -144,7 +144,7 @@ class CircuitTypeForm(BootstrapMixin, forms.ModelForm): ] -class CircuitTypeCSVForm(forms.ModelForm): +class CircuitTypeCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -187,7 +187,7 @@ class CircuitForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class CircuitCSVForm(forms.ModelForm): +class CircuitCSVForm(CustomFieldForm): provider = forms.ModelChoiceField( queryset=Provider.objects.all(), to_field_name='name', diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 6086491d0..5f10752bc 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -151,7 +151,7 @@ class RegionForm(BootstrapMixin, forms.ModelForm): } -class RegionCSVForm(forms.ModelForm): +class RegionCSVForm(CustomFieldForm): parent = forms.ModelChoiceField( queryset=Region.objects.all(), required=False, @@ -231,7 +231,7 @@ class SiteForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class SiteCSVForm(forms.ModelForm): +class SiteCSVForm(CustomFieldForm): status = CSVChoiceField( choices=SITE_STATUS_CHOICES, required=False, @@ -355,7 +355,7 @@ class RackGroupForm(BootstrapMixin, forms.ModelForm): } -class RackGroupCSVForm(forms.ModelForm): +class RackGroupCSVForm(CustomFieldForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', @@ -411,7 +411,7 @@ class RackRoleForm(BootstrapMixin, forms.ModelForm): ] -class RackRoleCSVForm(forms.ModelForm): +class RackRoleCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -472,7 +472,7 @@ class RackForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class RackCSVForm(forms.ModelForm): +class RackCSVForm(CustomFieldForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', @@ -852,7 +852,7 @@ class ManufacturerForm(BootstrapMixin, forms.ModelForm): ] -class ManufacturerCSVForm(forms.ModelForm): +class ManufacturerCSVForm(CustomFieldForm): class Meta: model = Manufacturer @@ -890,7 +890,7 @@ class DeviceTypeForm(BootstrapMixin, CustomFieldForm): } -class DeviceTypeCSVForm(forms.ModelForm): +class DeviceTypeCSVForm(CustomFieldForm): manufacturer = forms.ModelChoiceField( queryset=Manufacturer.objects.all(), required=True, @@ -1308,7 +1308,7 @@ class DeviceRoleForm(BootstrapMixin, forms.ModelForm): ] -class DeviceRoleCSVForm(forms.ModelForm): +class DeviceRoleCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -1342,7 +1342,7 @@ class PlatformForm(BootstrapMixin, forms.ModelForm): } -class PlatformCSVForm(forms.ModelForm): +class PlatformCSVForm(CustomFieldForm): slug = SlugField() manufacturer = forms.ModelChoiceField( queryset=Manufacturer.objects.all(), @@ -1564,7 +1564,7 @@ class DeviceForm(BootstrapMixin, TenancyForm, CustomFieldForm): self.initial['rack'] = self.instance.parent_bay.device.rack_id -class BaseDeviceCSVForm(forms.ModelForm): +class BaseDeviceCSVForm(CustomFieldForm): device_role = forms.ModelChoiceField( queryset=DeviceRole.objects.all(), to_field_name='name', @@ -2919,7 +2919,7 @@ class CableForm(BootstrapMixin, forms.ModelForm): ] -class CableCSVForm(forms.ModelForm): +class CableCSVForm(CustomFieldForm): # Termination A side_a_device = FlexibleModelChoiceField( @@ -3294,7 +3294,7 @@ class InventoryItemForm(BootstrapMixin, forms.ModelForm): } -class InventoryItemCSVForm(forms.ModelForm): +class InventoryItemCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -3623,7 +3623,7 @@ class PowerPanelForm(BootstrapMixin, forms.ModelForm): } -class PowerPanelCSVForm(forms.ModelForm): +class PowerPanelCSVForm(CustomFieldForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', @@ -3747,7 +3747,7 @@ class PowerFeedForm(BootstrapMixin, CustomFieldForm): self.initial['site'] = self.instance.power_panel.site -class PowerFeedCSVForm(forms.ModelForm): +class PowerFeedCSVForm(CustomFieldForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', diff --git a/netbox/ipam/forms.py b/netbox/ipam/forms.py index 413e72eaf..46346cf9c 100644 --- a/netbox/ipam/forms.py +++ b/netbox/ipam/forms.py @@ -48,7 +48,7 @@ class VRFForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class VRFCSVForm(forms.ModelForm): +class VRFCSVForm(CustomFieldForm): tenant = forms.ModelChoiceField( queryset=Tenant.objects.all(), required=False, @@ -118,7 +118,7 @@ class RIRForm(BootstrapMixin, forms.ModelForm): ] -class RIRCSVForm(forms.ModelForm): +class RIRCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -165,7 +165,7 @@ class AggregateForm(BootstrapMixin, CustomFieldForm): } -class AggregateCSVForm(forms.ModelForm): +class AggregateCSVForm(CustomFieldForm): rir = forms.ModelChoiceField( queryset=RIR.objects.all(), to_field_name='name', @@ -247,7 +247,7 @@ class RoleForm(BootstrapMixin, forms.ModelForm): ] -class RoleCSVForm(forms.ModelForm): +class RoleCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -340,7 +340,7 @@ class PrefixForm(BootstrapMixin, TenancyForm, CustomFieldForm): self.fields['vrf'].empty_label = 'Global' -class PrefixCSVForm(forms.ModelForm): +class PrefixCSVForm(CustomFieldForm): vrf = FlexibleModelChoiceField( queryset=VRF.objects.all(), to_field_name='rd', @@ -759,7 +759,7 @@ class IPAddressBulkAddForm(BootstrapMixin, TenancyForm, CustomFieldForm): self.fields['vrf'].empty_label = 'Global' -class IPAddressCSVForm(forms.ModelForm): +class IPAddressCSVForm(CustomFieldForm): vrf = FlexibleModelChoiceField( queryset=VRF.objects.all(), to_field_name='rd', @@ -1025,7 +1025,7 @@ class VLANGroupForm(BootstrapMixin, forms.ModelForm): } -class VLANGroupCSVForm(forms.ModelForm): +class VLANGroupCSVForm(CustomFieldForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), required=False, @@ -1122,7 +1122,7 @@ class VLANForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class VLANCSVForm(forms.ModelForm): +class VLANCSVForm(CustomFieldForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), required=False, diff --git a/netbox/secrets/forms.py b/netbox/secrets/forms.py index ed0f455c1..47fb27bbb 100644 --- a/netbox/secrets/forms.py +++ b/netbox/secrets/forms.py @@ -50,7 +50,7 @@ class SecretRoleForm(BootstrapMixin, forms.ModelForm): } -class SecretRoleCSVForm(forms.ModelForm): +class SecretRoleCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -113,7 +113,7 @@ class SecretForm(BootstrapMixin, CustomFieldForm): }) -class SecretCSVForm(forms.ModelForm): +class SecretCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', diff --git a/netbox/tenancy/forms.py b/netbox/tenancy/forms.py index f8aaa45e5..77f8305f3 100644 --- a/netbox/tenancy/forms.py +++ b/netbox/tenancy/forms.py @@ -23,7 +23,7 @@ class TenantGroupForm(BootstrapMixin, forms.ModelForm): ] -class TenantGroupCSVForm(forms.ModelForm): +class TenantGroupCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -57,7 +57,7 @@ class TenantForm(BootstrapMixin, CustomFieldForm): } -class TenantCSVForm(forms.ModelForm): +class TenantCSVForm(CustomFieldForm): slug = SlugField() group = forms.ModelChoiceField( queryset=TenantGroup.objects.all(), diff --git a/netbox/utilities/templatetags/helpers.py b/netbox/utilities/templatetags/helpers.py index 7b1e059a6..a13a5f2b0 100644 --- a/netbox/utilities/templatetags/helpers.py +++ b/netbox/utilities/templatetags/helpers.py @@ -149,6 +149,9 @@ def example_choices(field, arg=3): break if not value or not label: continue + # Handling for custom fields + if hasattr(label, 'value'): + label = label.value examples.append(label) return ', '.join(examples) or 'None' diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index 1aa358fba..96be35130 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -78,15 +78,28 @@ class ObjectListView(View): Export the queryset of objects as comma-separated value (CSV), using the model's to_csv() method. """ csv_data = [] + custom_fields = [] # Start with the column headers headers = ','.join(self.queryset.model.csv_headers) + + # Add custom field headers + content_type = ContentType.objects.get_for_model(self.queryset.model) + + for custom_field in CustomField.objects.filter(obj_type=content_type): + headers += ',cf_{}'.format(custom_field.name) + custom_fields.append(custom_field.name) + csv_data.append(headers) # Iterate through the queryset appending each object for obj in self.queryset: - data = csv_format(obj.to_csv()) - csv_data.append(data) + data = obj.to_csv() + + for custom_field in custom_fields: + data += (obj.cf.get(custom_field, ''),) + + csv_data.append(csv_format(data)) return csv_data diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index 427e676f6..90ba2e123 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -37,7 +37,7 @@ class ClusterTypeForm(BootstrapMixin, forms.ModelForm): ] -class ClusterTypeCSVForm(forms.ModelForm): +class ClusterTypeCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -62,7 +62,7 @@ class ClusterGroupForm(BootstrapMixin, forms.ModelForm): ] -class ClusterGroupCSVForm(forms.ModelForm): +class ClusterGroupCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -101,7 +101,7 @@ class ClusterForm(BootstrapMixin, CustomFieldForm): } -class ClusterCSVForm(forms.ModelForm): +class ClusterCSVForm(CustomFieldForm): type = forms.ModelChoiceField( queryset=ClusterType.objects.all(), to_field_name='name', @@ -416,7 +416,7 @@ class VirtualMachineForm(BootstrapMixin, TenancyForm, CustomFieldForm): self.fields['primary_ip6'].widget.attrs['readonly'] = True -class VirtualMachineCSVForm(forms.ModelForm): +class VirtualMachineCSVForm(CustomFieldForm): status = CSVChoiceField( choices=VM_STATUS_CHOICES, required=False, From 37322fc1006e0f267c592408c2f2445068617cfb Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Fri, 10 Jan 2020 14:58:15 +0000 Subject: [PATCH 02/27] Fixed import choice name --- netbox/extras/forms.py | 6 +++--- netbox/utilities/forms.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index 4f7f57fff..efb33905c 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -10,8 +10,8 @@ from dcim.models import DeviceRole, Platform, Region, Site from tenancy.models import Tenant, TenantGroup from utilities.forms import ( add_blank_choice, APISelectMultiple, BootstrapMixin, BulkEditForm, BulkEditNullBooleanSelect, ColorSelect, - CommentField, ContentTypeSelect, DatePicker, DateTimePicker, FilterChoiceField, LaxURLField, JSONField, - SlugField, StaticSelect2, BOOLEAN_WITH_BLANK_CHOICES, + CSVCustomFieldChoiceField, CommentField, ContentTypeSelect, DatePicker, DateTimePicker, FilterChoiceField, + LaxURLField, JSONField, SlugField, StaticSelect2, BOOLEAN_WITH_BLANK_CHOICES, ) from .constants import * from .models import ConfigContext, CustomField, CustomFieldValue, ImageAttachment, ObjectChange, Tag @@ -71,7 +71,7 @@ def get_custom_fields_for_model(content_type, filterable_only=False, bulk_edit=F default_choice = cf.choices.get(value=initial).pk except ObjectDoesNotExist: pass - field = forms.TypedChoiceField( + field = CSVCustomFieldChoiceField( choices=choices, coerce=int, required=cf.required, initial=default_choice, widget=StaticSelect2() ) diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index eeee719ae..a98f29a8e 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -469,6 +469,23 @@ class CSVChoiceField(forms.ChoiceField): return self.choice_values[value] +class CSVCustomFieldChoiceField(forms.TypedChoiceField): + """ + Invert the provided set of choices to take the human-friendly label as input, and return the database value. + """ + + def __init__(self, choices, *args, **kwargs): + super().__init__(choices=choices, *args, **kwargs) + self.choice_values = {str(label): value for value, label in unpack_grouped_choices(choices)} + + def clean(self, value): + if not value: + return None + if value in self.choice_values: + return self.choice_values[value] + return super().clean(value) + + class ExpandableNameField(forms.CharField): """ A field which allows for numeric range expansion From de1355e6bc6c42f10dd135c89478322a924073a4 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Fri, 10 Jan 2020 15:00:57 +0000 Subject: [PATCH 03/27] Changelog #568 --- docs/release-notes/version-2.6.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.6.md b/docs/release-notes/version-2.6.md index 52b682746..6f4019460 100644 --- a/docs/release-notes/version-2.6.md +++ b/docs/release-notes/version-2.6.md @@ -2,6 +2,7 @@ ## Enhancements +* [#568](https://github.com/netbox-community/netbox/issues/568) - Allow custom fields to be imported and exported using CSV * [#1982](https://github.com/netbox-community/netbox/issues/1982) - Improved NAPALM method documentation in Swagger * [#2050](https://github.com/netbox-community/netbox/issues/2050) - Preview image attachments when hovering the link * [#2113](https://github.com/netbox-community/netbox/issues/2113) - Allow NAPALM driver settings to be changed with request headers From a2d5aca1d9a9a3ff717c0eacda279464a6a9d7ea Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 16 Jan 2020 16:05:45 +0000 Subject: [PATCH 04/27] Moved changelog to v2.7 --- docs/release-notes/version-2.6.md | 8 -------- docs/release-notes/version-2.7.md | 1 + 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/docs/release-notes/version-2.6.md b/docs/release-notes/version-2.6.md index 51a397b3c..9fd258b0f 100644 --- a/docs/release-notes/version-2.6.md +++ b/docs/release-notes/version-2.6.md @@ -1,11 +1,3 @@ -# v2.6.13 (FUTURE) - -## Enhancements - -* [#568](https://github.com/netbox-community/netbox/issues/568) - Allow custom fields to be imported and exported using CSV - ---- - # v2.6.12 (2020-01-13) ## Enhancements diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index ac9d81e2c..800389228 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -221,6 +221,7 @@ PATCH) to maintain backward compatibility. This behavior will be discontinued be ## Enhancements * [#33](https://github.com/netbox-community/netbox/issues/33) - Add ability to clone objects (pre-populate form fields) +* [#568](https://github.com/netbox-community/netbox/issues/568) - Allow custom fields to be imported and exported using CSV * [#648](https://github.com/netbox-community/netbox/issues/648) - Pre-populate forms when selecting "create and add another" * [#792](https://github.com/netbox-community/netbox/issues/792) - Add power port and power outlet types * [#1865](https://github.com/netbox-community/netbox/issues/1865) - Add console port and console server port types From 9f68f8d1a615b210a8c33c76a68802a6c6bec1ea Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 16 Jan 2020 16:07:24 +0000 Subject: [PATCH 05/27] Update component CSV forms --- netbox/dcim/forms.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index a768039cf..9c3332ff3 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -2175,7 +2175,7 @@ class ConsolePortCreateForm(ComponentForm): ) -class ConsolePortCSVForm(forms.ModelForm): +class ConsolePortCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2267,7 +2267,7 @@ class ConsoleServerPortBulkDisconnectForm(ConfirmationForm): ) -class ConsoleServerPortCSVForm(forms.ModelForm): +class ConsoleServerPortCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2334,7 +2334,7 @@ class PowerPortCreateForm(ComponentForm): ) -class PowerPortCSVForm(forms.ModelForm): +class PowerPortCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2419,7 +2419,7 @@ class PowerOutletCreateForm(ComponentForm): self.fields['power_port'].queryset = PowerPort.objects.filter(device=self.parent) -class PowerOutletCSVForm(forms.ModelForm): +class PowerOutletCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2668,7 +2668,7 @@ class InterfaceCreateForm(InterfaceCommonForm, ComponentForm, forms.Form): self.fields['lag'].queryset = Interface.objects.none() -class InterfaceCSVForm(forms.ModelForm): +class InterfaceCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), required=False, @@ -2925,7 +2925,7 @@ class FrontPortCreateForm(ComponentForm): } -class FrontPortCSVForm(forms.ModelForm): +class FrontPortCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -3051,7 +3051,7 @@ class RearPortCreateForm(ComponentForm): ) -class RearPortCSVForm(forms.ModelForm): +class RearPortCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -3655,7 +3655,7 @@ class PopulateDeviceBayForm(BootstrapMixin, forms.Form): ).exclude(pk=device_bay.device.pk) -class DeviceBayCSVForm(forms.ModelForm): +class DeviceBayCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', From 9128435113e35bc6941a71fd9c4ee93d8e973f87 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 23 Jan 2020 17:03:14 +0000 Subject: [PATCH 06/27] Removed CustomFieldForm class from models without custom fields --- netbox/circuits/forms.py | 2 +- netbox/dcim/forms.py | 34 +++++++++++++++++----------------- netbox/ipam/forms.py | 6 +++--- netbox/secrets/forms.py | 2 +- netbox/tenancy/forms.py | 2 +- netbox/virtualization/forms.py | 4 ++-- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/netbox/circuits/forms.py b/netbox/circuits/forms.py index 5416a055f..d14403815 100644 --- a/netbox/circuits/forms.py +++ b/netbox/circuits/forms.py @@ -144,7 +144,7 @@ class CircuitTypeForm(BootstrapMixin, forms.ModelForm): ] -class CircuitTypeCSVForm(CustomFieldForm): +class CircuitTypeCSVForm(forms.ModelForm): slug = SlugField() class Meta: diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 75a0992ec..a5e8a782f 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -184,7 +184,7 @@ class RegionForm(BootstrapMixin, forms.ModelForm): } -class RegionCSVForm(CustomFieldForm): +class RegionCSVForm(forms.ModelForm): parent = forms.ModelChoiceField( queryset=Region.objects.all(), required=False, @@ -388,7 +388,7 @@ class RackGroupForm(BootstrapMixin, forms.ModelForm): } -class RackGroupCSVForm(CustomFieldForm): +class RackGroupCSVForm(forms.ModelForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', @@ -444,7 +444,7 @@ class RackRoleForm(BootstrapMixin, forms.ModelForm): ] -class RackRoleCSVForm(CustomFieldForm): +class RackRoleCSVForm(forms.ModelForm): slug = SlugField() class Meta: @@ -882,7 +882,7 @@ class ManufacturerForm(BootstrapMixin, forms.ModelForm): ] -class ManufacturerCSVForm(CustomFieldForm): +class ManufacturerCSVForm(forms.ModelForm): class Meta: model = Manufacturer @@ -1458,7 +1458,7 @@ class DeviceRoleForm(BootstrapMixin, forms.ModelForm): ] -class DeviceRoleCSVForm(CustomFieldForm): +class DeviceRoleCSVForm(forms.ModelForm): slug = SlugField() class Meta: @@ -1492,7 +1492,7 @@ class PlatformForm(BootstrapMixin, forms.ModelForm): } -class PlatformCSVForm(CustomFieldForm): +class PlatformCSVForm(forms.ModelForm): slug = SlugField() manufacturer = forms.ModelChoiceField( queryset=Manufacturer.objects.all(), @@ -2179,7 +2179,7 @@ class ConsolePortCreateForm(ComponentForm): ) -class ConsolePortCSVForm(CustomFieldForm): +class ConsolePortCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2271,7 +2271,7 @@ class ConsoleServerPortBulkDisconnectForm(ConfirmationForm): ) -class ConsoleServerPortCSVForm(CustomFieldForm): +class ConsoleServerPortCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2338,7 +2338,7 @@ class PowerPortCreateForm(ComponentForm): ) -class PowerPortCSVForm(CustomFieldForm): +class PowerPortCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2423,7 +2423,7 @@ class PowerOutletCreateForm(ComponentForm): self.fields['power_port'].queryset = PowerPort.objects.filter(device=self.parent) -class PowerOutletCSVForm(CustomFieldForm): +class PowerOutletCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2672,7 +2672,7 @@ class InterfaceCreateForm(InterfaceCommonForm, ComponentForm, forms.Form): self.fields['lag'].queryset = Interface.objects.none() -class InterfaceCSVForm(CustomFieldForm): +class InterfaceCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), required=False, @@ -2929,7 +2929,7 @@ class FrontPortCreateForm(ComponentForm): } -class FrontPortCSVForm(CustomFieldForm): +class FrontPortCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -3055,7 +3055,7 @@ class RearPortCreateForm(ComponentForm): ) -class RearPortCSVForm(CustomFieldForm): +class RearPortCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -3365,7 +3365,7 @@ class CableForm(BootstrapMixin, forms.ModelForm): ] -class CableCSVForm(CustomFieldForm): +class CableCSVForm(forms.ModelForm): # Termination A side_a_device = FlexibleModelChoiceField( @@ -3659,7 +3659,7 @@ class PopulateDeviceBayForm(BootstrapMixin, forms.Form): ).exclude(pk=device_bay.device.pk) -class DeviceBayCSVForm(CustomFieldForm): +class DeviceBayCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -3801,7 +3801,7 @@ class InventoryItemForm(BootstrapMixin, forms.ModelForm): } -class InventoryItemCSVForm(CustomFieldForm): +class InventoryItemCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -4130,7 +4130,7 @@ class PowerPanelForm(BootstrapMixin, forms.ModelForm): } -class PowerPanelCSVForm(CustomFieldForm): +class PowerPanelCSVForm(forms.ModelForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', diff --git a/netbox/ipam/forms.py b/netbox/ipam/forms.py index d2efc2bcc..3bafad51b 100644 --- a/netbox/ipam/forms.py +++ b/netbox/ipam/forms.py @@ -118,7 +118,7 @@ class RIRForm(BootstrapMixin, forms.ModelForm): ] -class RIRCSVForm(CustomFieldForm): +class RIRCSVForm(forms.ModelForm): slug = SlugField() class Meta: @@ -247,7 +247,7 @@ class RoleForm(BootstrapMixin, forms.ModelForm): ] -class RoleCSVForm(CustomFieldForm): +class RoleCSVForm(forms.ModelForm): slug = SlugField() class Meta: @@ -1026,7 +1026,7 @@ class VLANGroupForm(BootstrapMixin, forms.ModelForm): } -class VLANGroupCSVForm(CustomFieldForm): +class VLANGroupCSVForm(forms.ModelForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), required=False, diff --git a/netbox/secrets/forms.py b/netbox/secrets/forms.py index 0b2cb3cb4..bddb1c109 100644 --- a/netbox/secrets/forms.py +++ b/netbox/secrets/forms.py @@ -52,7 +52,7 @@ class SecretRoleForm(BootstrapMixin, forms.ModelForm): } -class SecretRoleCSVForm(CustomFieldForm): +class SecretRoleCSVForm(forms.ModelForm): slug = SlugField() class Meta: diff --git a/netbox/tenancy/forms.py b/netbox/tenancy/forms.py index 77f8305f3..22deae434 100644 --- a/netbox/tenancy/forms.py +++ b/netbox/tenancy/forms.py @@ -23,7 +23,7 @@ class TenantGroupForm(BootstrapMixin, forms.ModelForm): ] -class TenantGroupCSVForm(CustomFieldForm): +class TenantGroupCSVForm(forms.ModelForm): slug = SlugField() class Meta: diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index 457369851..27c22fa71 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -33,7 +33,7 @@ class ClusterTypeForm(BootstrapMixin, forms.ModelForm): ] -class ClusterTypeCSVForm(CustomFieldForm): +class ClusterTypeCSVForm(forms.ModelForm): slug = SlugField() class Meta: @@ -58,7 +58,7 @@ class ClusterGroupForm(BootstrapMixin, forms.ModelForm): ] -class ClusterGroupCSVForm(CustomFieldForm): +class ClusterGroupCSVForm(forms.ModelForm): slug = SlugField() class Meta: From 0ab19d723dc75975c1c5840d9d91abfbbaece950 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 23 Jan 2020 17:18:58 +0000 Subject: [PATCH 07/27] Moved the header join logic after the custom fields are added --- netbox/utilities/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index 67af4371f..4801d61a3 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -91,16 +91,16 @@ class ObjectListView(View): custom_fields = [] # Start with the column headers - headers = ','.join(self.queryset.model.csv_headers) + headers = self.queryset.model.csv_headers.copy() # Add custom field headers content_type = ContentType.objects.get_for_model(self.queryset.model) for custom_field in CustomField.objects.filter(obj_type=content_type): - headers += ',cf_{}'.format(custom_field.name) + headers.append(custom_field.name) custom_fields.append(custom_field.name) - csv_data.append(headers) + csv_data.append(','.join(headers)) # Iterate through the queryset appending each object for obj in self.queryset: From 0a5eecd0e3114043f4939a4e473c65eaf762b88a Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 23 Jan 2020 17:37:51 +0000 Subject: [PATCH 08/27] Explicitly use the value of the choice, instead of relying on __str__ --- netbox/extras/forms.py | 2 +- netbox/utilities/templatetags/helpers.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index 17ae6c907..6aa0e3552 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -61,7 +61,7 @@ def get_custom_fields_for_model(content_type, filterable_only=False, bulk_edit=F # Select elif cf.type == CustomFieldTypeChoices.TYPE_SELECT: - choices = [(cfc.pk, cfc) for cfc in cf.choices.all()] + choices = [(cfc.pk, cfc.value) for cfc in cf.choices.all()] if not cf.required or bulk_edit or filterable_only: choices = [(None, '---------')] + choices # Check for a default choice diff --git a/netbox/utilities/templatetags/helpers.py b/netbox/utilities/templatetags/helpers.py index 19a26b3c9..c4b3bb6ea 100644 --- a/netbox/utilities/templatetags/helpers.py +++ b/netbox/utilities/templatetags/helpers.py @@ -149,9 +149,6 @@ def example_choices(field, arg=3): break if not value or not label: continue - # Handling for custom fields - if hasattr(label, 'value'): - label = label.value examples.append(label) return ', '.join(examples) or 'None' From 8f86244b4fe0c38134022a90f80151f6e446fcd9 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 23 Jan 2020 18:54:37 +0000 Subject: [PATCH 09/27] Cleaned the CustomField choice field --- netbox/extras/forms.py | 4 ++-- netbox/utilities/forms.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index 6aa0e3552..314455b83 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -10,7 +10,7 @@ from dcim.models import DeviceRole, Platform, Region, Site from tenancy.models import Tenant, TenantGroup from utilities.forms import ( add_blank_choice, APISelectMultiple, BootstrapMixin, BulkEditForm, BulkEditNullBooleanSelect, ColorSelect, - CSVCustomFieldChoiceField, CommentField, ContentTypeSelect, DatePicker, DateTimePicker, FilterChoiceField, + CustomFieldChoiceField, CommentField, ContentTypeSelect, DatePicker, DateTimePicker, FilterChoiceField, LaxURLField, JSONField, SlugField, StaticSelect2, BOOLEAN_WITH_BLANK_CHOICES, ) from .choices import * @@ -71,7 +71,7 @@ def get_custom_fields_for_model(content_type, filterable_only=False, bulk_edit=F default_choice = cf.choices.get(value=initial).pk except ObjectDoesNotExist: pass - field = CSVCustomFieldChoiceField( + field = CustomFieldChoiceField( choices=choices, coerce=int, required=cf.required, initial=default_choice, widget=StaticSelect2() ) diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index 453561303..1dd2c06a7 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -442,18 +442,18 @@ class CSVChoiceField(forms.ChoiceField): return self.choice_values[value] -class CSVCustomFieldChoiceField(forms.TypedChoiceField): +class CustomFieldChoiceField(forms.TypedChoiceField): """ - Invert the provided set of choices to take the human-friendly label as input, and return the database value. + Accept human-friendly label as input, and return the database value. If the label is not matched, the normal, + value-based input is assumed. """ def __init__(self, choices, *args, **kwargs): super().__init__(choices=choices, *args, **kwargs) - self.choice_values = {str(label): value for value, label in unpack_grouped_choices(choices)} + self.choice_values = {label: value for value, label in unpack_grouped_choices(choices)} def clean(self, value): - if not value: - return None + # Check if the value is actually a label if value in self.choice_values: return self.choice_values[value] return super().clean(value) From bed08a7b07eccbb6797cfe7ee29dbb38a1a1976e Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 23 Jan 2020 20:26:21 +0000 Subject: [PATCH 10/27] Use model's `get_custom_fields` --- netbox/utilities/views.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index 4801d61a3..d900a8545 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -93,12 +93,11 @@ class ObjectListView(View): # Start with the column headers headers = self.queryset.model.csv_headers.copy() - # Add custom field headers - content_type = ContentType.objects.get_for_model(self.queryset.model) - - for custom_field in CustomField.objects.filter(obj_type=content_type): - headers.append(custom_field.name) - custom_fields.append(custom_field.name) + # Add custom field headers, if any + if hasattr(self.queryset.model, 'get_custom_fields'): + for custom_field in self.queryset.model().get_custom_fields(): + headers.append(custom_field.name) + custom_fields.append(custom_field.name) csv_data.append(','.join(headers)) From c22024b618b32928a98ee18398f10c7f9bfb27ca Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Fri, 24 Jan 2020 22:15:09 +0000 Subject: [PATCH 11/27] Added CSV import test --- netbox/extras/tests/test_customfields.py | 64 +++++++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index 192958840..7871e031e 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -1,14 +1,14 @@ from datetime import date from django.contrib.contenttypes.models import ContentType -from django.test import TestCase +from django.test import Client, TestCase from django.urls import reverse from rest_framework import status from dcim.models import Site from extras.choices import * from extras.models import CustomField, CustomFieldValue, CustomFieldChoice -from utilities.testing import APITestCase +from utilities.testing import APITestCase, create_test_user from virtualization.models import VirtualMachine @@ -364,3 +364,63 @@ class CustomFieldChoiceAPITest(APITestCase): self.assertEqual(self.cf_choice_1.pk, response.data[self.cf_1.name][self.cf_choice_1.value]) self.assertEqual(self.cf_choice_2.pk, response.data[self.cf_1.name][self.cf_choice_2.value]) self.assertEqual(self.cf_choice_3.pk, response.data[self.cf_2.name][self.cf_choice_3.value]) + + +class CustomFieldCSV(TestCase): + def setUp(self): + super().setUp() + + user = create_test_user( + permissions=[ + 'dcim.view_site', + 'dcim.add_site', + ] + ) + self.client = Client() + self.client.force_login(user) + + obj_type = ContentType.objects.get_for_model(Site) + + self.cf_text = CustomField.objects.create(name="text", type=CustomFieldTypeChoices.TYPE_TEXT) + self.cf_text.obj_type.set([obj_type]) + self.cf_text.save() + + self.cf_choice = CustomField.objects.create(name="choice", type=CustomFieldTypeChoices.TYPE_SELECT) + self.cf_choice.obj_type.set([obj_type]) + self.cf_choice.save() + + self.cf_choice_1 = CustomFieldChoice.objects.create(field=self.cf_choice, value="cf_field_1") + self.cf_choice_2 = CustomFieldChoice.objects.create(field=self.cf_choice, value="cf_field_2") + self.cf_choice_3 = CustomFieldChoice.objects.create(field=self.cf_choice, value="cf_field_3") + + def test_import(self): + """ + Import a site with custom fields + """ + csv_data = ( + "name,slug,cf_text,cf_choice", + "Site 1,site-1,something,cf_field_1", + ) + + response = self.client.post(reverse('dcim:site_import'), {'csv': '\n'.join(csv_data)}) + self.assertEqual(response.status_code, 200) + + site1_custom_fields = Site.objects.get(name='Site 1').get_custom_fields() + self.assertEqual(len(site1_custom_fields), 2) + self.assertEqual(site1_custom_fields[self.cf_text], 'something') + self.assertEqual(site1_custom_fields[self.cf_choice], self.cf_choice_1) + + + def test_import_invalid_choice(self): + """ + Import a site with an invalid choice + """ + csv_data = ( + "name,slug,cf_choice", + "Site 2,site-2,cf_field_4", + ) + + response = self.client.post(reverse('dcim:site_import'), {'csv': '\n'.join(csv_data)}) + self.assertEqual(response.status_code, 200) + + self.assertFalse(len(Site.objects.filter(name="Site 2")), 0) From 8ec0ad96bddbc98c724ce2072a96f0e5844cf30a Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Fri, 24 Jan 2020 22:20:41 +0000 Subject: [PATCH 12/27] Formatting --- netbox/extras/tests/test_customfields.py | 1 - 1 file changed, 1 deletion(-) diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index 7871e031e..005f049b5 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -410,7 +410,6 @@ class CustomFieldCSV(TestCase): self.assertEqual(site1_custom_fields[self.cf_text], 'something') self.assertEqual(site1_custom_fields[self.cf_choice], self.cf_choice_1) - def test_import_invalid_choice(self): """ Import a site with an invalid choice From bc7cf63958b9f625c2e05b5aaae2944d957df1bc Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Jan 2020 10:49:02 -0500 Subject: [PATCH 13/27] Rename and refactor CustomFieldForm --- netbox/circuits/forms.py | 10 +++++----- netbox/dcim/forms.py | 20 ++++++++++---------- netbox/extras/forms.py | 29 ++++++++++++++++------------- netbox/ipam/forms.py | 26 +++++++++++++------------- netbox/secrets/forms.py | 6 +++--- netbox/tenancy/forms.py | 6 +++--- netbox/virtualization/forms.py | 10 +++++----- 7 files changed, 55 insertions(+), 52 deletions(-) diff --git a/netbox/circuits/forms.py b/netbox/circuits/forms.py index 6a528e5d6..b97f531b5 100644 --- a/netbox/circuits/forms.py +++ b/netbox/circuits/forms.py @@ -2,7 +2,7 @@ from django import forms from taggit.forms import TagField from dcim.models import Region, Site -from extras.forms import AddRemoveTagsForm, CustomFieldForm, CustomFieldBulkEditForm, CustomFieldFilterForm +from extras.forms import AddRemoveTagsForm, CustomFieldModelForm, CustomFieldBulkEditForm, CustomFieldFilterForm from tenancy.forms import TenancyFilterForm, TenancyForm from tenancy.models import Tenant from utilities.forms import ( @@ -17,7 +17,7 @@ from .models import Circuit, CircuitTermination, CircuitType, Provider # Providers # -class ProviderForm(BootstrapMixin, CustomFieldForm): +class ProviderForm(BootstrapMixin, CustomFieldModelForm): slug = SlugField() comments = CommentField() tags = TagField( @@ -46,7 +46,7 @@ class ProviderForm(BootstrapMixin, CustomFieldForm): } -class ProviderCSVForm(CustomFieldForm): +class ProviderCSVForm(CustomFieldModelForm): slug = SlugField() class Meta: @@ -160,7 +160,7 @@ class CircuitTypeCSVForm(forms.ModelForm): # Circuits # -class CircuitForm(BootstrapMixin, TenancyForm, CustomFieldForm): +class CircuitForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): comments = CommentField() tags = TagField( required=False @@ -188,7 +188,7 @@ class CircuitForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class CircuitCSVForm(CustomFieldForm): +class CircuitCSVForm(CustomFieldModelForm): provider = forms.ModelChoiceField( queryset=Provider.objects.all(), to_field_name='name', diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 18779217a..216af8b63 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -13,7 +13,7 @@ from timezone_field import TimeZoneFormField from circuits.models import Circuit, Provider from extras.forms import ( - AddRemoveTagsForm, CustomFieldForm, CustomFieldBulkEditForm, CustomFieldFilterForm, LocalConfigContextFilterForm + AddRemoveTagsForm, CustomFieldModelForm, CustomFieldBulkEditForm, CustomFieldFilterForm, LocalConfigContextFilterForm ) from ipam.constants import BGP_ASN_MAX, BGP_ASN_MIN from ipam.models import IPAddress, VLAN @@ -215,7 +215,7 @@ class RegionFilterForm(BootstrapMixin, forms.Form): # Sites # -class SiteForm(BootstrapMixin, TenancyForm, CustomFieldForm): +class SiteForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): region = TreeNodeChoiceField( queryset=Region.objects.all(), required=False, @@ -263,7 +263,7 @@ class SiteForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class SiteCSVForm(CustomFieldForm): +class SiteCSVForm(CustomFieldModelForm): status = CSVChoiceField( choices=SiteStatusChoices, required=False, @@ -459,7 +459,7 @@ class RackRoleCSVForm(forms.ModelForm): # Racks # -class RackForm(BootstrapMixin, TenancyForm, CustomFieldForm): +class RackForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): group = ChainedModelChoiceField( queryset=RackGroup.objects.all(), chains=( @@ -504,7 +504,7 @@ class RackForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class RackCSVForm(CustomFieldForm): +class RackCSVForm(CustomFieldModelForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', @@ -897,7 +897,7 @@ class ManufacturerCSVForm(forms.ModelForm): # Device types # -class DeviceTypeForm(BootstrapMixin, CustomFieldForm): +class DeviceTypeForm(BootstrapMixin, CustomFieldModelForm): slug = SlugField( slug_source='model' ) @@ -1516,7 +1516,7 @@ class PlatformCSVForm(forms.ModelForm): # Devices # -class DeviceForm(BootstrapMixin, TenancyForm, CustomFieldForm): +class DeviceForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), widget=APISelect( @@ -1724,7 +1724,7 @@ class DeviceForm(BootstrapMixin, TenancyForm, CustomFieldForm): self.initial['rack'] = self.instance.parent_bay.device.rack_id -class BaseDeviceCSVForm(CustomFieldForm): +class BaseDeviceCSVForm(CustomFieldModelForm): device_role = forms.ModelChoiceField( queryset=DeviceRole.objects.all(), to_field_name='name', @@ -4241,7 +4241,7 @@ class PowerPanelFilterForm(BootstrapMixin, CustomFieldFilterForm): # Power feeds # -class PowerFeedForm(BootstrapMixin, CustomFieldForm): +class PowerFeedForm(BootstrapMixin, CustomFieldModelForm): site = ChainedModelChoiceField( queryset=Site.objects.all(), required=False, @@ -4286,7 +4286,7 @@ class PowerFeedForm(BootstrapMixin, CustomFieldForm): self.initial['site'] = self.instance.power_panel.site -class PowerFeedCSVForm(CustomFieldForm): +class PowerFeedCSVForm(CustomFieldModelForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index 314455b83..bb1015638 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -10,8 +10,8 @@ from dcim.models import DeviceRole, Platform, Region, Site from tenancy.models import Tenant, TenantGroup from utilities.forms import ( add_blank_choice, APISelectMultiple, BootstrapMixin, BulkEditForm, BulkEditNullBooleanSelect, ColorSelect, - CustomFieldChoiceField, CommentField, ContentTypeSelect, DatePicker, DateTimePicker, FilterChoiceField, - LaxURLField, JSONField, SlugField, StaticSelect2, BOOLEAN_WITH_BLANK_CHOICES, + CommentField, ContentTypeSelect, DatePicker, DateTimePicker, FilterChoiceField, LaxURLField, JSONField, SlugField, + StaticSelect2, BOOLEAN_WITH_BLANK_CHOICES, ) from .choices import * from .models import ConfigContext, CustomField, CustomFieldValue, ImageAttachment, ObjectChange, Tag @@ -71,7 +71,7 @@ def get_custom_fields_for_model(content_type, filterable_only=False, bulk_edit=F default_choice = cf.choices.get(value=initial).pk except ObjectDoesNotExist: pass - field = CustomFieldChoiceField( + field = forms.TypedChoiceField( choices=choices, coerce=int, required=cf.required, initial=default_choice, widget=StaticSelect2() ) @@ -93,21 +93,15 @@ def get_custom_fields_for_model(content_type, filterable_only=False, bulk_edit=F return field_dict -class CustomFieldForm(forms.ModelForm): +class CustomFieldModelForm(forms.ModelForm): def __init__(self, *args, **kwargs): - self.custom_fields = [] - self.obj_type = ContentType.objects.get_for_model(self._meta.model) - super().__init__(*args, **kwargs) - # Add all applicable CustomFields to the form - custom_fields = [] - for name, field in get_custom_fields_for_model(self.obj_type).items(): - self.fields[name] = field - custom_fields.append(name) - self.custom_fields = custom_fields + # Append form fields for CustomFields + self.obj_type = ContentType.objects.get_for_model(self._meta.model) + self._append_customfield_fields() # If editing an existing object, initialize values for all custom fields if self.instance.pk: @@ -118,6 +112,15 @@ class CustomFieldForm(forms.ModelForm): for cfv in existing_values: self.initial['cf_{}'.format(str(cfv.field.name))] = cfv.serialized_value + def _append_customfield_fields(self): + """ + Append form fields for all applicable CustomFields. + """ + self.custom_fields = [] + for name, field in get_custom_fields_for_model(self.obj_type).items(): + self.fields[name] = field + self.custom_fields.append(name) + def _save_custom_fields(self): for field_name in self.custom_fields: diff --git a/netbox/ipam/forms.py b/netbox/ipam/forms.py index 3dda6aba5..594e95f58 100644 --- a/netbox/ipam/forms.py +++ b/netbox/ipam/forms.py @@ -4,7 +4,7 @@ from django.core.validators import MaxValueValidator, MinValueValidator from taggit.forms import TagField from dcim.models import Device, Interface, Rack, Region, Site -from extras.forms import AddRemoveTagsForm, CustomFieldForm, CustomFieldBulkEditForm, CustomFieldFilterForm +from extras.forms import AddRemoveTagsForm, CustomFieldModelForm, CustomFieldBulkEditForm, CustomFieldFilterForm from tenancy.forms import TenancyFilterForm, TenancyForm from tenancy.models import Tenant from utilities.forms import ( @@ -31,7 +31,7 @@ IPADDRESS_MASK_LENGTH_CHOICES = add_blank_choice([ # VRFs # -class VRFForm(BootstrapMixin, TenancyForm, CustomFieldForm): +class VRFForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): tags = TagField( required=False ) @@ -49,7 +49,7 @@ class VRFForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class VRFCSVForm(CustomFieldForm): +class VRFCSVForm(CustomFieldModelForm): tenant = forms.ModelChoiceField( queryset=Tenant.objects.all(), required=False, @@ -144,7 +144,7 @@ class RIRFilterForm(BootstrapMixin, forms.Form): # Aggregates # -class AggregateForm(BootstrapMixin, CustomFieldForm): +class AggregateForm(BootstrapMixin, CustomFieldModelForm): tags = TagField( required=False ) @@ -166,7 +166,7 @@ class AggregateForm(BootstrapMixin, CustomFieldForm): } -class AggregateCSVForm(CustomFieldForm): +class AggregateCSVForm(CustomFieldModelForm): rir = forms.ModelChoiceField( queryset=RIR.objects.all(), to_field_name='name', @@ -263,7 +263,7 @@ class RoleCSVForm(forms.ModelForm): # Prefixes # -class PrefixForm(BootstrapMixin, TenancyForm, CustomFieldForm): +class PrefixForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), required=False, @@ -341,7 +341,7 @@ class PrefixForm(BootstrapMixin, TenancyForm, CustomFieldForm): self.fields['vrf'].empty_label = 'Global' -class PrefixCSVForm(CustomFieldForm): +class PrefixCSVForm(CustomFieldModelForm): vrf = FlexibleModelChoiceField( queryset=VRF.objects.all(), to_field_name='rd', @@ -584,7 +584,7 @@ class PrefixFilterForm(BootstrapMixin, TenancyFilterForm, CustomFieldFilterForm) # IP addresses # -class IPAddressForm(BootstrapMixin, TenancyForm, ReturnURLForm, CustomFieldForm): +class IPAddressForm(BootstrapMixin, TenancyForm, ReturnURLForm, CustomFieldModelForm): interface = forms.ModelChoiceField( queryset=Interface.objects.all(), required=False @@ -751,7 +751,7 @@ class IPAddressBulkCreateForm(BootstrapMixin, forms.Form): ) -class IPAddressBulkAddForm(BootstrapMixin, TenancyForm, CustomFieldForm): +class IPAddressBulkAddForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): class Meta: model = IPAddress @@ -771,7 +771,7 @@ class IPAddressBulkAddForm(BootstrapMixin, TenancyForm, CustomFieldForm): self.fields['vrf'].empty_label = 'Global' -class IPAddressCSVForm(CustomFieldForm): +class IPAddressCSVForm(CustomFieldModelForm): vrf = FlexibleModelChoiceField( queryset=VRF.objects.all(), to_field_name='rd', @@ -1087,7 +1087,7 @@ class VLANGroupFilterForm(BootstrapMixin, forms.Form): # VLANs # -class VLANForm(BootstrapMixin, TenancyForm, CustomFieldForm): +class VLANForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), required=False, @@ -1135,7 +1135,7 @@ class VLANForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class VLANCSVForm(CustomFieldForm): +class VLANCSVForm(CustomFieldModelForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), required=False, @@ -1310,7 +1310,7 @@ class VLANFilterForm(BootstrapMixin, TenancyFilterForm, CustomFieldFilterForm): # Services # -class ServiceForm(BootstrapMixin, CustomFieldForm): +class ServiceForm(BootstrapMixin, CustomFieldModelForm): port = forms.IntegerField( min_value=SERVICE_PORT_MIN, max_value=SERVICE_PORT_MAX diff --git a/netbox/secrets/forms.py b/netbox/secrets/forms.py index ace977b3a..e8300a4cc 100644 --- a/netbox/secrets/forms.py +++ b/netbox/secrets/forms.py @@ -4,7 +4,7 @@ from django import forms from taggit.forms import TagField from dcim.models import Device -from extras.forms import AddRemoveTagsForm, CustomFieldBulkEditForm, CustomFieldFilterForm, CustomFieldForm +from extras.forms import AddRemoveTagsForm, CustomFieldBulkEditForm, CustomFieldFilterForm, CustomFieldModelForm from utilities.forms import ( APISelect, APISelectMultiple, BootstrapMixin, FilterChoiceField, FlexibleModelChoiceField, SlugField, StaticSelect2Multiple @@ -68,7 +68,7 @@ class SecretRoleCSVForm(forms.ModelForm): # Secrets # -class SecretForm(BootstrapMixin, CustomFieldForm): +class SecretForm(BootstrapMixin, CustomFieldModelForm): plaintext = forms.CharField( max_length=SECRET_PLAINTEXT_MAX_LENGTH, required=False, @@ -116,7 +116,7 @@ class SecretForm(BootstrapMixin, CustomFieldForm): }) -class SecretCSVForm(CustomFieldForm): +class SecretCSVForm(CustomFieldModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', diff --git a/netbox/tenancy/forms.py b/netbox/tenancy/forms.py index 22deae434..c713f37f5 100644 --- a/netbox/tenancy/forms.py +++ b/netbox/tenancy/forms.py @@ -1,7 +1,7 @@ from django import forms from taggit.forms import TagField -from extras.forms import AddRemoveTagsForm, CustomFieldForm, CustomFieldBulkEditForm, CustomFieldFilterForm +from extras.forms import AddRemoveTagsForm, CustomFieldModelForm, CustomFieldBulkEditForm, CustomFieldFilterForm from utilities.forms import ( APISelect, APISelectMultiple, BootstrapMixin, ChainedFieldsMixin, ChainedModelChoiceField, CommentField, FilterChoiceField, SlugField, @@ -38,7 +38,7 @@ class TenantGroupCSVForm(forms.ModelForm): # Tenants # -class TenantForm(BootstrapMixin, CustomFieldForm): +class TenantForm(BootstrapMixin, CustomFieldModelForm): slug = SlugField() comments = CommentField() tags = TagField( @@ -57,7 +57,7 @@ class TenantForm(BootstrapMixin, CustomFieldForm): } -class TenantCSVForm(CustomFieldForm): +class TenantCSVForm(CustomFieldModelForm): slug = SlugField() group = forms.ModelChoiceField( queryset=TenantGroup.objects.all(), diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index a6bda262f..fb6a633b8 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -6,7 +6,7 @@ from dcim.choices import InterfaceModeChoices from dcim.constants import INTERFACE_MTU_MAX, INTERFACE_MTU_MIN from dcim.forms import INTERFACE_MODE_HELP_TEXT from dcim.models import Device, DeviceRole, Interface, Platform, Rack, Region, Site -from extras.forms import AddRemoveTagsForm, CustomFieldBulkEditForm, CustomFieldForm, CustomFieldFilterForm +from extras.forms import AddRemoveTagsForm, CustomFieldBulkEditForm, CustomFieldModelForm, CustomFieldFilterForm from ipam.models import IPAddress, VLANGroup, VLAN from tenancy.forms import TenancyFilterForm, TenancyForm from tenancy.models import Tenant @@ -74,7 +74,7 @@ class ClusterGroupCSVForm(forms.ModelForm): # Clusters # -class ClusterForm(BootstrapMixin, TenancyForm, CustomFieldForm): +class ClusterForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): comments = CommentField() tags = TagField( required=False @@ -98,7 +98,7 @@ class ClusterForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class ClusterCSVForm(CustomFieldForm): +class ClusterCSVForm(CustomFieldModelForm): type = forms.ModelChoiceField( queryset=ClusterType.objects.all(), to_field_name='name', @@ -327,7 +327,7 @@ class ClusterRemoveDevicesForm(ConfirmationForm): # Virtual Machines # -class VirtualMachineForm(BootstrapMixin, TenancyForm, CustomFieldForm): +class VirtualMachineForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): cluster_group = forms.ModelChoiceField( queryset=ClusterGroup.objects.all(), required=False, @@ -430,7 +430,7 @@ class VirtualMachineForm(BootstrapMixin, TenancyForm, CustomFieldForm): self.fields['primary_ip6'].widget.attrs['readonly'] = True -class VirtualMachineCSVForm(CustomFieldForm): +class VirtualMachineCSVForm(CustomFieldModelForm): status = CSVChoiceField( choices=VirtualMachineStatusChoices, required=False, From f12199dcb589faef4ee192f3c586a99c21a08420 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Jan 2020 11:00:03 -0500 Subject: [PATCH 14/27] Rename and simplify CustomFieldChoiceField --- netbox/utilities/forms.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index 1dd2c06a7..c175df3cd 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -442,21 +442,18 @@ class CSVChoiceField(forms.ChoiceField): return self.choice_values[value] -class CustomFieldChoiceField(forms.TypedChoiceField): +class CSVCustomFieldChoiceField(forms.TypedChoiceField): """ - Accept human-friendly label as input, and return the database value. If the label is not matched, the normal, - value-based input is assumed. + Invert the choice tuples: CSV import takes the human-friendly label as input rather than the database value """ + def __init__(self, *args, **kwargs): - def __init__(self, choices, *args, **kwargs): - super().__init__(choices=choices, *args, **kwargs) - self.choice_values = {label: value for value, label in unpack_grouped_choices(choices)} + if 'choices' in kwargs: + kwargs['choices'] = { + label: value for value, label in kwargs['choices'] + } - def clean(self, value): - # Check if the value is actually a label - if value in self.choice_values: - return self.choice_values[value] - return super().clean(value) + super().__init__(*args, **kwargs) class ExpandableNameField(forms.CharField): From 9929a05bfe2be621e8d9e53cae6eba322662f28b Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Jan 2020 11:00:46 -0500 Subject: [PATCH 15/27] Update release notes --- docs/release-notes/version-2.7.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index ada79c9fa..163a9c55c 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,5 +1,9 @@ # v2.7.4 (FUTURE) +## Enhancements + +* [#568](https://github.com/netbox-community/netbox/issues/568) - Allow custom fields to be imported and exported using CSV + ## Bug Fixes * [#4043](https://github.com/netbox-community/netbox/issues/4043) - Fix toggling of required fields in custom scripts @@ -10,7 +14,6 @@ ## Enhancements -* [#568](https://github.com/netbox-community/netbox/issues/568) - Allow custom fields to be imported and exported using CSV * [#3310](https://github.com/netbox-community/netbox/issues/3310) - Pre-select site/rack for B side when creating a new cable * [#3338](https://github.com/netbox-community/netbox/issues/3338) - Include circuit terminations in API representation of circuits * [#3509](https://github.com/netbox-community/netbox/issues/3509) - Add IP address variables for custom scripts From 585ea71d1a5deed877ea42ad63faa4e940013e36 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Jan 2020 11:25:51 -0500 Subject: [PATCH 16/27] Move form field generation logic to CustomField class --- netbox/extras/forms.py | 63 ++------------------------------------- netbox/extras/models.py | 66 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 60 deletions(-) diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index bb1015638..674c3ff4c 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -3,15 +3,14 @@ from collections import OrderedDict from django import forms from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType -from django.core.exceptions import ObjectDoesNotExist from taggit.forms import TagField from dcim.models import DeviceRole, Platform, Region, Site from tenancy.models import Tenant, TenantGroup from utilities.forms import ( add_blank_choice, APISelectMultiple, BootstrapMixin, BulkEditForm, BulkEditNullBooleanSelect, ColorSelect, - CommentField, ContentTypeSelect, DatePicker, DateTimePicker, FilterChoiceField, LaxURLField, JSONField, SlugField, - StaticSelect2, BOOLEAN_WITH_BLANK_CHOICES, + CommentField, ContentTypeSelect, DateTimePicker, FilterChoiceField, JSONField, SlugField, StaticSelect2, + BOOLEAN_WITH_BLANK_CHOICES, ) from .choices import * from .models import ConfigContext, CustomField, CustomFieldValue, ImageAttachment, ObjectChange, Tag @@ -32,63 +31,7 @@ def get_custom_fields_for_model(content_type, filterable_only=False, bulk_edit=F for cf in custom_fields: field_name = 'cf_{}'.format(str(cf.name)) - initial = cf.default if not bulk_edit else None - - # Integer - if cf.type == CustomFieldTypeChoices.TYPE_INTEGER: - field = forms.IntegerField(required=cf.required, initial=initial) - - # Boolean - elif cf.type == CustomFieldTypeChoices.TYPE_BOOLEAN: - choices = ( - (None, '---------'), - (1, 'True'), - (0, 'False'), - ) - if initial is not None and initial.lower() in ['true', 'yes', '1']: - initial = 1 - elif initial is not None and initial.lower() in ['false', 'no', '0']: - initial = 0 - else: - initial = None - field = forms.NullBooleanField( - required=cf.required, initial=initial, widget=StaticSelect2(choices=choices) - ) - - # Date - elif cf.type == CustomFieldTypeChoices.TYPE_DATE: - field = forms.DateField(required=cf.required, initial=initial, widget=DatePicker()) - - # Select - elif cf.type == CustomFieldTypeChoices.TYPE_SELECT: - choices = [(cfc.pk, cfc.value) for cfc in cf.choices.all()] - if not cf.required or bulk_edit or filterable_only: - choices = [(None, '---------')] + choices - # Check for a default choice - default_choice = None - if initial: - try: - default_choice = cf.choices.get(value=initial).pk - except ObjectDoesNotExist: - pass - field = forms.TypedChoiceField( - choices=choices, coerce=int, required=cf.required, initial=default_choice, widget=StaticSelect2() - ) - - # URL - elif cf.type == CustomFieldTypeChoices.TYPE_URL: - field = LaxURLField(required=cf.required, initial=initial) - - # Text - else: - field = forms.CharField(max_length=255, required=cf.required, initial=initial) - - field.model = cf - field.label = cf.label if cf.label else cf.name.replace('_', ' ').capitalize() - if cf.description: - field.help_text = cf.description - - field_dict[field_name] = field + field_dict[field_name] = cf.to_form_field(set_initial=not bulk_edit) return field_dict diff --git a/netbox/extras/models.py b/netbox/extras/models.py index a03494bb2..be8feda7b 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -1,6 +1,7 @@ from collections import OrderedDict from datetime import date +from django import forms from django.contrib.auth.models import User from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType @@ -14,6 +15,7 @@ from django.utils.text import slugify from taggit.models import TagBase, GenericTaggedItemBase from utilities.fields import ColorField +from utilities.forms import DatePicker, LaxURLField, StaticSelect2, add_blank_choice from utilities.utils import deepmerge, render_jinja2 from .choices import * from .constants import * @@ -280,6 +282,70 @@ class CustomField(models.Model): return self.choices.get(pk=int(serialized_value)) return serialized_value + def to_form_field(self, set_initial=True): + """ + Return a form field suitable for setting a CustomField's value for an object. + + set_initial: Set initial date for the field. This should be false when generating a field for bulk editing. + """ + initial = self.default if set_initial else None + + # Integer + if self.type == CustomFieldTypeChoices.TYPE_INTEGER: + field = forms.IntegerField(required=self.required, initial=initial) + + # Boolean + elif self.type == CustomFieldTypeChoices.TYPE_BOOLEAN: + choices = ( + (None, '---------'), + (1, 'True'), + (0, 'False'), + ) + if initial is not None and initial.lower() in ['true', 'yes', '1']: + initial = 1 + elif initial is not None and initial.lower() in ['false', 'no', '0']: + initial = 0 + else: + initial = None + field = forms.NullBooleanField( + required=self.required, initial=initial, widget=StaticSelect2(choices=choices) + ) + + # Date + elif self.type == CustomFieldTypeChoices.TYPE_DATE: + field = forms.DateField(required=self.required, initial=initial, widget=DatePicker()) + + # Select + elif self.type == CustomFieldTypeChoices.TYPE_SELECT: + choices = [(cfc.pk, cfc.value) for cfc in self.choices.all()] + # TODO: Accommodate bulk edit/filtering + # if not self.required or bulk_edit or filterable_only: + if not self.required: + choices = add_blank_choice(choices) + # Set the initial value to the PK of the default choice, if any + if set_initial: + default_choice = self.choices.filter(value=self.default).first() + if default_choice: + initial = default_choice.pk + field = forms.TypedChoiceField( + choices=choices, coerce=int, required=self.required, initial=initial, widget=StaticSelect2() + ) + + # URL + elif self.type == CustomFieldTypeChoices.TYPE_URL: + field = LaxURLField(required=self.required, initial=initial) + + # Text + else: + field = forms.CharField(max_length=255, required=self.required, initial=initial) + + field.model = self + field.label = self.label if self.label else self.name.replace('_', ' ').capitalize() + if self.description: + field.help_text = self.description + + return field + class CustomFieldValue(models.Model): field = models.ForeignKey( From c3f86456d669d989661f4e3451ad7eb4a862aa0a Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Jan 2020 12:12:47 -0500 Subject: [PATCH 17/27] Remove get_custom_fields_for_model() --- netbox/extras/forms.py | 45 ++++++++++++++--------------------------- netbox/extras/models.py | 20 +++++++++--------- 2 files changed, 26 insertions(+), 39 deletions(-) diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index 674c3ff4c..f5d331ac4 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -20,22 +20,6 @@ from .models import ConfigContext, CustomField, CustomFieldValue, ImageAttachmen # Custom fields # -def get_custom_fields_for_model(content_type, filterable_only=False, bulk_edit=False): - """ - Retrieve all CustomFields applicable to the given ContentType - """ - field_dict = OrderedDict() - custom_fields = CustomField.objects.filter(obj_type=content_type) - if filterable_only: - custom_fields = custom_fields.exclude(filter_logic=CustomFieldFilterLogicChoices.FILTER_DISABLED) - - for cf in custom_fields: - field_name = 'cf_{}'.format(str(cf.name)) - field_dict[field_name] = cf.to_form_field(set_initial=not bulk_edit) - - return field_dict - - class CustomFieldModelForm(forms.ModelForm): def __init__(self, *args, **kwargs): @@ -60,9 +44,10 @@ class CustomFieldModelForm(forms.ModelForm): Append form fields for all applicable CustomFields. """ self.custom_fields = [] - for name, field in get_custom_fields_for_model(self.obj_type).items(): - self.fields[name] = field - self.custom_fields.append(name) + custom_fields = CustomField.objects.filter(obj_type=self.obj_type) + for cf in custom_fields: + self.fields[cf.name] = cf.to_form_field() + self.custom_fields.append(cf.name) def _save_custom_fields(self): @@ -106,15 +91,14 @@ class CustomFieldBulkEditForm(BulkEditForm): self.obj_type = ContentType.objects.get_for_model(self.model) # Add all applicable CustomFields to the form - custom_fields = get_custom_fields_for_model(self.obj_type, bulk_edit=True).items() - for name, field in custom_fields: + custom_fields = CustomField.objects.filter(obj_type=self.obj_type) + for cf in custom_fields: # Annotate non-required custom fields as nullable - if not field.required: - self.nullable_fields.append(name) - field.required = False - self.fields[name] = field + if not cf.required: + self.nullable_fields.append(cf.name) + self.fields[cf.name] = cf.to_form_field(set_initial=False, enforce_required=False) # Annotate this as a custom field - self.custom_fields.append(name) + self.custom_fields.append(cf.name) class CustomFieldFilterForm(forms.Form): @@ -126,10 +110,11 @@ class CustomFieldFilterForm(forms.Form): super().__init__(*args, **kwargs) # Add all applicable CustomFields to the form - custom_fields = get_custom_fields_for_model(self.obj_type, filterable_only=True).items() - for name, field in custom_fields: - field.required = False - self.fields[name] = field + custom_fields = CustomField.objects.filter(obj_type=self.obj_type).exclude( + filter_logic=CustomFieldFilterLogicChoices.FILTER_DISABLED + ) + for cf in custom_fields: + self.fields[cf.name] = cf.to_form_field(set_initial=True, enforce_required=False) # diff --git a/netbox/extras/models.py b/netbox/extras/models.py index be8feda7b..db42fc845 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -282,17 +282,19 @@ class CustomField(models.Model): return self.choices.get(pk=int(serialized_value)) return serialized_value - def to_form_field(self, set_initial=True): + def to_form_field(self, set_initial=True, enforce_required=True): """ Return a form field suitable for setting a CustomField's value for an object. - set_initial: Set initial date for the field. This should be false when generating a field for bulk editing. + set_initial: Set initial date for the field. This should be False when generating a field for bulk editing. + enforce_required: Honor the value of CustomField.required. Set to False for filtering/bulk editing. """ initial = self.default if set_initial else None + required = self.required if enforce_required else False # Integer if self.type == CustomFieldTypeChoices.TYPE_INTEGER: - field = forms.IntegerField(required=self.required, initial=initial) + field = forms.IntegerField(required=required, initial=initial) # Boolean elif self.type == CustomFieldTypeChoices.TYPE_BOOLEAN: @@ -308,19 +310,19 @@ class CustomField(models.Model): else: initial = None field = forms.NullBooleanField( - required=self.required, initial=initial, widget=StaticSelect2(choices=choices) + required=required, initial=initial, widget=StaticSelect2(choices=choices) ) # Date elif self.type == CustomFieldTypeChoices.TYPE_DATE: - field = forms.DateField(required=self.required, initial=initial, widget=DatePicker()) + field = forms.DateField(required=required, initial=initial, widget=DatePicker()) # Select elif self.type == CustomFieldTypeChoices.TYPE_SELECT: choices = [(cfc.pk, cfc.value) for cfc in self.choices.all()] # TODO: Accommodate bulk edit/filtering # if not self.required or bulk_edit or filterable_only: - if not self.required: + if not required: choices = add_blank_choice(choices) # Set the initial value to the PK of the default choice, if any if set_initial: @@ -328,16 +330,16 @@ class CustomField(models.Model): if default_choice: initial = default_choice.pk field = forms.TypedChoiceField( - choices=choices, coerce=int, required=self.required, initial=initial, widget=StaticSelect2() + choices=choices, coerce=int, required=required, initial=initial, widget=StaticSelect2() ) # URL elif self.type == CustomFieldTypeChoices.TYPE_URL: - field = LaxURLField(required=self.required, initial=initial) + field = LaxURLField(required=required, initial=initial) # Text else: - field = forms.CharField(max_length=255, required=self.required, initial=initial) + field = forms.CharField(max_length=255, required=required, initial=initial) field.model = self field.label = self.label if self.label else self.name.replace('_', ' ').capitalize() From 35f2291edc43265864b6cf6aece3467d4ce4e120 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Jan 2020 13:31:36 -0500 Subject: [PATCH 18/27] Fix assignment of initial CustomField values when editing an object --- netbox/extras/forms.py | 43 +++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index f5d331ac4..0123fead9 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -1,5 +1,3 @@ -from collections import OrderedDict - from django import forms from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType @@ -24,30 +22,37 @@ class CustomFieldModelForm(forms.ModelForm): def __init__(self, *args, **kwargs): + self.obj_type = ContentType.objects.get_for_model(self._meta.model) + self.custom_fields = [] + self.custom_field_values = {} + super().__init__(*args, **kwargs) - # Append form fields for CustomFields - self.obj_type = ContentType.objects.get_for_model(self._meta.model) self._append_customfield_fields() - # If editing an existing object, initialize values for all custom fields - if self.instance.pk: - existing_values = CustomFieldValue.objects.filter( - obj_type=self.obj_type, - obj_id=self.instance.pk - ).prefetch_related('field') - for cfv in existing_values: - self.initial['cf_{}'.format(str(cfv.field.name))] = cfv.serialized_value - def _append_customfield_fields(self): """ - Append form fields for all applicable CustomFields. + Append form fields for all CustomFields assigned to this model. """ - self.custom_fields = [] - custom_fields = CustomField.objects.filter(obj_type=self.obj_type) - for cf in custom_fields: - self.fields[cf.name] = cf.to_form_field() - self.custom_fields.append(cf.name) + # Retrieve initial CustomField values for the instance + if self.instance.pk: + for cfv in CustomFieldValue.objects.filter( + obj_type=self.obj_type, + obj_id=self.instance.pk + ).prefetch_related('field'): + self.custom_field_values[cfv.field.name] = cfv.serialized_value + + # Append form fields; assign initial values if modifying and existing object + for cf in CustomField.objects.filter(obj_type=self.obj_type): + field_name = 'cf_{}'.format(cf.name) + if self.instance.pk: + self.fields[field_name] = cf.to_form_field(set_initial=False) + self.fields[field_name].initial = self.custom_field_values.get(cf.name) + else: + self.fields[field_name] = cf.to_form_field() + + # Annotate the field in the list of CustomField form fields + self.custom_fields.append(field_name) def _save_custom_fields(self): From e6b018909db8a2eb8ca650c439953547adee1b80 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Jan 2020 13:53:26 -0500 Subject: [PATCH 19/27] Introduced CustomFieldModelCSVForm --- netbox/circuits/forms.py | 8 +++++--- netbox/dcim/forms.py | 11 ++++++----- netbox/extras/forms.py | 13 +++++++++++++ netbox/ipam/forms.py | 14 ++++++++------ netbox/secrets/forms.py | 6 ++++-- netbox/tenancy/forms.py | 4 +++- netbox/virtualization/forms.py | 8 +++++--- 7 files changed, 44 insertions(+), 20 deletions(-) diff --git a/netbox/circuits/forms.py b/netbox/circuits/forms.py index b97f531b5..643359b74 100644 --- a/netbox/circuits/forms.py +++ b/netbox/circuits/forms.py @@ -2,7 +2,9 @@ from django import forms from taggit.forms import TagField from dcim.models import Region, Site -from extras.forms import AddRemoveTagsForm, CustomFieldModelForm, CustomFieldBulkEditForm, CustomFieldFilterForm +from extras.forms import ( + AddRemoveTagsForm, CustomFieldBulkEditForm, CustomFieldFilterForm, CustomFieldModelForm, CustomFieldModelCSVForm, +) from tenancy.forms import TenancyFilterForm, TenancyForm from tenancy.models import Tenant from utilities.forms import ( @@ -46,7 +48,7 @@ class ProviderForm(BootstrapMixin, CustomFieldModelForm): } -class ProviderCSVForm(CustomFieldModelForm): +class ProviderCSVForm(CustomFieldModelCSVForm): slug = SlugField() class Meta: @@ -188,7 +190,7 @@ class CircuitForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): } -class CircuitCSVForm(CustomFieldModelForm): +class CircuitCSVForm(CustomFieldModelCSVForm): provider = forms.ModelChoiceField( queryset=Provider.objects.all(), to_field_name='name', diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 216af8b63..f57f942e0 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -13,7 +13,8 @@ from timezone_field import TimeZoneFormField from circuits.models import Circuit, Provider from extras.forms import ( - AddRemoveTagsForm, CustomFieldModelForm, CustomFieldBulkEditForm, CustomFieldFilterForm, LocalConfigContextFilterForm + AddRemoveTagsForm, CustomFieldBulkEditForm, CustomFieldModelCSVForm, CustomFieldFilterForm, CustomFieldModelForm, + LocalConfigContextFilterForm, ) from ipam.constants import BGP_ASN_MAX, BGP_ASN_MIN from ipam.models import IPAddress, VLAN @@ -263,7 +264,7 @@ class SiteForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): } -class SiteCSVForm(CustomFieldModelForm): +class SiteCSVForm(CustomFieldModelCSVForm): status = CSVChoiceField( choices=SiteStatusChoices, required=False, @@ -504,7 +505,7 @@ class RackForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): } -class RackCSVForm(CustomFieldModelForm): +class RackCSVForm(CustomFieldModelCSVForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', @@ -1724,7 +1725,7 @@ class DeviceForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): self.initial['rack'] = self.instance.parent_bay.device.rack_id -class BaseDeviceCSVForm(CustomFieldModelForm): +class BaseDeviceCSVForm(CustomFieldModelCSVForm): device_role = forms.ModelChoiceField( queryset=DeviceRole.objects.all(), to_field_name='name', @@ -4286,7 +4287,7 @@ class PowerFeedForm(BootstrapMixin, CustomFieldModelForm): self.initial['site'] = self.instance.power_panel.site -class PowerFeedCSVForm(CustomFieldModelForm): +class PowerFeedCSVForm(CustomFieldModelCSVForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index 0123fead9..2c6b6cb65 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -87,6 +87,19 @@ class CustomFieldModelForm(forms.ModelForm): return obj +class CustomFieldModelCSVForm(CustomFieldModelForm): + + def _append_customfield_fields(self): + + # Append form fields + for cf in CustomField.objects.filter(obj_type=self.obj_type): + field_name = 'cf_{}'.format(cf.name) + self.fields[field_name] = cf.to_form_field() + + # Annotate the field in the list of CustomField form fields + self.custom_fields.append(field_name) + + class CustomFieldBulkEditForm(BulkEditForm): def __init__(self, *args, **kwargs): diff --git a/netbox/ipam/forms.py b/netbox/ipam/forms.py index 594e95f58..237ee2238 100644 --- a/netbox/ipam/forms.py +++ b/netbox/ipam/forms.py @@ -4,7 +4,9 @@ from django.core.validators import MaxValueValidator, MinValueValidator from taggit.forms import TagField from dcim.models import Device, Interface, Rack, Region, Site -from extras.forms import AddRemoveTagsForm, CustomFieldModelForm, CustomFieldBulkEditForm, CustomFieldFilterForm +from extras.forms import ( + AddRemoveTagsForm, CustomFieldBulkEditForm, CustomFieldModelCSVForm, CustomFieldModelForm, CustomFieldFilterForm, +) from tenancy.forms import TenancyFilterForm, TenancyForm from tenancy.models import Tenant from utilities.forms import ( @@ -49,7 +51,7 @@ class VRFForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): } -class VRFCSVForm(CustomFieldModelForm): +class VRFCSVForm(CustomFieldModelCSVForm): tenant = forms.ModelChoiceField( queryset=Tenant.objects.all(), required=False, @@ -166,7 +168,7 @@ class AggregateForm(BootstrapMixin, CustomFieldModelForm): } -class AggregateCSVForm(CustomFieldModelForm): +class AggregateCSVForm(CustomFieldModelCSVForm): rir = forms.ModelChoiceField( queryset=RIR.objects.all(), to_field_name='name', @@ -341,7 +343,7 @@ class PrefixForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): self.fields['vrf'].empty_label = 'Global' -class PrefixCSVForm(CustomFieldModelForm): +class PrefixCSVForm(CustomFieldModelCSVForm): vrf = FlexibleModelChoiceField( queryset=VRF.objects.all(), to_field_name='rd', @@ -771,7 +773,7 @@ class IPAddressBulkAddForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): self.fields['vrf'].empty_label = 'Global' -class IPAddressCSVForm(CustomFieldModelForm): +class IPAddressCSVForm(CustomFieldModelCSVForm): vrf = FlexibleModelChoiceField( queryset=VRF.objects.all(), to_field_name='rd', @@ -1135,7 +1137,7 @@ class VLANForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): } -class VLANCSVForm(CustomFieldModelForm): +class VLANCSVForm(CustomFieldModelCSVForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), required=False, diff --git a/netbox/secrets/forms.py b/netbox/secrets/forms.py index e8300a4cc..46b2c12f4 100644 --- a/netbox/secrets/forms.py +++ b/netbox/secrets/forms.py @@ -4,7 +4,9 @@ from django import forms from taggit.forms import TagField from dcim.models import Device -from extras.forms import AddRemoveTagsForm, CustomFieldBulkEditForm, CustomFieldFilterForm, CustomFieldModelForm +from extras.forms import ( + AddRemoveTagsForm, CustomFieldBulkEditForm, CustomFieldFilterForm, CustomFieldModelForm, CustomFieldModelCSVForm, +) from utilities.forms import ( APISelect, APISelectMultiple, BootstrapMixin, FilterChoiceField, FlexibleModelChoiceField, SlugField, StaticSelect2Multiple @@ -116,7 +118,7 @@ class SecretForm(BootstrapMixin, CustomFieldModelForm): }) -class SecretCSVForm(CustomFieldModelForm): +class SecretCSVForm(CustomFieldModelCSVForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', diff --git a/netbox/tenancy/forms.py b/netbox/tenancy/forms.py index c713f37f5..22a4809de 100644 --- a/netbox/tenancy/forms.py +++ b/netbox/tenancy/forms.py @@ -1,7 +1,9 @@ from django import forms from taggit.forms import TagField -from extras.forms import AddRemoveTagsForm, CustomFieldModelForm, CustomFieldBulkEditForm, CustomFieldFilterForm +from extras.forms import ( + AddRemoveTagsForm, CustomFieldModelForm, CustomFieldBulkEditForm, CustomFieldModelCSVForm, CustomFieldFilterForm, +) from utilities.forms import ( APISelect, APISelectMultiple, BootstrapMixin, ChainedFieldsMixin, ChainedModelChoiceField, CommentField, FilterChoiceField, SlugField, diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index fb6a633b8..066e162d3 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -6,7 +6,9 @@ from dcim.choices import InterfaceModeChoices from dcim.constants import INTERFACE_MTU_MAX, INTERFACE_MTU_MIN from dcim.forms import INTERFACE_MODE_HELP_TEXT from dcim.models import Device, DeviceRole, Interface, Platform, Rack, Region, Site -from extras.forms import AddRemoveTagsForm, CustomFieldBulkEditForm, CustomFieldModelForm, CustomFieldFilterForm +from extras.forms import ( + AddRemoveTagsForm, CustomFieldBulkEditForm, CustomFieldModelCSVForm, CustomFieldModelForm, CustomFieldFilterForm, +) from ipam.models import IPAddress, VLANGroup, VLAN from tenancy.forms import TenancyFilterForm, TenancyForm from tenancy.models import Tenant @@ -98,7 +100,7 @@ class ClusterForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): } -class ClusterCSVForm(CustomFieldModelForm): +class ClusterCSVForm(CustomFieldModelCSVForm): type = forms.ModelChoiceField( queryset=ClusterType.objects.all(), to_field_name='name', @@ -430,7 +432,7 @@ class VirtualMachineForm(BootstrapMixin, TenancyForm, CustomFieldModelForm): self.fields['primary_ip6'].widget.attrs['readonly'] = True -class VirtualMachineCSVForm(CustomFieldModelForm): +class VirtualMachineCSVForm(CustomFieldModelCSVForm): status = CSVChoiceField( choices=VirtualMachineStatusChoices, required=False, From 193435b554c59df0dcc188f769ee7f452d466960 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Jan 2020 14:29:47 -0500 Subject: [PATCH 20/27] Enable CSV import for custom fields --- netbox/extras/forms.py | 2 +- netbox/extras/models.py | 15 +++++++++------ netbox/utilities/forms.py | 14 -------------- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index 2c6b6cb65..751b2ddb0 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -94,7 +94,7 @@ class CustomFieldModelCSVForm(CustomFieldModelForm): # Append form fields for cf in CustomField.objects.filter(obj_type=self.obj_type): field_name = 'cf_{}'.format(cf.name) - self.fields[field_name] = cf.to_form_field() + self.fields[field_name] = cf.to_form_field(for_csv_import=True) # Annotate the field in the list of CustomField form fields self.custom_fields.append(field_name) diff --git a/netbox/extras/models.py b/netbox/extras/models.py index db42fc845..70f7661c8 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -15,7 +15,7 @@ from django.utils.text import slugify from taggit.models import TagBase, GenericTaggedItemBase from utilities.fields import ColorField -from utilities.forms import DatePicker, LaxURLField, StaticSelect2, add_blank_choice +from utilities.forms import CSVChoiceField, DatePicker, LaxURLField, StaticSelect2, add_blank_choice from utilities.utils import deepmerge, render_jinja2 from .choices import * from .constants import * @@ -282,12 +282,13 @@ class CustomField(models.Model): return self.choices.get(pk=int(serialized_value)) return serialized_value - def to_form_field(self, set_initial=True, enforce_required=True): + def to_form_field(self, set_initial=True, enforce_required=True, for_csv_import=False): """ Return a form field suitable for setting a CustomField's value for an object. set_initial: Set initial date for the field. This should be False when generating a field for bulk editing. enforce_required: Honor the value of CustomField.required. Set to False for filtering/bulk editing. + for_csv_import: Return a form field suitable for bulk import of objects in CSV format. """ initial = self.default if set_initial else None required = self.required if enforce_required else False @@ -320,17 +321,19 @@ class CustomField(models.Model): # Select elif self.type == CustomFieldTypeChoices.TYPE_SELECT: choices = [(cfc.pk, cfc.value) for cfc in self.choices.all()] - # TODO: Accommodate bulk edit/filtering - # if not self.required or bulk_edit or filterable_only: + if not required: choices = add_blank_choice(choices) + # Set the initial value to the PK of the default choice, if any if set_initial: default_choice = self.choices.filter(value=self.default).first() if default_choice: initial = default_choice.pk - field = forms.TypedChoiceField( - choices=choices, coerce=int, required=required, initial=initial, widget=StaticSelect2() + + field_class = CSVChoiceField if for_csv_import else forms.ChoiceField + field = field_class( + choices=choices, required=required, initial=initial, widget=StaticSelect2() ) # URL diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index c175df3cd..a14ec9305 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -442,20 +442,6 @@ class CSVChoiceField(forms.ChoiceField): return self.choice_values[value] -class CSVCustomFieldChoiceField(forms.TypedChoiceField): - """ - Invert the choice tuples: CSV import takes the human-friendly label as input rather than the database value - """ - def __init__(self, *args, **kwargs): - - if 'choices' in kwargs: - kwargs['choices'] = { - label: value for value, label in kwargs['choices'] - } - - super().__init__(*args, **kwargs) - - class ExpandableNameField(forms.CharField): """ A field which allows for numeric range expansion From c75315fda63a6b9809344bccea99b08ecfb3a365 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Jan 2020 15:34:55 -0500 Subject: [PATCH 21/27] Extend CSV import test --- netbox/extras/tests/test_customfields.py | 89 +++++++++++++++--------- 1 file changed, 55 insertions(+), 34 deletions(-) diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index 005f049b5..703b0b0d0 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -366,9 +366,9 @@ class CustomFieldChoiceAPITest(APITestCase): self.assertEqual(self.cf_choice_3.pk, response.data[self.cf_2.name][self.cf_choice_3.value]) -class CustomFieldCSV(TestCase): +class CustomFieldImportTest(TestCase): + def setUp(self): - super().setUp() user = create_test_user( permissions=[ @@ -379,47 +379,68 @@ class CustomFieldCSV(TestCase): self.client = Client() self.client.force_login(user) - obj_type = ContentType.objects.get_for_model(Site) + @classmethod + def setUpTestData(cls): - self.cf_text = CustomField.objects.create(name="text", type=CustomFieldTypeChoices.TYPE_TEXT) - self.cf_text.obj_type.set([obj_type]) - self.cf_text.save() + custom_fields = ( + CustomField(name='text', type=CustomFieldTypeChoices.TYPE_TEXT), + CustomField(name='integer', type=CustomFieldTypeChoices.TYPE_INTEGER), + CustomField(name='boolean', type=CustomFieldTypeChoices.TYPE_BOOLEAN), + CustomField(name='date', type=CustomFieldTypeChoices.TYPE_DATE), + CustomField(name='url', type=CustomFieldTypeChoices.TYPE_URL), + CustomField(name='select', type=CustomFieldTypeChoices.TYPE_SELECT), + ) + for cf in custom_fields: + cf.save() + cf.obj_type.set([ContentType.objects.get_for_model(Site)]) - self.cf_choice = CustomField.objects.create(name="choice", type=CustomFieldTypeChoices.TYPE_SELECT) - self.cf_choice.obj_type.set([obj_type]) - self.cf_choice.save() - - self.cf_choice_1 = CustomFieldChoice.objects.create(field=self.cf_choice, value="cf_field_1") - self.cf_choice_2 = CustomFieldChoice.objects.create(field=self.cf_choice, value="cf_field_2") - self.cf_choice_3 = CustomFieldChoice.objects.create(field=self.cf_choice, value="cf_field_3") + CustomFieldChoice.objects.bulk_create(( + CustomFieldChoice(field=custom_fields[5], value='Choice A'), + CustomFieldChoice(field=custom_fields[5], value='Choice B'), + CustomFieldChoice(field=custom_fields[5], value='Choice C'), + )) def test_import(self): """ - Import a site with custom fields + Import a Site in CSV format, including a value for each CustomField. """ - csv_data = ( - "name,slug,cf_text,cf_choice", - "Site 1,site-1,something,cf_field_1", + data = ( + ('name', 'slug', 'cf_text', 'cf_integer', 'cf_boolean', 'cf_date', 'cf_url', 'cf_select'), + ('Site 1', 'site-1', 'ABC', '123', 'True', '2020-01-01', 'http://example.com/1', 'Choice A'), + ('Site 2', 'site-2', 'DEF', '456', 'False', '2020-01-02', 'http://example.com/2', 'Choice B'), + ('Site 3', 'site-3', '', '', '', '', '', ''), ) + csv_data = '\n'.join(','.join(row) for row in data) - response = self.client.post(reverse('dcim:site_import'), {'csv': '\n'.join(csv_data)}) + response = self.client.post(reverse('dcim:site_import'), {'csv': csv_data}) self.assertEqual(response.status_code, 200) - site1_custom_fields = Site.objects.get(name='Site 1').get_custom_fields() - self.assertEqual(len(site1_custom_fields), 2) - self.assertEqual(site1_custom_fields[self.cf_text], 'something') - self.assertEqual(site1_custom_fields[self.cf_choice], self.cf_choice_1) + # Validate data for site 1 + custom_field_values = { + cf.name: value for cf, value in Site.objects.get(name='Site 1').get_custom_fields().items() + } + self.assertEqual(len(custom_field_values), 6) + self.assertEqual(custom_field_values['text'], 'ABC') + self.assertEqual(custom_field_values['integer'], 123) + self.assertEqual(custom_field_values['boolean'], True) + self.assertEqual(custom_field_values['date'], date(2020, 1, 1)) + self.assertEqual(custom_field_values['url'], 'http://example.com/1') + self.assertEqual(custom_field_values['select'].value, 'Choice A') - def test_import_invalid_choice(self): - """ - Import a site with an invalid choice - """ - csv_data = ( - "name,slug,cf_choice", - "Site 2,site-2,cf_field_4", - ) + # Validate data for site 2 + custom_field_values = { + cf.name: value for cf, value in Site.objects.get(name='Site 2').get_custom_fields().items() + } + self.assertEqual(len(custom_field_values), 6) + self.assertEqual(custom_field_values['text'], 'DEF') + self.assertEqual(custom_field_values['integer'], 456) + self.assertEqual(custom_field_values['boolean'], False) + self.assertEqual(custom_field_values['date'], date(2020, 1, 2)) + self.assertEqual(custom_field_values['url'], 'http://example.com/2') + self.assertEqual(custom_field_values['select'].value, 'Choice B') - response = self.client.post(reverse('dcim:site_import'), {'csv': '\n'.join(csv_data)}) - self.assertEqual(response.status_code, 200) - - self.assertFalse(len(Site.objects.filter(name="Site 2")), 0) + # No CustomFieldValues should be created for site 3 + obj_type = ContentType.objects.get_for_model(Site) + site3 = Site.objects.get(name='Site 3') + self.assertFalse(CustomFieldValue.objects.filter(obj_type=obj_type, obj_id=site3.pk).exists()) + self.assertEqual(CustomFieldValue.objects.count(), 12) # Sanity check From eafeaab014ce31e1703d0d634986dea4d00e8318 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Jan 2020 16:07:32 -0500 Subject: [PATCH 22/27] Add tests for invalid import data --- netbox/extras/tests/test_customfields.py | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index 703b0b0d0..a6e2bfcec 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -5,6 +5,7 @@ from django.test import Client, TestCase from django.urls import reverse from rest_framework import status +from dcim.forms import SiteCSVForm from dcim.models import Site from extras.choices import * from extras.models import CustomField, CustomFieldValue, CustomFieldChoice @@ -444,3 +445,33 @@ class CustomFieldImportTest(TestCase): site3 = Site.objects.get(name='Site 3') self.assertFalse(CustomFieldValue.objects.filter(obj_type=obj_type, obj_id=site3.pk).exists()) self.assertEqual(CustomFieldValue.objects.count(), 12) # Sanity check + + def test_import_missing_required(self): + """ + Attempt to import an object missing a required custom field. + """ + # Set one of our CustomFields to required + CustomField.objects.filter(name='text').update(required=True) + + form_data = { + 'name': 'Site 1', + 'slug': 'site-1', + } + + form = SiteCSVForm(data=form_data) + self.assertFalse(form.is_valid()) + self.assertIn('cf_text', form.errors) + + def test_import_invalid_choice(self): + """ + Attempt to import an object with an invalid choice selection. + """ + form_data = { + 'name': 'Site 1', + 'slug': 'site-1', + 'cf_select': 'Choice X' + } + + form = SiteCSVForm(data=form_data) + self.assertFalse(form.is_valid()) + self.assertIn('cf_select', form.errors) From 923c2728b38407335dec892386aa41ddc980204d Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 30 Jan 2020 12:08:40 -0500 Subject: [PATCH 23/27] Fixes #4056: Repair schema migration for Rack.outer_unit (from #3569) --- docs/release-notes/version-2.7.md | 1 + .../dcim/migrations/0079_3569_rack_fields.py | 2 +- .../migrations/0092_fix_rack_outer_unit.py | 27 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 netbox/dcim/migrations/0092_fix_rack_outer_unit.py diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 68487ebb8..9fc1c1c91 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -4,6 +4,7 @@ * [#4043](https://github.com/netbox-community/netbox/issues/4043) - Fix toggling of required fields in custom scripts * [#4049](https://github.com/netbox-community/netbox/issues/4049) - Restore missing `tags` field in IPAM service serializer +* [#4056](https://github.com/netbox-community/netbox/issues/4056) - Repair schema migration for Rack.outer_unit (from #3569) --- diff --git a/netbox/dcim/migrations/0079_3569_rack_fields.py b/netbox/dcim/migrations/0079_3569_rack_fields.py index 4e76a270f..da544bb7a 100644 --- a/netbox/dcim/migrations/0079_3569_rack_fields.py +++ b/netbox/dcim/migrations/0079_3569_rack_fields.py @@ -37,7 +37,7 @@ def rack_status_to_slug(apps, schema_editor): def rack_outer_unit_to_slug(apps, schema_editor): Rack = apps.get_model('dcim', 'Rack') for id, slug in RACK_DIMENSION_CHOICES: - Rack.objects.filter(status=str(id)).update(status=slug) + Rack.objects.filter(outer_unit=str(id)).update(outer_unit=slug) class Migration(migrations.Migration): diff --git a/netbox/dcim/migrations/0092_fix_rack_outer_unit.py b/netbox/dcim/migrations/0092_fix_rack_outer_unit.py new file mode 100644 index 000000000..2a8cbf4e5 --- /dev/null +++ b/netbox/dcim/migrations/0092_fix_rack_outer_unit.py @@ -0,0 +1,27 @@ +from django.db import migrations + +RACK_DIMENSION_CHOICES = ( + (1000, 'mm'), + (2000, 'in'), +) + + +def rack_outer_unit_to_slug(apps, schema_editor): + Rack = apps.get_model('dcim', 'Rack') + for id, slug in RACK_DIMENSION_CHOICES: + Rack.objects.filter(outer_unit=str(id)).update(outer_unit=slug) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dcim', '0091_interface_type_other'), + ] + + operations = [ + # Fixes a missed field migration from #3569; see bug #4056. The original migration has also been fixed, + # so this can be omitted when squashing in the future. + migrations.RunPython( + code=rack_outer_unit_to_slug + ), + ] From d9b8bc0422144780846c7ce2b977f123dc861ef4 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 30 Jan 2020 13:39:50 -0500 Subject: [PATCH 24/27] Fix VM interfaces table header alignment --- netbox/templates/virtualization/virtualmachine.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/netbox/templates/virtualization/virtualmachine.html b/netbox/templates/virtualization/virtualmachine.html index 8cf1fd490..982ef0e5e 100644 --- a/netbox/templates/virtualization/virtualmachine.html +++ b/netbox/templates/virtualization/virtualmachine.html @@ -265,7 +265,9 @@ Name LAG Description + MTU Mode + Cable Connection From 4b02d294ceafc6d5fa7fa955992b88d1e3b42a07 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 30 Jan 2020 13:55:39 -0500 Subject: [PATCH 25/27] Fixes #4052: Fix error when bulk importing interfaces to virtual machines --- docs/release-notes/version-2.7.md | 1 + netbox/dcim/forms.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 9fc1c1c91..1f6917730 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -4,6 +4,7 @@ * [#4043](https://github.com/netbox-community/netbox/issues/4043) - Fix toggling of required fields in custom scripts * [#4049](https://github.com/netbox-community/netbox/issues/4049) - Restore missing `tags` field in IPAM service serializer +* [#4052](https://github.com/netbox-community/netbox/issues/4052) - Fix error when bulk importing interfaces to virtual machines * [#4056](https://github.com/netbox-community/netbox/issues/4056) - Repair schema migration for Rack.outer_unit (from #3569) --- diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index ca5d25389..70fe7afaf 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -2726,7 +2726,7 @@ class InterfaceCSVForm(forms.ModelForm): super().__init__(*args, **kwargs) # Limit LAG choices to interfaces belonging to this device (or VC master) - if self.is_bound: + if self.is_bound and 'device' in self.data: try: device = self.fields['device'].to_python(self.data['device']) except forms.ValidationError: From d0d2af4cab9426a2e67767b708b2de0beedf4de1 Mon Sep 17 00:00:00 2001 From: agrrajag <57500846+agrrajag@users.noreply.github.com> Date: Thu, 30 Jan 2020 13:00:37 -0600 Subject: [PATCH 26/27] Update 3-http-daemon.md (#4055) There was no documentation to move back into the netbox folder after installing/configuring nginx. You would move into nginx on line 42 then try and figure out why you couldn't copy gunicorn on line 113. --- docs/installation/3-http-daemon.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/installation/3-http-daemon.md b/docs/installation/3-http-daemon.md index 5e19f54a2..cc1065fef 100644 --- a/docs/installation/3-http-daemon.md +++ b/docs/installation/3-http-daemon.md @@ -107,9 +107,10 @@ Install gunicorn: # pip3 install gunicorn ``` -Copy `contrib/gunicorn.py` to `/opt/netbox/gunicorn.py`. We make a copy of this file to ensure that any changes to it do not get overwritten by a future upgrade. +Copy `/opt/netbox/contrib/gunicorn.py` to `/opt/netbox/gunicorn.py`. We make a copy of this file to ensure that any changes to it do not get overwritten by a future upgrade. ```no-highlight +# cd /opt/netbox # cp contrib/gunicorn.py /opt/netbox/gunicorn.py ``` From 1a25f5a7f255ad332a6e4bfa43cb97294565c215 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 30 Jan 2020 15:12:10 -0500 Subject: [PATCH 27/27] Fixes #4030: Fix exception when bulk editing interfaces (revised) --- docs/release-notes/version-2.7.md | 1 + netbox/dcim/models/device_components.py | 2 +- netbox/utilities/views.py | 28 ++++++++++++++++++------- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 196c041c5..ba33e062c 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -6,6 +6,7 @@ ## Bug Fixes +* [#4030](https://github.com/netbox-community/netbox/issues/4030) - Fix exception when bulk editing interfaces (revised) * [#4043](https://github.com/netbox-community/netbox/issues/4043) - Fix toggling of required fields in custom scripts * [#4049](https://github.com/netbox-community/netbox/issues/4049) - Restore missing `tags` field in IPAM service serializer * [#4052](https://github.com/netbox-community/netbox/issues/4052) - Fix error when bulk importing interfaces to virtual machines diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 68bab8037..e37569f79 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -676,7 +676,7 @@ class Interface(CableTermination, ComponentModel): self.untagged_vlan = None # Only "tagged" interfaces may have tagged VLANs assigned. ("tagged all" implies all VLANs are assigned.) - if self.pk and self.mode is not InterfaceModeChoices.MODE_TAGGED: + if self.pk and self.mode != InterfaceModeChoices.MODE_TAGGED: self.tagged_vlans.clear() return super().save(*args, **kwargs) diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index d900a8545..88e5005bc 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -6,7 +6,7 @@ from django.contrib import messages from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError from django.db import transaction, IntegrityError -from django.db.models import Count, ProtectedError +from django.db.models import Count, ManyToManyField, ProtectedError from django.db.models.query import QuerySet from django.forms import CharField, Form, ModelMultipleChoiceField, MultipleHiddenInput, Textarea from django.http import HttpResponse, HttpResponseServerError @@ -650,7 +650,9 @@ class BulkEditView(GetReturnURLMixin, View): if form.is_valid(): custom_fields = form.custom_fields if hasattr(form, 'custom_fields') else [] - standard_fields = [field for field in form.fields if field not in custom_fields and field != 'pk'] + standard_fields = [ + field for field in form.fields if field not in custom_fields + ['pk', 'add_tags', 'remove_tags'] + ] nullified_fields = request.POST.getlist('_nullify') try: @@ -662,14 +664,24 @@ class BulkEditView(GetReturnURLMixin, View): # Update standard fields. If a field is listed in _nullify, delete its value. for name in standard_fields: - if name in form.nullable_fields and name in nullified_fields and isinstance(form.cleaned_data[name], QuerySet): - getattr(obj, name).set([]) - elif name in form.nullable_fields and name in nullified_fields: - setattr(obj, name, '' if isinstance(form.fields[name], CharField) else None) - elif isinstance(form.cleaned_data[name], QuerySet) and form.cleaned_data[name]: + + model_field = model._meta.get_field(name) + + # Handle nullification + if name in form.nullable_fields and name in nullified_fields: + if isinstance(model_field, ManyToManyField): + getattr(obj, name).set([]) + else: + setattr(obj, name, None if model_field.null else '') + + # ManyToManyFields + elif isinstance(model_field, ManyToManyField): getattr(obj, name).set(form.cleaned_data[name]) - elif form.cleaned_data[name] not in (None, '') and not isinstance(form.cleaned_data[name], QuerySet): + + # Normal fields + elif form.cleaned_data[name] not in (None, ''): setattr(obj, name, form.cleaned_data[name]) + obj.full_clean() obj.save()