Add choices ArrayField to CustomField; drop CustomFieldChoice

This commit is contained in:
Jeremy Stretch 2020-08-25 13:24:46 -04:00
parent d9e5adc032
commit f7b8d6ede5
10 changed files with 127 additions and 222 deletions

View File

@ -2,7 +2,7 @@ from django import forms
from django.contrib import admin
from utilities.forms import LaxURLField
from .models import CustomField, CustomFieldChoice, CustomLink, ExportTemplate, JobResult, Webhook
from .models import CustomField, CustomLink, ExportTemplate, JobResult, Webhook
def order_content_types(field):
@ -81,14 +81,8 @@ class CustomFieldForm(forms.ModelForm):
order_content_types(self.fields['obj_type'])
class CustomFieldChoiceAdmin(admin.TabularInline):
model = CustomFieldChoice
extra = 5
@admin.register(CustomField)
class CustomFieldAdmin(admin.ModelAdmin):
inlines = [CustomFieldChoiceAdmin]
list_display = [
'name', 'models', 'type', 'required', 'filter_logic', 'default', 'weight', 'description',
]

View File

@ -7,7 +7,7 @@ from rest_framework.exceptions import ValidationError
from rest_framework.fields import CreateOnlyDefault
from extras.choices import *
from extras.models import CustomField, CustomFieldChoice
from extras.models import CustomField
from utilities.api import ValidatedModelSerializer
@ -37,12 +37,6 @@ class CustomFieldDefaultValues:
elif field.type == CustomFieldTypeChoices.TYPE_BOOLEAN:
# TODO: Fix default value assignment for boolean custom fields
field_value = False if field.default.lower() == 'false' else bool(field.default)
elif field.type == CustomFieldTypeChoices.TYPE_SELECT:
try:
field_value = field.choices.get(value=field.default).pk
except ObjectDoesNotExist:
# Invalid default value
field_value = None
else:
field_value = field.default
value[field.name] = field_value
@ -69,9 +63,7 @@ class CustomFieldsSerializer(serializers.BaseSerializer):
try:
cf = custom_fields[field_name]
except KeyError:
raise ValidationError(
"Invalid custom field for {} objects: {}".format(content_type, field_name)
)
raise ValidationError(f"Invalid custom field for {content_type} objects: {field_name}")
# Data validation
if value not in [None, '']:
@ -81,15 +73,11 @@ class CustomFieldsSerializer(serializers.BaseSerializer):
try:
int(value)
except ValueError:
raise ValidationError(
"Invalid value for integer field {}: {}".format(field_name, value)
)
raise ValidationError(f"Invalid value for integer field {field_name}: {value}")
# Validate boolean
if cf.type == CustomFieldTypeChoices.TYPE_BOOLEAN and value not in [True, False, 1, 0]:
raise ValidationError(
"Invalid value for boolean field {}: {}".format(field_name, value)
)
raise ValidationError(f"Invalid value for boolean field {field_name}: {value}")
# Validate date
if cf.type == CustomFieldTypeChoices.TYPE_DATE:
@ -97,25 +85,16 @@ class CustomFieldsSerializer(serializers.BaseSerializer):
datetime.strptime(value, '%Y-%m-%d')
except ValueError:
raise ValidationError(
"Invalid date for field {}: {}. (Required format is YYYY-MM-DD.)".format(field_name, value)
f"Invalid date for field {field_name}: {value}. (Required format is YYYY-MM-DD.)"
)
# Validate selected choice
if cf.type == CustomFieldTypeChoices.TYPE_SELECT:
try:
value = int(value)
except ValueError:
raise ValidationError(
"{}: Choice selections must be passed as integers.".format(field_name)
)
valid_choices = [c.pk for c in cf.choices.all()]
if value not in valid_choices:
raise ValidationError(
"Invalid choice for field {}: {}".format(field_name, value)
)
if value not in cf.choices:
raise ValidationError(f"Invalid choice for field {field_name}: {value}")
elif cf.required:
raise ValidationError("Required field {} cannot be empty.".format(field_name))
raise ValidationError(f"Required field {field_name} cannot be empty.")
# Check for missing required fields
missing_fields = []
@ -157,20 +136,4 @@ class CustomFieldModelSerializer(ValidatedModelSerializer):
def _populate_custom_fields(self, instance, custom_fields):
instance.custom_fields = {}
for field in custom_fields:
value = instance.cf.get(field.name)
if field.type == CustomFieldTypeChoices.TYPE_SELECT and value is not None:
instance.custom_fields[field.name] = CustomFieldChoiceSerializer(value).data
else:
instance.custom_fields[field.name] = value
class CustomFieldChoiceSerializer(serializers.ModelSerializer):
"""
Imitate utilities.api.ChoiceFieldSerializer
"""
value = serializers.IntegerField(source='pk')
label = serializers.CharField(source='value')
class Meta:
model = CustomFieldChoice
fields = ['value', 'label']
instance.custom_fields[field.name] = instance.cf.get(field.name)

View File

@ -5,9 +5,6 @@ from . import views
router = OrderedDefaultRouter()
router.APIRootView = views.ExtrasRootView
# Custom field choices
router.register('_custom_field_choices', views.CustomFieldChoicesViewSet, basename='custom-field-choice')
# Export templates
router.register('export-templates', views.ExportTemplateViewSet)

View File

@ -14,9 +14,7 @@ from rq import Worker
from extras import filters
from extras.choices import JobResultStatusChoices
from extras.models import (
ConfigContext, CustomFieldChoice, ExportTemplate, ImageAttachment, ObjectChange, JobResult, Tag,
)
from extras.models import ConfigContext, ExportTemplate, ImageAttachment, ObjectChange, JobResult, Tag
from extras.reports import get_report, get_reports, run_report
from extras.scripts import get_script, get_scripts, run_script
from utilities.api import IsAuthenticatedOrLoginNotRequired, ModelViewSet
@ -34,36 +32,6 @@ class ExtrasRootView(APIRootView):
return 'Extras'
#
# Custom field choices
#
class CustomFieldChoicesViewSet(ViewSet):
"""
"""
permission_classes = [IsAuthenticatedOrLoginNotRequired]
def __init__(self, *args, **kwargs):
super(CustomFieldChoicesViewSet, self).__init__(*args, **kwargs)
self._fields = OrderedDict()
for cfc in CustomFieldChoice.objects.all():
self._fields.setdefault(cfc.field.name, {})
self._fields[cfc.field.name][cfc.value] = cfc.pk
def list(self, request):
return Response(self._fields)
def retrieve(self, request, pk):
if pk not in self._fields:
raise Http404
return Response(self._fields[pk])
def get_view_name(self):
return "Custom Field choices"
#
# Custom fields
#
@ -77,19 +45,11 @@ class CustomFieldModelViewSet(ModelViewSet):
# Gather all custom fields for the model
content_type = ContentType.objects.get_for_model(self.queryset.model)
custom_fields = content_type.custom_fields.prefetch_related('choices')
# Cache all relevant CustomFieldChoices. This saves us from having to do a lookup per select field per object.
custom_field_choices = {}
for field in custom_fields:
for cfc in field.choices.all():
custom_field_choices[cfc.id] = cfc.value
custom_field_choices = custom_field_choices
custom_fields = content_type.custom_fields.all()
context = super().get_serializer_context()
context.update({
'custom_fields': custom_fields,
'custom_field_choices': custom_field_choices,
})
return context

View File

@ -0,0 +1,35 @@
import django.contrib.postgres.fields
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('extras', '0049_remove_graph'),
]
operations = [
# Rename reverse relation on CustomFieldChoice
migrations.AlterField(
model_name='customfieldchoice',
name='field',
field=models.ForeignKey(
limit_choices_to={'type': 'select'},
on_delete=django.db.models.deletion.CASCADE,
related_name='_choices',
to='extras.customfield'
),
),
# Add choices field to CustomField
migrations.AddField(
model_name='customfield',
name='choices',
field=django.contrib.postgres.fields.ArrayField(
base_field=models.CharField(max_length=100),
blank=True,
null=True,
size=None
),
),
]

View File

@ -3,18 +3,38 @@ from django.db import migrations
from extras.choices import CustomFieldTypeChoices
def deserialize_value(field_type, value):
def deserialize_value(field, value):
"""
Convert serialized values to JSON equivalents.
"""
if field_type in (CustomFieldTypeChoices.TYPE_INTEGER, CustomFieldTypeChoices.TYPE_SELECT):
if field.type in (CustomFieldTypeChoices.TYPE_INTEGER):
return int(value)
if field_type == CustomFieldTypeChoices.TYPE_BOOLEAN:
if field.type == CustomFieldTypeChoices.TYPE_BOOLEAN:
return bool(int(value))
if field.type == CustomFieldTypeChoices.TYPE_SELECT:
return field._choices.get(pk=int(value)).value
return value
def migrate_customfieldchoices(apps, schema_editor):
"""
Collect all CustomFieldChoices for each applicable CustomField, and save them locally as an array on
the CustomField instance.
"""
CustomField = apps.get_model('extras', 'CustomField')
CustomFieldChoice = apps.get_model('extras', 'CustomFieldChoice')
for cf in CustomField.objects.filter(type='select'):
cf.choices = [
cfc.value for cfc in CustomFieldChoice.objects.filter(field=cf).order_by('weight', 'value')
]
cf.save()
def migrate_customfieldvalues(apps, schema_editor):
"""
Copy data from CustomFieldValues into the custom_field_data JSON field on each model instance.
"""
CustomFieldValue = apps.get_model('extras', 'CustomFieldValue')
for cfv in CustomFieldValue.objects.prefetch_related('field').exclude(serialized_value=''):
@ -24,7 +44,7 @@ def migrate_customfieldvalues(apps, schema_editor):
# TODO: This can be done more efficiently once .update() is supported for JSON fields
cf_data = model.objects.filter(pk=cfv.obj_id).values('custom_field_data').first()
try:
cf_data['custom_field_data'][cfv.field.name] = deserialize_value(cfv.field.type, cfv.serialized_value)
cf_data['custom_field_data'][cfv.field.name] = deserialize_value(cfv.field, cfv.serialized_value)
except ValueError as e:
print(f'{cfv.field.name} ({cfv.field.type}): {cfv.serialized_value} ({cfv.pk})')
raise e
@ -36,7 +56,7 @@ class Migration(migrations.Migration):
dependencies = [
('circuits', '0020_custom_field_data'),
('dcim', '0115_custom_field_data'),
('extras', '0049_remove_graph'),
('extras', '0050_customfield_add_choices'),
('ipam', '0038_custom_field_data'),
('secrets', '0010_custom_field_data'),
('tenancy', '0010_custom_field_data'),
@ -44,6 +64,9 @@ class Migration(migrations.Migration):
]
operations = [
migrations.RunPython(
code=migrate_customfieldchoices
),
migrations.RunPython(
code=migrate_customfieldvalues
),

View File

@ -1,15 +1,16 @@
# Generated by Django 3.1 on 2020-08-21 19:52
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('extras', '0050_migrate_customfieldvalues'),
('extras', '0051_migrate_customfields'),
]
operations = [
migrations.DeleteModel(
name='CustomFieldChoice',
),
migrations.DeleteModel(
name='CustomFieldValue',
),

View File

@ -1,5 +1,5 @@
from .change_logging import ChangeLoggedModel, ObjectChange
from .customfields import CustomField, CustomFieldChoice, CustomFieldModel
from .customfields import CustomField, CustomFieldModel
from .models import (
ConfigContext, ConfigContextModel, CustomLink, ExportTemplate, ImageAttachment, JobResult, Report, Script,
Webhook,
@ -11,7 +11,6 @@ __all__ = (
'ConfigContext',
'ConfigContextModel',
'CustomField',
'CustomFieldChoice',
'CustomFieldModel',
'CustomLink',
'ExportTemplate',

View File

@ -3,6 +3,7 @@ from datetime import date
from django import forms
from django.contrib.contenttypes.models import ContentType
from django.contrib.postgres.fields import ArrayField
from django.core.validators import ValidationError
from django.db import models
@ -11,11 +12,10 @@ from extras.choices import *
from extras.utils import FeatureQuery
#
# Custom fields
#
class CustomFieldModel(models.Model):
"""
Abstract class for any model which may have custom fields associated with it.
"""
custom_field_data = models.JSONField(
blank=True,
default=dict
@ -104,6 +104,12 @@ class CustomField(models.Model):
default=100,
help_text='Fields with higher weights appear lower in a form.'
)
choices = ArrayField(
base_field=models.CharField(max_length=100),
blank=True,
null=True,
help_text='Comma-separated list of available choices (for selection fields)'
)
objects = CustomFieldManager()
@ -113,6 +119,19 @@ class CustomField(models.Model):
def __str__(self):
return self.label or self.name.replace('_', ' ').capitalize()
def clean(self):
# Choices can be set only on selection fields
if self.choices and self.type != CustomFieldTypeChoices.TYPE_SELECT:
raise ValidationError({
'choices': "Choices may be set only for selection-type custom fields."
})
# A selection field's default (if any) must be present in its available choices
if self.type == CustomFieldTypeChoices.TYPE_SELECT and self.default and self.default not in self.choices:
raise ValidationError({
'default': f"The specified default value ({self.default}) is not listed as an available choice."
})
def serialize_value(self, value):
"""
Serialize the given value to a string suitable for storage as a CustomFieldValue
@ -187,16 +206,14 @@ class CustomField(models.Model):
# Select
elif self.type == CustomFieldTypeChoices.TYPE_SELECT:
choices = [(cfc.pk, cfc.value) for cfc in self.choices.all()]
choices = [(c, c) for c in self.choices]
if not required:
choices = add_blank_choice(choices)
# Set the initial value to the PK of the default choice, if any
if set_initial:
default_choice = self.choices.filter(value=self.default).first()
if default_choice:
initial = default_choice.pk
# Set the initial value to the first available choice (if any)
if set_initial and self.choices:
initial = self.choices[0]
field_class = CSVChoiceField if for_csv_import else forms.ChoiceField
field = field_class(
@ -217,41 +234,3 @@ class CustomField(models.Model):
field.help_text = self.description
return field
class CustomFieldChoice(models.Model):
field = models.ForeignKey(
to='extras.CustomField',
on_delete=models.CASCADE,
related_name='choices',
limit_choices_to={'type': CustomFieldTypeChoices.TYPE_SELECT}
)
value = models.CharField(
max_length=100
)
weight = models.PositiveSmallIntegerField(
default=100,
help_text='Higher weights appear lower in the list'
)
class Meta:
ordering = ['field', 'weight', 'value']
unique_together = ['field', 'value']
def __str__(self):
return self.value
def clean(self):
if self.field.type != CustomFieldTypeChoices.TYPE_SELECT:
raise ValidationError("Custom field choices can only be assigned to selection fields.")
def delete(self, *args, **kwargs):
# TODO: Prevent deletion of CustomFieldChoices which are in use?
field_name = f'custom_field_data__{self.field.name}'
for ct in self.field.obj_type.all():
model = ct.model_class()
for instance in model.objects.filter(**{field_name: self.pk}):
instance.custom_field_data.pop(self.field.name)
instance.save()
super().delete(*args, **kwargs)

View File

@ -5,7 +5,7 @@ from rest_framework import status
from dcim.forms import SiteCSVForm
from dcim.models import Site
from extras.choices import *
from extras.models import CustomField, CustomFieldChoice
from extras.models import CustomField
from utilities.testing import APITestCase, TestCase
from virtualization.models import VirtualMachine
@ -65,21 +65,19 @@ class CustomFieldTest(TestCase):
obj_type = ContentType.objects.get_for_model(Site)
# Create a custom field
cf = CustomField(type=CustomFieldTypeChoices.TYPE_SELECT, name='my_field', required=False)
cf = CustomField(
type=CustomFieldTypeChoices.TYPE_SELECT,
name='my_field',
required=False,
choices=['Option A', 'Option B', 'Option C']
)
cf.save()
cf.obj_type.set([obj_type])
cf.save()
# Create some choices for the field
CustomFieldChoice.objects.bulk_create([
CustomFieldChoice(field=cf, value='Option A'),
CustomFieldChoice(field=cf, value='Option B'),
CustomFieldChoice(field=cf, value='Option C'),
])
# Assign a value to the first Site
site = Site.objects.first()
site.custom_field_data[cf.name] = cf.choices.first().pk
site.custom_field_data[cf.name] = 'Option A'
site.save()
# Retrieve the stored value
@ -141,18 +139,10 @@ class CustomFieldAPITest(APITestCase):
cls.cf_url.obj_type.set([content_type])
# Select custom field
cls.cf_select = CustomField(type=CustomFieldTypeChoices.TYPE_SELECT, name='choice_field')
cls.cf_select = CustomField(type=CustomFieldTypeChoices.TYPE_SELECT, name='choice_field', choices=['Foo', 'Bar', 'Baz'])
cls.cf_select.default = 'Foo'
cls.cf_select.save()
cls.cf_select.obj_type.set([content_type])
cls.cf_select_choice1 = CustomFieldChoice(field=cls.cf_select, value='Foo')
cls.cf_select_choice1.save()
cls.cf_select_choice2 = CustomFieldChoice(field=cls.cf_select, value='Bar')
cls.cf_select_choice2.save()
cls.cf_select_choice3 = CustomFieldChoice(field=cls.cf_select, value='Baz')
cls.cf_select_choice3.save()
cls.cf_select.default = cls.cf_select_choice1.value
cls.cf_select.save()
# Create some sites
cls.sites = (
@ -168,7 +158,7 @@ class CustomFieldAPITest(APITestCase):
cls.cf_boolean.name: True,
cls.cf_date.name: '2020-01-02',
cls.cf_url.name: 'http://example.com/2',
cls.cf_select.name: cls.cf_select_choice2.pk,
cls.cf_select.name: 'Bar',
}
cls.sites[1].save()
@ -205,7 +195,7 @@ class CustomFieldAPITest(APITestCase):
self.assertEqual(response.data['custom_fields']['boolean_field'], site2_cfvs['boolean_field'])
self.assertEqual(response.data['custom_fields']['date_field'], site2_cfvs['date_field'])
self.assertEqual(response.data['custom_fields']['url_field'], site2_cfvs['url_field'])
self.assertEqual(response.data['custom_fields']['choice_field']['label'], self.cf_select_choice2.value)
self.assertEqual(response.data['custom_fields']['choice_field'], site2_cfvs['choice_field'])
def test_create_single_object_with_defaults(self):
"""
@ -228,7 +218,7 @@ class CustomFieldAPITest(APITestCase):
self.assertEqual(response_cf['boolean_field'], self.cf_boolean.default)
self.assertEqual(response_cf['date_field'], self.cf_date.default)
self.assertEqual(response_cf['url_field'], self.cf_url.default)
self.assertEqual(response_cf['choice_field'], self.cf_select_choice1.pk)
self.assertEqual(response_cf['choice_field'], self.cf_select.default)
# Validate database data
site = Site.objects.get(pk=response.data['id'])
@ -237,7 +227,7 @@ class CustomFieldAPITest(APITestCase):
self.assertEqual(site.custom_field_data['boolean_field'], self.cf_boolean.default)
self.assertEqual(str(site.custom_field_data['date_field']), self.cf_date.default)
self.assertEqual(site.custom_field_data['url_field'], self.cf_url.default)
self.assertEqual(site.custom_field_data['choice_field'], self.cf_select_choice1.pk)
self.assertEqual(site.custom_field_data['choice_field'], self.cf_select.default)
def test_create_single_object_with_values(self):
"""
@ -252,7 +242,7 @@ class CustomFieldAPITest(APITestCase):
'boolean_field': True,
'date_field': '2020-01-02',
'url_field': 'http://example.com/2',
'choice_field': self.cf_select_choice2.pk,
'choice_field': 'Bar',
},
}
url = reverse('dcim-api:site-list')
@ -315,7 +305,7 @@ class CustomFieldAPITest(APITestCase):
self.assertEqual(response_cf['boolean_field'], self.cf_boolean.default)
self.assertEqual(response_cf['date_field'], self.cf_date.default)
self.assertEqual(response_cf['url_field'], self.cf_url.default)
self.assertEqual(response_cf['choice_field'], self.cf_select_choice1.pk)
self.assertEqual(response_cf['choice_field'], self.cf_select.default)
# Validate database data
site = Site.objects.get(pk=response.data[i]['id'])
@ -324,7 +314,7 @@ class CustomFieldAPITest(APITestCase):
self.assertEqual(site.custom_field_data['boolean_field'], self.cf_boolean.default)
self.assertEqual(str(site.custom_field_data['date_field']), self.cf_date.default)
self.assertEqual(site.custom_field_data['url_field'], self.cf_url.default)
self.assertEqual(site.custom_field_data['choice_field'], self.cf_select_choice1.pk)
self.assertEqual(site.custom_field_data['choice_field'], self.cf_select.default)
def test_create_multiple_objects_with_values(self):
"""
@ -336,7 +326,7 @@ class CustomFieldAPITest(APITestCase):
'boolean_field': True,
'date_field': '2020-01-02',
'url_field': 'http://example.com/2',
'choice_field': self.cf_select_choice2.pk,
'choice_field': 'Bar',
}
data = (
{
@ -410,7 +400,7 @@ class CustomFieldAPITest(APITestCase):
# self.assertEqual(response_cf['boolean_field'], site2_original_cfvs['boolean_field'])
# self.assertEqual(response_cf['date_field'], site2_original_cfvs['date_field'])
# self.assertEqual(response_cf['url_field'], site2_original_cfvs['url_field'])
# self.assertEqual(response_cf['choice_field']['label'], site2_original_cfvs['choice_field'].value)
# self.assertEqual(response_cf['choice_field'], site2_original_cfvs['choice_field'].value)
# Validate database data
site.refresh_from_db()
@ -422,36 +412,6 @@ class CustomFieldAPITest(APITestCase):
self.assertEqual(site.custom_field_data['choice_field'], original_cfvs['choice_field'])
class CustomFieldChoiceAPITest(APITestCase):
def setUp(self):
super().setUp()
vm_content_type = ContentType.objects.get_for_model(VirtualMachine)
self.cf_1 = CustomField.objects.create(name="cf_1", type=CustomFieldTypeChoices.TYPE_SELECT)
self.cf_2 = CustomField.objects.create(name="cf_2", type=CustomFieldTypeChoices.TYPE_SELECT)
self.cf_choice_1 = CustomFieldChoice.objects.create(field=self.cf_1, value="cf_field_1", weight=100)
self.cf_choice_2 = CustomFieldChoice.objects.create(field=self.cf_1, value="cf_field_2", weight=50)
self.cf_choice_3 = CustomFieldChoice.objects.create(field=self.cf_2, value="cf_field_3", weight=10)
def test_list_cfc(self):
url = reverse('extras-api:custom-field-choice-list')
response = self.client.get(url, **self.header)
self.assertEqual(len(response.data), 2)
self.assertEqual(len(response.data[self.cf_1.name]), 2)
self.assertEqual(len(response.data[self.cf_2.name]), 1)
self.assertTrue(self.cf_choice_1.value in response.data[self.cf_1.name])
self.assertTrue(self.cf_choice_2.value in response.data[self.cf_1.name])
self.assertTrue(self.cf_choice_3.value in response.data[self.cf_2.name])
self.assertEqual(self.cf_choice_1.pk, response.data[self.cf_1.name][self.cf_choice_1.value])
self.assertEqual(self.cf_choice_2.pk, response.data[self.cf_1.name][self.cf_choice_2.value])
self.assertEqual(self.cf_choice_3.pk, response.data[self.cf_2.name][self.cf_choice_3.value])
class CustomFieldImportTest(TestCase):
user_permissions = (
'dcim.view_site',
@ -467,18 +427,12 @@ class CustomFieldImportTest(TestCase):
CustomField(name='boolean', type=CustomFieldTypeChoices.TYPE_BOOLEAN),
CustomField(name='date', type=CustomFieldTypeChoices.TYPE_DATE),
CustomField(name='url', type=CustomFieldTypeChoices.TYPE_URL),
CustomField(name='select', type=CustomFieldTypeChoices.TYPE_SELECT),
CustomField(name='select', type=CustomFieldTypeChoices.TYPE_SELECT, choices=['Choice A', 'Choice B', 'Choice C']),
)
for cf in custom_fields:
cf.save()
cf.obj_type.set([ContentType.objects.get_for_model(Site)])
CustomFieldChoice.objects.bulk_create((
CustomFieldChoice(field=custom_fields[5], value='Choice A'),
CustomFieldChoice(field=custom_fields[5], value='Choice B'),
CustomFieldChoice(field=custom_fields[5], value='Choice C'),
))
def test_import(self):
"""
Import a Site in CSV format, including a value for each CustomField.