Merge branch 'develop-2.10' into 1503-secret-assignment

docs/release-notes/version-2.10.md

@@ -8,6 +8,7 @@
 
 * [#1503](https://github.com/netbox-community/netbox/issues/1503) - Allow assigment of secrets to virtual machines
 * [#1692](https://github.com/netbox-community/netbox/issues/1692) - Allow assigment of inventory items to parent items in web UI
+* [#2179](https://github.com/netbox-community/netbox/issues/2179) - Support the assignment of multiple port numbers for services
 * [#4956](https://github.com/netbox-community/netbox/issues/4956) - Include inventory items on primary device view
 * [#5003](https://github.com/netbox-community/netbox/issues/5003) - CSV import now accepts slug values for choice fields
 * [#5146](https://github.com/netbox-community/netbox/issues/5146) - Add custom fields support for cables, power panels, rack reservations, and virtual chassis
@@ -29,4 +30,5 @@
 * dcim.VirtualChassis: Added `custom_fields`
 * extras.ExportTemplate: The `template_language` field has been removed
 * extras.Graph: This API endpoint has been removed (see #4349)
+* ipam.Service: Renamed `port` to `ports`; now holds a list of one or more port numbers
 * secrets.Secret: Removed `device` field; replaced with `assigned_object` generic foreign key. This may represent either a device or a virtual machine. Assign an object by setting `assigned_object_type` and `assigned_object_id`.

netbox/dcim/models/racks.py

@@ -1,5 +1,4 @@
 from collections import OrderedDict
-from itertools import count, groupby
 
 from django.conf import settings
 from django.contrib.auth.models import User
@@ -22,7 +21,7 @@ from utilities.choices import ColorChoices
 from utilities.fields import ColorField, NaturalOrderingField
 from utilities.querysets import RestrictedQuerySet
 from utilities.mptt import TreeManager
-from utilities.utils import serialize_object
+from utilities.utils import array_to_string, serialize_object
 from .devices import Device
 from .power import PowerFeed
 
@@ -642,9 +641,4 @@ class RackReservation(ChangeLoggedModel, CustomFieldModel):
 
     @property
     def unit_list(self):
-        """
+        return array_to_string(self.units)
-        Express the assigned units as a string of summarized ranges. For example:
-            [0, 1, 2, 10, 14, 15, 16] => "0-2, 10, 14-16"
-        """
-        group = (list(x) for _, x in groupby(sorted(self.units), lambda x, c=count(): next(c) - x))
-        return ', '.join('-'.join(map(str, (g[0], g[-1])[:len(g)])) for g in group)

netbox/ipam/api/nested_serializers.py

@@ -117,4 +117,4 @@ class NestedServiceSerializer(WritableNestedSerializer):
 
     class Meta:
         model = models.Service
-        fields = ['id', 'url', 'name', 'protocol', 'port']
+        fields = ['id', 'url', 'name', 'protocol', 'ports']

netbox/ipam/api/serializers.py

@@ -282,6 +282,6 @@ class ServiceSerializer(TaggedObjectSerializer, CustomFieldModelSerializer):
     class Meta:
         model = Service
         fields = [
-            'id', 'url', 'device', 'virtual_machine', 'name', 'port', 'protocol', 'ipaddresses', 'description', 'tags',
+            'id', 'url', 'device', 'virtual_machine', 'name', 'ports', 'protocol', 'ipaddresses', 'description', 'tags',
             'custom_fields', 'created', 'last_updated',
         ]

netbox/ipam/filters.py

@@ -8,7 +8,7 @@ from dcim.models import Device, Interface, Region, Site
 from extras.filters import CustomFieldFilterSet, CreatedUpdatedFilterSet
 from tenancy.filters import TenancyFilterSet
 from utilities.filters import (
-    BaseFilterSet, MultiValueCharFilter, MultiValueNumberFilter, NameSlugSearchFilterSet, TagFilter,
+    BaseFilterSet, MultiValueCharFilter, MultiValueNumberFilter, NameSlugSearchFilterSet, NumericArrayFilter, TagFilter,
     TreeNodeMultipleChoiceFilter,
 )
 from virtualization.models import VirtualMachine, VMInterface
@@ -542,11 +542,15 @@ class ServiceFilterSet(BaseFilterSet, CreatedUpdatedFilterSet):
         to_field_name='name',
         label='Virtual machine (name)',
     )
+    port = NumericArrayFilter(
+        field_name='ports',
+        lookup_expr='contains'
+    )
     tag = TagFilter()
 
     class Meta:
         model = Service
-        fields = ['id', 'name', 'protocol', 'port']
+        fields = ['id', 'name', 'protocol']
 
     def search(self, queryset, name, value):
         if not value.strip():

netbox/ipam/forms.py

@@ -10,8 +10,8 @@ from tenancy.forms import TenancyFilterForm, TenancyForm
 from tenancy.models import Tenant
 from utilities.forms import (
     add_blank_choice, BootstrapMixin, BulkEditNullBooleanSelect, CSVChoiceField, CSVModelChoiceField, CSVModelForm,
-    DatePicker, DynamicModelChoiceField, DynamicModelMultipleChoiceField, ExpandableIPAddressField, ReturnURLForm,
+    DatePicker, DynamicModelChoiceField, DynamicModelMultipleChoiceField, ExpandableIPAddressField, NumericArrayField,
-    SlugField, StaticSelect2, StaticSelect2Multiple, TagFilterField, BOOLEAN_WITH_BLANK_CHOICES,
+    ReturnURLForm, SlugField, StaticSelect2, StaticSelect2Multiple, TagFilterField, BOOLEAN_WITH_BLANK_CHOICES,
 )
 from virtualization.models import Cluster, VirtualMachine, VMInterface
 from .choices import *
@@ -1155,9 +1155,12 @@ class VLANFilterForm(BootstrapMixin, TenancyFilterForm, CustomFieldFilterForm):
 #
 
 class ServiceForm(BootstrapMixin, CustomFieldModelForm):
-    port = forms.IntegerField(
+    ports = NumericArrayField(
+        base_field=forms.IntegerField(
             min_value=SERVICE_PORT_MIN,
             max_value=SERVICE_PORT_MAX
+        ),
+        help_text="Comma-separated list of numeric unit IDs. A range may be specified using a hyphen."
     )
     tags = DynamicModelMultipleChoiceField(
         queryset=Tag.objects.all(),
@@ -1167,7 +1170,7 @@ class ServiceForm(BootstrapMixin, CustomFieldModelForm):
     class Meta:
         model = Service
         fields = [
-            'name', 'protocol', 'port', 'ipaddresses', 'description', 'tags',
+            'name', 'protocol', 'ports', 'ipaddresses', 'description', 'tags',
         ]
         help_texts = {
             'ipaddresses': "IP address assignment is optional. If no IPs are selected, the service is assumed to be "
@@ -1244,11 +1247,11 @@ class ServiceBulkEditForm(BootstrapMixin, AddRemoveTagsForm, CustomFieldBulkEdit
         required=False,
         widget=StaticSelect2()
     )
-    port = forms.IntegerField(
+    ports = NumericArrayField(
-        validators=[
+        base_field=forms.IntegerField(
-            MinValueValidator(1),
+            min_value=SERVICE_PORT_MIN,
-            MaxValueValidator(65535),
+            max_value=SERVICE_PORT_MAX
-        ],
+        ),
         required=False
     )
     description = forms.CharField(

netbox/ipam/migrations/0039_service_ports_array.py

@@ -0,0 +1,43 @@
+import django.contrib.postgres.fields
+import django.core.validators
+from django.db import migrations, models
+
+
+def replicate_ports(apps, schema_editor):
+    Service = apps.get_model('ipam', 'Service')
+    # TODO: Figure out how to cast IntegerField to an array so we can use .update()
+    for service in Service.objects.all():
+        Service.objects.filter(pk=service.pk).update(ports=[service.port])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ipam', '0038_custom_field_data'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='service',
+            name='ports',
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.PositiveIntegerField(
+                    validators=[
+                        django.core.validators.MinValueValidator(1),
+                        django.core.validators.MaxValueValidator(65535)
+                    ]
+                ),
+                default=[],
+                size=None
+            ),
+            preserve_default=False,
+        ),
+
+        migrations.AlterModelOptions(
+            name='service',
+            options={'ordering': ('protocol', 'ports', 'pk')},
+        ),
+        migrations.RunPython(
+            code=replicate_ports
+        ),
+    ]

netbox/ipam/migrations/0040_service_drop_port.py

@@ -0,0 +1,15 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ipam', '0039_service_ports_array'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='service',
+            name='port',
+        ),
+    ]

netbox/ipam/models.py

@@ -2,6 +2,7 @@ import netaddr
 from django.conf import settings
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
+from django.contrib.postgres.fields import ArrayField
 from django.core.exceptions import ValidationError
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
@@ -13,7 +14,7 @@ from dcim.models import Device, Interface
 from extras.models import ChangeLoggedModel, CustomFieldModel, ObjectChange, TaggedItem
 from extras.utils import extras_features
 from utilities.querysets import RestrictedQuerySet
-from utilities.utils import serialize_object
+from utilities.utils import array_to_string, serialize_object
 from virtualization.models import VirtualMachine, VMInterface
 from .choices import *
 from .constants import *
@@ -1008,12 +1009,14 @@ class Service(ChangeLoggedModel, CustomFieldModel):
         max_length=50,
         choices=ServiceProtocolChoices
     )
-    port = models.PositiveIntegerField(
+    ports = ArrayField(
+        base_field=models.PositiveIntegerField(
             validators=[
                 MinValueValidator(SERVICE_PORT_MIN),
                 MaxValueValidator(SERVICE_PORT_MAX)
-        ],
+            ]
-        verbose_name='Port number'
+        ),
+        verbose_name='Port numbers'
     )
     ipaddresses = models.ManyToManyField(
         to='ipam.IPAddress',
@@ -1029,13 +1032,13 @@ class Service(ChangeLoggedModel, CustomFieldModel):
 
     objects = RestrictedQuerySet.as_manager()
 
-    csv_headers = ['device', 'virtual_machine', 'name', 'protocol', 'port', 'description']
+    csv_headers = ['device', 'virtual_machine', 'name', 'protocol', 'ports', 'description']
 
     class Meta:
-        ordering = ('protocol', 'port', 'pk')  # (protocol, port) may be non-unique
+        ordering = ('protocol', 'ports', 'pk')  # (protocol, port) may be non-unique
 
     def __str__(self):
-        return '{} ({}/{})'.format(self.name, self.port, self.get_protocol_display())
+        return f'{self.name} ({self.get_protocol_display()}/{self.port_list})'
 
     def get_absolute_url(self):
         return reverse('ipam:service', args=[self.pk])
@@ -1058,6 +1061,10 @@ class Service(ChangeLoggedModel, CustomFieldModel):
             self.virtual_machine.name if self.virtual_machine else None,
             self.name,
             self.get_protocol_display(),
-            self.port,
+            self.ports,
             self.description,
         )
+
+    @property
+    def port_list(self):
+        return array_to_string(self.ports)

netbox/ipam/tables.py

@@ -623,11 +623,15 @@ class ServiceTable(BaseTable):
     parent = tables.LinkColumn(
         order_by=('device', 'virtual_machine')
     )
+    ports = tables.TemplateColumn(
+        template_code='{{ record.port_list }}',
+        verbose_name='Ports'
+    )
     tags = TagColumn(
         url_name='ipam:service_list'
     )
 
     class Meta(BaseTable.Meta):
         model = Service
-        fields = ('pk', 'name', 'parent', 'protocol', 'port', 'ipaddresses', 'description', 'tags')
+        fields = ('pk', 'name', 'parent', 'protocol', 'ports', 'ipaddresses', 'description', 'tags')
-        default_columns = ('pk', 'name', 'parent', 'protocol', 'port', 'description')
+        default_columns = ('pk', 'name', 'parent', 'protocol', 'ports', 'description')

netbox/ipam/tests/test_api.py

@@ -428,7 +428,7 @@ class VLANTest(APIViewTestCases.APIViewTestCase):
 
 class ServiceTest(APIViewTestCases.APIViewTestCase):
     model = Service
-    brief_fields = ['id', 'name', 'port', 'protocol', 'url']
+    brief_fields = ['id', 'name', 'ports', 'protocol', 'url']
 
     @classmethod
     def setUpTestData(cls):
@@ -444,9 +444,9 @@ class ServiceTest(APIViewTestCases.APIViewTestCase):
         Device.objects.bulk_create(devices)
 
         services = (
-            Service(device=devices[0], name='Service 1', protocol=ServiceProtocolChoices.PROTOCOL_TCP, port=1),
+            Service(device=devices[0], name='Service 1', protocol=ServiceProtocolChoices.PROTOCOL_TCP, ports=[1]),
-            Service(device=devices[0], name='Service 2', protocol=ServiceProtocolChoices.PROTOCOL_TCP, port=2),
+            Service(device=devices[0], name='Service 2', protocol=ServiceProtocolChoices.PROTOCOL_TCP, ports=[2]),
-            Service(device=devices[0], name='Service 3', protocol=ServiceProtocolChoices.PROTOCOL_TCP, port=3),
+            Service(device=devices[0], name='Service 3', protocol=ServiceProtocolChoices.PROTOCOL_TCP, ports=[3]),
         )
         Service.objects.bulk_create(services)
 
@@ -455,18 +455,18 @@ class ServiceTest(APIViewTestCases.APIViewTestCase):
                 'device': devices[1].pk,
                 'name': 'Service 4',
                 'protocol': ServiceProtocolChoices.PROTOCOL_TCP,
-                'port': 4,
+                'ports': [4],
             },
             {
                 'device': devices[1].pk,
                 'name': 'Service 5',
                 'protocol': ServiceProtocolChoices.PROTOCOL_TCP,
-                'port': 5,
+                'ports': [5],
             },
             {
                 'device': devices[1].pk,
                 'name': 'Service 6',
                 'protocol': ServiceProtocolChoices.PROTOCOL_TCP,
-                'port': 6,
+                'ports': [6],
             },
         ]

netbox/ipam/tests/test_filters.py

@@ -742,12 +742,12 @@ class ServiceTestCase(TestCase):
         VirtualMachine.objects.bulk_create(virtual_machines)
 
         services = (
-            Service(device=devices[0], name='Service 1', protocol=ServiceProtocolChoices.PROTOCOL_TCP, port=1001),
+            Service(device=devices[0], name='Service 1', protocol=ServiceProtocolChoices.PROTOCOL_TCP, ports=[1001]),
-            Service(device=devices[1], name='Service 2', protocol=ServiceProtocolChoices.PROTOCOL_TCP, port=1002),
+            Service(device=devices[1], name='Service 2', protocol=ServiceProtocolChoices.PROTOCOL_TCP, ports=[1002]),
-            Service(device=devices[2], name='Service 3', protocol=ServiceProtocolChoices.PROTOCOL_UDP, port=1003),
+            Service(device=devices[2], name='Service 3', protocol=ServiceProtocolChoices.PROTOCOL_UDP, ports=[1003]),
-            Service(virtual_machine=virtual_machines[0], name='Service 4', protocol=ServiceProtocolChoices.PROTOCOL_TCP, port=2001),
+            Service(virtual_machine=virtual_machines[0], name='Service 4', protocol=ServiceProtocolChoices.PROTOCOL_TCP, ports=[2001]),
-            Service(virtual_machine=virtual_machines[1], name='Service 5', protocol=ServiceProtocolChoices.PROTOCOL_TCP, port=2002),
+            Service(virtual_machine=virtual_machines[1], name='Service 5', protocol=ServiceProtocolChoices.PROTOCOL_TCP, ports=[2002]),
-            Service(virtual_machine=virtual_machines[2], name='Service 6', protocol=ServiceProtocolChoices.PROTOCOL_UDP, port=2003),
+            Service(virtual_machine=virtual_machines[2], name='Service 6', protocol=ServiceProtocolChoices.PROTOCOL_UDP, ports=[2003]),
         )
         Service.objects.bulk_create(services)
 
@@ -764,8 +764,8 @@ class ServiceTestCase(TestCase):
         self.assertEqual(self.filterset(params, self.queryset).qs.count(), 4)
 
     def test_port(self):
-        params = {'port': ['1001', '1002', '1003']}
+        params = {'port': '1001'}
-        self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3)
+        self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1)
 
     def test_device(self):
         devices = Device.objects.all()[:2]

netbox/ipam/tests/test_views.py

@@ -373,9 +373,9 @@ class ServiceTestCase(
         device = Device.objects.create(name='Device 1', site=site, device_type=devicetype, device_role=devicerole)
 
         Service.objects.bulk_create([
-            Service(device=device, name='Service 1', protocol=ServiceProtocolChoices.PROTOCOL_TCP, port=101),
+            Service(device=device, name='Service 1', protocol=ServiceProtocolChoices.PROTOCOL_TCP, ports=[101]),
-            Service(device=device, name='Service 2', protocol=ServiceProtocolChoices.PROTOCOL_TCP, port=102),
+            Service(device=device, name='Service 2', protocol=ServiceProtocolChoices.PROTOCOL_TCP, ports=[102]),
-            Service(device=device, name='Service 3', protocol=ServiceProtocolChoices.PROTOCOL_TCP, port=103),
+            Service(device=device, name='Service 3', protocol=ServiceProtocolChoices.PROTOCOL_TCP, ports=[103]),
         ])
 
         tags = cls.create_tags('Alpha', 'Bravo', 'Charlie')
@@ -385,14 +385,14 @@ class ServiceTestCase(
             'virtual_machine': None,
             'name': 'Service X',
             'protocol': ServiceProtocolChoices.PROTOCOL_TCP,
-            'port': 999,
+            'ports': '104,105',
             'ipaddresses': [],
             'description': 'A new service',
             'tags': [t.pk for t in tags],
         }
 
         cls.csv_data = (
-            "device,name,protocol,port,description",
+            "device,name,protocol,ports,description",
             "Device 1,Service 1,tcp,1,First service",
             "Device 1,Service 2,tcp,2,Second service",
             "Device 1,Service 3,udp,3,Third service",
@@ -400,6 +400,6 @@ class ServiceTestCase(
 
         cls.bulk_edit_data = {
             'protocol': ServiceProtocolChoices.PROTOCOL_UDP,
-            'port': 888,
+            'ports': '106,107',
             'description': 'New description',
         }

netbox/ipam/views.py

@@ -843,9 +843,6 @@ class ServiceEditView(ObjectEditView):
             )
         return obj
 
-    def get_return_url(self, request, service):
-        return service.parent.get_absolute_url()
-
 
 class ServiceBulkImportView(BulkImportView):
     queryset = Service.objects.all()

netbox/templates/ipam/inc/service.html

@@ -1,13 +1,10 @@
 <tr>
-    <td>
+    <td><a href="{{ service.get_absolute_url }}">{{ service.name }}</a></td>
-        <a href="{{ service.get_absolute_url }}">{{ service.name }}</a>
+    <td>{{ service.get_protocol_display }}</td>
-    </td>
+    <td>{{ service.port_list }}</td>
-    <td>
-        {{ service.get_protocol_display }}/{{ service.port }}
-    </td>
     <td>
         {% for ip in service.ipaddresses.all %}
-            <span>{{ ip.address.ip }}</span><br />
+            <a href="{{ ip.get_absolute_url }}">{{ ip.address.ip }}</a><br />
         {% empty %}
             <span class="text-muted">All IPs</span>
         {% endfor %}
@@ -18,7 +15,7 @@
             <i class="fa fa-history"></i>
         </a>
         {% if perms.ipam.change_service %}
-            <a href="{% url 'ipam:service_edit' pk=service.pk %}" class="btn btn-info btn-xs" title="Edit service">
+            <a href="{% url 'ipam:service_edit' pk=service.pk %}?return_url={{ service.parent.get_absolute_url }}" class="btn btn-info btn-xs" title="Edit service">
                 <i class="glyphicon glyphicon-pencil"></i>
             </a>
         {% endif %}

netbox/templates/ipam/service.html

@@ -62,8 +62,8 @@
                     <td>{{ service.get_protocol_display }}</td>
                 </tr>
                 <tr>
-                    <td>Port</td>
+                    <td>Ports</td>
-                    <td>{{ service.port }}</td>
+                    <td>{{ service.port_list }}</td>
                 </tr>
                 <tr>
                     <td>IP Addresses</td>

netbox/templates/ipam/service_edit.html

@@ -25,7 +25,7 @@
                 <label class="col-md-3 control-label required">Port</label>
                 <div class="col-md-9">
                     {{ form.protocol }}
-                    {{ form.port }}
+                    {{ form.ports }}
                 </div>
             </div>
             {% render_field form.ipaddresses %}

netbox/utilities/filters.py

@@ -68,7 +68,6 @@ class TreeNodeMultipleChoiceFilter(django_filters.ModelMultipleChoiceFilter):
     """
     Filters for a set of Models, including all descendant models within a Tree.  Example: [<Region: R1>,<Region: R2>]
     """
-
     def get_filter_predicate(self, v):
         # null value filtering
         if v is None:
@@ -84,7 +83,6 @@ class NullableCharFieldFilter(django_filters.CharFilter):
     """
     Allow matching on null field values by passing a special string used to signify NULL.
     """
-
     def filter(self, qs, value):
         if value != settings.FILTERS_NULL_CHOICE_VALUE:
             return super().filter(qs, value)
@@ -107,6 +105,16 @@ class TagFilter(django_filters.ModelMultipleChoiceFilter):
         super().__init__(*args, **kwargs)
 
 
+class NumericArrayFilter(django_filters.NumberFilter):
+    """
+    Filter based on the presence of an integer within an ArrayField.
+    """
+    def filter(self, qs, value):
+        if value:
+            value = [value]
+        return super().filter(qs, value)
+
+
 #
 # FilterSets
 #

netbox/utilities/utils.py

@@ -1,6 +1,7 @@
 import datetime
 import json
 from collections import OrderedDict
+from itertools import count, groupby
 
 from django.core.serializers import serialize
 from django.db.models import Count, OuterRef, Subquery
@@ -282,6 +283,16 @@ def curry(_curried_func, *args, **kwargs):
     return _curried
 
 
+def array_to_string(array):
+    """
+    Generate an efficient, human-friendly string from a set of integers. Intended for use with ArrayField.
+    For example:
+        [0, 1, 2, 10, 14, 15, 16] => "0-2, 10, 14-16"
+    """
+    group = (list(x) for _, x in groupby(sorted(array), lambda x, c=count(): next(c) - x))
+    return ', '.join('-'.join(map(str, (g[0], g[-1])[:len(g)])) for g in group)
+
+
 #
 # Fake request object
 #