Merge branch 'develop-2.2' into develop-2.1

This commit is contained in:
Jeremy Stretch
2017-09-29 12:30:36 -04:00
committed by GitHub
105 changed files with 4662 additions and 562 deletions

View File

@@ -7,6 +7,18 @@ from django.utils.safestring import mark_safe
from .models import CustomField, CustomFieldChoice, Graph, ExportTemplate, TopologyMap, UserAction
def order_content_types(field):
"""
Order the list of available ContentTypes by application
"""
queryset = field.queryset.order_by('app_label', 'model')
field.choices = [(ct.pk, '{} > {}'.format(ct.app_label, ct.name)) for ct in queryset]
#
# Custom fields
#
class CustomFieldForm(forms.ModelForm):
class Meta:
@@ -16,9 +28,7 @@ class CustomFieldForm(forms.ModelForm):
def __init__(self, *args, **kwargs):
super(CustomFieldForm, self).__init__(*args, **kwargs)
# Organize the available ContentTypes
queryset = self.fields['obj_type'].queryset.order_by('app_label', 'model')
self.fields['obj_type'].choices = [(ct.pk, '{} > {}'.format(ct.app_label, ct.name)) for ct in queryset]
order_content_types(self.fields['obj_type'])
class CustomFieldChoiceAdmin(admin.TabularInline):
@@ -36,16 +46,43 @@ class CustomFieldAdmin(admin.ModelAdmin):
return ', '.join([ct.name for ct in obj.obj_type.all()])
#
# Graphs
#
@admin.register(Graph)
class GraphAdmin(admin.ModelAdmin):
list_display = ['name', 'type', 'weight', 'source']
#
# Export templates
#
class ExportTemplateForm(forms.ModelForm):
class Meta:
model = ExportTemplate
exclude = []
def __init__(self, *args, **kwargs):
super(ExportTemplateForm, self).__init__(*args, **kwargs)
# Format ContentType choices
order_content_types(self.fields['content_type'])
self.fields['content_type'].choices.insert(0, ('', '---------'))
@admin.register(ExportTemplate)
class ExportTemplateAdmin(admin.ModelAdmin):
list_display = ['name', 'content_type', 'description', 'mime_type', 'file_extension']
form = ExportTemplateForm
#
# Topology maps
#
@admin.register(TopologyMap)
class TopologyMapAdmin(admin.ModelAdmin):
list_display = ['name', 'slug', 'site']
@@ -54,6 +91,10 @@ class TopologyMapAdmin(admin.ModelAdmin):
}
#
# User actions
#
@admin.register(UserAction)
class UserActionAdmin(admin.ModelAdmin):
actions = None

View File

@@ -7,7 +7,7 @@ from rest_framework import serializers
from dcim.api.serializers import NestedDeviceSerializer, NestedRackSerializer, NestedSiteSerializer
from dcim.models import Device, Rack, Site
from extras.models import (
ACTION_CHOICES, ExportTemplate, Graph, GRAPH_TYPE_CHOICES, ImageAttachment, TopologyMap, UserAction,
ACTION_CHOICES, ExportTemplate, Graph, GRAPH_TYPE_CHOICES, ImageAttachment, ReportResult, TopologyMap, UserAction,
)
from users.api.serializers import NestedUserSerializer
from utilities.api import ChoiceFieldSerializer, ContentTypeFieldSerializer, ValidatedModelSerializer
@@ -127,6 +127,41 @@ class WritableImageAttachmentSerializer(ValidatedModelSerializer):
return data
#
# Reports
#
class ReportResultSerializer(serializers.ModelSerializer):
class Meta:
model = ReportResult
fields = ['created', 'user', 'failed', 'data']
class NestedReportResultSerializer(serializers.ModelSerializer):
url = serializers.HyperlinkedIdentityField(
view_name='extras-api:report-detail',
lookup_field='report',
lookup_url_kwarg='pk'
)
class Meta:
model = ReportResult
fields = ['url', 'created', 'user', 'failed']
class ReportSerializer(serializers.Serializer):
module = serializers.CharField(max_length=255)
name = serializers.CharField(max_length=255)
description = serializers.CharField(max_length=255, required=False)
test_methods = serializers.ListField(child=serializers.CharField(max_length=255))
result = NestedReportResultSerializer()
class ReportDetailSerializer(ReportSerializer):
result = ReportResultSerializer()
#
# User actions
#

View File

@@ -28,6 +28,9 @@ router.register(r'topology-maps', views.TopologyMapViewSet)
# Image attachments
router.register(r'image-attachments', views.ImageAttachmentViewSet)
# Reports
router.register(r'reports', views.ReportViewSet, base_name='report')
# Recent activity
router.register(r'recent-activity', views.RecentActivityViewSet)

View File

@@ -1,14 +1,17 @@
from __future__ import unicode_literals
from rest_framework.decorators import detail_route
from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet
from rest_framework.exceptions import PermissionDenied
from rest_framework.response import Response
from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet, ViewSet
from django.contrib.contenttypes.models import ContentType
from django.http import HttpResponse
from django.http import Http404, HttpResponse
from django.shortcuts import get_object_or_404
from extras import filters
from extras.models import ExportTemplate, Graph, ImageAttachment, TopologyMap, UserAction
from extras.models import ExportTemplate, Graph, ImageAttachment, ReportResult, TopologyMap, UserAction
from extras.reports import get_report, get_reports
from utilities.api import WritableSerializerMixin
from . import serializers
@@ -88,6 +91,77 @@ class ImageAttachmentViewSet(WritableSerializerMixin, ModelViewSet):
write_serializer_class = serializers.WritableImageAttachmentSerializer
class ReportViewSet(ViewSet):
_ignore_model_permissions = True
exclude_from_schema = True
lookup_value_regex = '[^/]+' # Allow dots
def _retrieve_report(self, pk):
# Read the PK as "<module>.<report>"
if '.' not in pk:
raise Http404
module_name, report_name = pk.split('.', 1)
# Raise a 404 on an invalid Report module/name
report = get_report(module_name, report_name)
if report is None:
raise Http404
return report
def list(self, request):
"""
Compile all reports and their related results (if any). Result data is deferred in the list view.
"""
report_list = []
# Iterate through all available Reports.
for module_name, reports in get_reports():
for report in reports:
# Attach the relevant ReportResult (if any) to each Report.
report.result = ReportResult.objects.filter(report=report.full_name).defer('data').first()
report_list.append(report)
serializer = serializers.ReportSerializer(report_list, many=True, context={
'request': request,
})
return Response(serializer.data)
def retrieve(self, request, pk):
"""
Retrieve a single Report identified as "<module>.<report>".
"""
# Retrieve the Report and ReportResult, if any.
report = self._retrieve_report(pk)
report.result = ReportResult.objects.filter(report=report.full_name).first()
serializer = serializers.ReportDetailSerializer(report)
return Response(serializer.data)
@detail_route(methods=['post'])
def run(self, request, pk):
"""
Run a Report and create a new ReportResult, overwriting any previous result for the Report.
"""
# Check that the user has permission to run reports.
if not request.user.has_perm('extras.add_reportresult'):
raise PermissionDenied("This user does not have permission to run reports.")
# Retrieve and run the Report. This will create a new ReportResult.
report = self._retrieve_report(pk)
report.run()
serializer = serializers.ReportDetailSerializer(report)
return Response(serializer.data)
class RecentActivityViewSet(ReadOnlyModelViewSet):
"""
List all UserActions to provide a log of recent activity.

View File

@@ -3,10 +3,11 @@ from __future__ import unicode_literals
# Models which support custom fields
CUSTOMFIELD_MODELS = (
'provider', 'circuit', # Circuits
'site', 'rack', 'devicetype', 'device', # DCIM
'aggregate', 'prefix', 'ipaddress', 'vlan', 'vrf', # IPAM
'provider', 'circuit', # Circuits
'tenant', # Tenants
'tenant', # Tenancy
'cluster', 'virtualmachine', # Virtualization
)
# Custom field types
@@ -37,11 +38,12 @@ GRAPH_TYPE_CHOICES = (
# Models which support export templates
EXPORTTEMPLATE_MODELS = [
'provider', 'circuit', # Circuits
'site', 'region', 'rack', 'rackgroup', 'manufacturer', 'devicetype', 'device', # DCIM
'consoleport', 'powerport', 'interfaceconnection', # DCIM
'aggregate', 'prefix', 'ipaddress', 'vlan', # IPAM
'provider', 'circuit', # Circuits
'tenant', # Tenants
'tenant', # Tenancy
'cluster', 'virtualmachine', # Virtualization
]
# User action types
@@ -61,3 +63,17 @@ ACTION_CHOICES = (
(ACTION_DELETE, 'deleted'),
(ACTION_BULK_DELETE, 'bulk deleted'),
)
# Report logging levels
LOG_DEFAULT = 0
LOG_SUCCESS = 10
LOG_INFO = 20
LOG_WARNING = 30
LOG_FAILURE = 40
LOG_LEVEL_CODES = {
LOG_DEFAULT: 'default',
LOG_SUCCESS: 'success',
LOG_INFO: 'info',
LOG_WARNING: 'warning',
LOG_FAILURE: 'failure',
}

View File

@@ -11,7 +11,7 @@ from django.core.management.base import BaseCommand
from django.db.models import Model
APPS = ['circuits', 'dcim', 'extras', 'ipam', 'secrets', 'tenancy', 'users']
APPS = ['circuits', 'dcim', 'extras', 'ipam', 'secrets', 'tenancy', 'users', 'virtualization']
BANNER_TEXT = """### NetBox interactive shell ({node})
### Python {python} | Django {django} | NetBox {netbox}

View File

@@ -0,0 +1,48 @@
from __future__ import unicode_literals
from django.core.management.base import BaseCommand
from django.utils import timezone
from extras.models import ReportResult
from extras.reports import get_reports
class Command(BaseCommand):
help = "Run a report to validate data in NetBox"
def add_arguments(self, parser):
parser.add_argument('reports', nargs='+', help="Report(s) to run")
# parser.add_argument('--verbose', action='store_true', default=False, help="Print all logs")
def handle(self, *args, **options):
# Gather all available reports
reports = get_reports()
# Run reports
for module_name, report_list in reports:
for report in report_list:
if module_name in options['reports'] or report.full_namel in options['reports']:
# Run the report and create a new ReportResult
self.stdout.write(
"[{:%H:%M:%S}] Running {}...".format(timezone.now(), report.full_name)
)
report.run()
# Report on success/failure
status = self.style.ERROR('FAILED') if report.failed else self.style.SUCCESS('SUCCESS')
for test_name, attrs in report.result.data.items():
self.stdout.write(
"\t{}: {} success, {} info, {} warning, {} failure".format(
test_name, attrs['success'], attrs['info'], attrs['warning'], attrs['failure']
)
)
self.stdout.write(
"[{:%H:%M:%S}] {}: {}".format(timezone.now(), report.full_name, status)
)
# Wrap things up
self.stdout.write(
"[{:%H:%M:%S}] Finished".format(timezone.now())
)

View File

@@ -0,0 +1,33 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.4 on 2017-09-26 21:25
from __future__ import unicode_literals
from django.conf import settings
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('extras', '0007_unicode_literals'),
]
operations = [
migrations.CreateModel(
name='ReportResult',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('report', models.CharField(max_length=255, unique=True)),
('created', models.DateTimeField(auto_now_add=True)),
('failed', models.BooleanField()),
('data', django.contrib.postgres.fields.jsonb.JSONField()),
('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
],
options={
'ordering': ['report'],
},
),
]

View File

@@ -6,6 +6,7 @@ import graphviz
from django.contrib.auth.models import User
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.contrib.postgres.fields import JSONField
from django.core.validators import ValidationError
from django.db import models
from django.db.models import Q
@@ -388,6 +389,24 @@ class ImageAttachment(models.Model):
return None
#
# Report results
#
class ReportResult(models.Model):
"""
This model stores the results from running a user-defined report.
"""
report = models.CharField(max_length=255, unique=True)
created = models.DateTimeField(auto_now_add=True)
user = models.ForeignKey(User, on_delete=models.SET_NULL, related_name='+', blank=True, null=True)
failed = models.BooleanField()
data = JSONField()
class Meta:
ordering = ['report']
#
# User actions
#

178
netbox/extras/reports.py Normal file
View File

@@ -0,0 +1,178 @@
from collections import OrderedDict
import importlib
import inspect
import pkgutil
from django.utils import timezone
from .constants import LOG_DEFAULT, LOG_FAILURE, LOG_INFO, LOG_LEVEL_CODES, LOG_SUCCESS, LOG_WARNING
from .models import ReportResult
import reports as custom_reports
def is_report(obj):
"""
Returns True if the given object is a Report.
"""
if obj in Report.__subclasses__():
return True
return False
def get_report(module_name, report_name):
"""
Return a specific report from within a module.
"""
module = importlib.import_module('reports.{}'.format(module_name))
report = getattr(module, report_name, None)
if report is None:
return None
return report()
def get_reports():
"""
Compile a list of all reports available across all modules in the reports path. Returns a list of tuples:
[
(module_name, (report, report, report, ...)),
(module_name, (report, report, report, ...)),
...
]
"""
module_list = []
# Iterate through all modules within the reports path. These are the user-defined files in which reports are
# defined.
for importer, module_name, is_pkg in pkgutil.walk_packages(custom_reports.__path__):
module = importlib.import_module('reports.{}'.format(module_name))
report_list = [cls() for _, cls in inspect.getmembers(module, is_report)]
module_list.append((module_name, report_list))
return module_list
class Report(object):
"""
NetBox users can extend this object to write custom reports to be used for validating data within NetBox. Each
report must have one or more test methods named `test_*`.
The `_results` attribute of a completed report will take the following form:
{
'test_bar': {
'failures': 42,
'log': [
(<datetime>, <level>, <object>, <message>),
...
]
},
'test_foo': {
'failures': 0,
'log': [
(<datetime>, <level>, <object>, <message>),
...
]
}
}
"""
description = None
def __init__(self):
self._results = OrderedDict()
self.active_test = None
self.failed = False
# Compile test methods and initialize results skeleton
test_methods = []
for method in dir(self):
if method.startswith('test_') and callable(getattr(self, method)):
test_methods.append(method)
self._results[method] = OrderedDict([
('success', 0),
('info', 0),
('warning', 0),
('failure', 0),
('log', []),
])
if not test_methods:
raise Exception("A report must contain at least one test method.")
self.test_methods = test_methods
@property
def module(self):
return self.__module__.rsplit('.', 1)[1]
@property
def name(self):
return self.__class__.__name__
@property
def full_name(self):
return '.'.join([self.module, self.name])
def _log(self, obj, message, level=LOG_DEFAULT):
"""
Log a message from a test method. Do not call this method directly; use one of the log_* wrappers below.
"""
if level not in LOG_LEVEL_CODES:
raise Exception("Unknown logging level: {}".format(level))
self._results[self.active_test]['log'].append((
timezone.now().isoformat(),
LOG_LEVEL_CODES.get(level),
str(obj) if obj else None,
obj.get_absolute_url() if getattr(obj, 'get_absolute_url', None) else None,
message,
))
def log(self, message):
"""
Log a message which is not associated with a particular object.
"""
self._log(None, message, level=LOG_DEFAULT)
def log_success(self, obj, message=None):
"""
Record a successful test against an object. Logging a message is optional.
"""
if message:
self._log(obj, message, level=LOG_SUCCESS)
self._results[self.active_test]['success'] += 1
def log_info(self, obj, message):
"""
Log an informational message.
"""
self._log(obj, message, level=LOG_INFO)
self._results[self.active_test]['info'] += 1
def log_warning(self, obj, message):
"""
Log a warning.
"""
self._log(obj, message, level=LOG_WARNING)
self._results[self.active_test]['warning'] += 1
def log_failure(self, obj, message):
"""
Log a failure. Calling this method will automatically mark the report as failed.
"""
self._log(obj, message, level=LOG_FAILURE)
self._results[self.active_test]['failure'] += 1
self.failed = True
def run(self):
"""
Run the report and return its results. Each test method will be executed in order.
"""
for method_name in self.test_methods:
self.active_test = method_name
test_method = getattr(self, method_name)
test_method()
# Delete any previous ReportResult and create a new one to record the result.
ReportResult.objects.filter(report=self.full_name).delete()
result = ReportResult(report=self.full_name, failed=self.failed, data=self._results)
result.save()
self.result = result

View File

@@ -20,19 +20,6 @@ class RPCClient(object):
except AttributeError:
raise Exception("Specified device ({}) does not have a primary IP defined.".format(device))
def get_lldp_neighbors(self):
"""
Returns a list of dictionaries, each representing an LLDP neighbor adjacency.
{
'local-interface': <str>,
'name': <str>,
'remote-interface': <str>,
'chassis-id': <str>,
}
"""
raise NotImplementedError("Feature not implemented for this platform.")
def get_inventory(self):
"""
Returns a dictionary representing the device chassis and installed inventory items.
@@ -123,30 +110,6 @@ class JunosNC(RPCClient):
# Close the connection to the device
self.manager.close_session()
def get_lldp_neighbors(self):
rpc_reply = self.manager.dispatch('get-lldp-neighbors-information')
lldp_neighbors_raw = xmltodict.parse(rpc_reply.xml)['rpc-reply']['lldp-neighbors-information']['lldp-neighbor-information']
result = []
for neighbor_raw in lldp_neighbors_raw:
neighbor = dict()
neighbor['local-interface'] = neighbor_raw.get('lldp-local-port-id')
name = neighbor_raw.get('lldp-remote-system-name')
if name:
neighbor['name'] = name.split('.')[0] # Split hostname from domain if one is present
else:
neighbor['name'] = ''
try:
neighbor['remote-interface'] = neighbor_raw['lldp-remote-port-description']
except KeyError:
# Older versions of Junos report on interface ID instead of description
neighbor['remote-interface'] = neighbor_raw.get('lldp-remote-port-id')
neighbor['chassis-id'] = neighbor_raw.get('lldp-remote-chassis-id')
result.append(neighbor)
return result
def get_inventory(self):
def glean_items(node, depth=0):

View File

@@ -12,4 +12,9 @@ urlpatterns = [
url(r'^image-attachments/(?P<pk>\d+)/edit/$', views.ImageAttachmentEditView.as_view(), name='imageattachment_edit'),
url(r'^image-attachments/(?P<pk>\d+)/delete/$', views.ImageAttachmentDeleteView.as_view(), name='imageattachment_delete'),
# Reports
url(r'^reports/$', views.ReportListView.as_view(), name='report_list'),
url(r'^reports/(?P<name>[^/]+\.[^/]+)/$', views.ReportView.as_view(), name='report'),
url(r'^reports/(?P<name>[^/]+\.[^/]+)/run/$', views.ReportRunView.as_view(), name='report_run'),
]

View File

@@ -1,17 +1,27 @@
from __future__ import unicode_literals
from django.contrib.auth.mixins import PermissionRequiredMixin
from django.shortcuts import get_object_or_404
from django.contrib import messages
from django.http import Http404
from django.shortcuts import get_object_or_404, redirect, render
from django.utils.safestring import mark_safe
from django.views.generic import View
from utilities.forms import ConfirmationForm
from utilities.views import ObjectDeleteView, ObjectEditView
from .forms import ImageAttachmentForm
from .models import ImageAttachment
from .models import ImageAttachment, ReportResult, UserAction
from .reports import get_report, get_reports
#
# Image attachments
#
class ImageAttachmentEditView(PermissionRequiredMixin, ObjectEditView):
permission_required = 'extras.change_imageattachment'
model = ImageAttachment
form_class = ImageAttachmentForm
model_form = ImageAttachmentForm
def alter_obj(self, imageattachment, request, args, kwargs):
if not imageattachment.pk:
@@ -30,3 +40,79 @@ class ImageAttachmentDeleteView(PermissionRequiredMixin, ObjectDeleteView):
def get_return_url(self, request, imageattachment):
return imageattachment.parent.get_absolute_url()
#
# Reports
#
class ReportListView(View):
"""
Retrieve all of the available reports from disk and the recorded ReportResult (if any) for each.
"""
def get(self, request):
reports = get_reports()
results = {r.report: r for r in ReportResult.objects.all()}
ret = []
for module, report_list in reports:
module_reports = []
for report in report_list:
report.result = results.get(report.full_name, None)
module_reports.append(report)
ret.append((module, module_reports))
return render(request, 'extras/report_list.html', {
'reports': ret,
})
class ReportView(View):
"""
Display a single Report and its associated ReportResult (if any).
"""
def get(self, request, name):
# Retrieve the Report by "<module>.<report>"
module_name, report_name = name.split('.')
report = get_report(module_name, report_name)
if report is None:
raise Http404
# Attach the ReportResult (if any)
report.result = ReportResult.objects.filter(report=report.full_name).first()
return render(request, 'extras/report.html', {
'report': report,
'run_form': ConfirmationForm(),
})
class ReportRunView(PermissionRequiredMixin, View):
"""
Run a Report and record a new ReportResult.
"""
permission_required = 'extras.add_reportresult'
def post(self, request, name):
# Retrieve the Report by "<module>.<report>"
module_name, report_name = name.split('.')
report = get_report(module_name, report_name)
if report is None:
raise Http404
form = ConfirmationForm(request.POST)
if form.is_valid():
# Run the Report. A new ReportResult is created.
report.run()
result = 'failed' if report.failed else 'passed'
msg = "Ran report {} ({})".format(report.full_name, result)
messages.success(request, mark_safe(msg))
UserAction.objects.log_create(request.user, report.result, msg)
return redirect('extras:report', name=report.full_name)