Compare commits

...

11 Commits

Author SHA1 Message Date
Jeremy Stretch
2f64a66c27
Merge a3ee46a32a into 6022433a40 2025-07-09 10:18:55 -07:00
Jeremy Stretch
6022433a40
Closes #19134: Allow negative values for interface TX power (#19847)
Some checks are pending
CI / build (20.x, 3.10) (push) Waiting to run
CI / build (20.x, 3.11) (push) Waiting to run
CI / build (20.x, 3.12) (push) Waiting to run
2025-07-09 10:17:41 -07:00
Jeremy Stretch
878c624eaf
Closes #19722: Extend the object types REST API endpoint (#19826) 2025-07-09 08:43:24 -07:00
Jeremy Stretch
a3ee46a32a Update documentation
Some checks failed
CI / build (20.x, 3.10) (push) Has been cancelled
CI / build (20.x, 3.11) (push) Has been cancelled
CI / build (20.x, 3.12) (push) Has been cancelled
2025-07-09 10:36:18 -04:00
Jeremy Stretch
eb86b9d6b0 Refactor HTML templates 2025-07-09 10:25:44 -04:00
Jeremy Stretch
37fb8ae8ae Test logging 2025-07-09 10:19:44 -04:00
Jeremy Stretch
6b2643c9cd Misc cleanup 2025-07-09 10:09:31 -04:00
Jeremy Stretch
9946ff910c Repurpose RQJobStatusColumn to display job entry level badges 2025-07-09 10:00:49 -04:00
Jeremy Stretch
e95c3825de Deserialize JobLogEntry timestamp 2025-07-09 09:30:38 -04:00
Jeremy Stretch
92a13a4226 Use TZ-aware timestamps 2025-07-09 08:51:03 -04:00
Jeremy Stretch
90e8a61670
Closes #19739: Add a user preference for CSV delimiter in table exports (#19824)
* Closes #19739: Add a user preference for CSV delimiter in table exports

* Pass custom delimiter when exporting entire table
2025-07-08 14:11:40 -05:00
27 changed files with 280 additions and 80 deletions

View File

@ -158,6 +158,7 @@ LOGGING = {
* `netbox.<app>.<model>` - Generic form for model-specific log messages
* `netbox.auth.*` - Authentication events
* `netbox.api.views.*` - Views which handle business logic for the REST API
* `netbox.jobs.*` - Background jobs
* `netbox.reports.*` - Report execution (`module.name`)
* `netbox.scripts.*` - Custom script execution (`module.name`)
* `netbox.views.*` - Views which handle business logic for the web UI

View File

@ -38,6 +38,27 @@ You can schedule the background job from within your code (e.g. from a model's `
This is the human-friendly names of your background job. If omitted, the class name will be used.
### Logging
!!! info "This feature was introduced in NetBox v4.4."
A Python logger is instantiated by the runner for each job. It can be utilized within a job's `run()` method as needed:
```python
def run(self, *args, **kwargs):
obj = MyModel.objects.get(pk=kwargs.get('pk'))
self.logger.info("Retrieved object {obj}")
```
Four of the standard Python logging levels are supported:
* `debug()`
* `info()`
* `warning()`
* `error()`
Log entries recorded using the runner's logger will be saved in the job's log in the database in addition to being processed by other [system logging handlers](../../configuration/system.md#logging).
### Scheduled Jobs
As described above, jobs can be scheduled for immediate execution or at any later time using the `enqueue()` method. However, for management purposes, the `enqueue_once()` method allows a job to be scheduled exactly once avoiding duplicates. If a job is already scheduled for a particular instance, a second one won't be scheduled, respecting thread safety. An example use case would be to schedule a periodic task that is bound to an instance in general, but not to any event of that instance (such as updates). The parameters of the `enqueue_once()` method are identical to those of `enqueue()`.

View File

@ -4,23 +4,31 @@ from django.utils.translation import gettext_lazy as _
from rq.job import JobStatus
__all__ = (
'JOB_LOG_ENTRY_LEVELS',
'RQ_TASK_STATUSES',
)
@dataclass
class Status:
class Badge:
label: str
color: str
RQ_TASK_STATUSES = {
JobStatus.QUEUED: Status(_('Queued'), 'cyan'),
JobStatus.FINISHED: Status(_('Finished'), 'green'),
JobStatus.FAILED: Status(_('Failed'), 'red'),
JobStatus.STARTED: Status(_('Started'), 'blue'),
JobStatus.DEFERRED: Status(_('Deferred'), 'gray'),
JobStatus.SCHEDULED: Status(_('Scheduled'), 'purple'),
JobStatus.STOPPED: Status(_('Stopped'), 'orange'),
JobStatus.CANCELED: Status(_('Cancelled'), 'yellow'),
JobStatus.QUEUED: Badge(_('Queued'), 'cyan'),
JobStatus.FINISHED: Badge(_('Finished'), 'green'),
JobStatus.FAILED: Badge(_('Failed'), 'red'),
JobStatus.STARTED: Badge(_('Started'), 'blue'),
JobStatus.DEFERRED: Badge(_('Deferred'), 'gray'),
JobStatus.SCHEDULED: Badge(_('Scheduled'), 'purple'),
JobStatus.STOPPED: Badge(_('Stopped'), 'orange'),
JobStatus.CANCELED: Badge(_('Cancelled'), 'yellow'),
}
JOB_LOG_ENTRY_LEVELS = {
'debug': Badge(_('Debug'), 'gray'),
'info': Badge(_('Info'), 'blue'),
'warning': Badge(_('Warning'), 'orange'),
'error': Badge(_('Error'), 'red'),
}

View File

@ -3,6 +3,8 @@ import logging
from dataclasses import dataclass, field
from datetime import datetime
from django.utils import timezone
__all__ = (
'JobLogEntry',
)
@ -12,7 +14,7 @@ __all__ = (
class JobLogEntry:
level: str
message: str
timestamp: datetime = field(default_factory=datetime.now)
timestamp: datetime = field(default_factory=timezone.now)
@classmethod
def from_logrecord(cls, record: logging.LogRecord):

View File

@ -7,7 +7,6 @@ from netbox.jobs import JobRunner, system_job
from netbox.search.backends import search_backend
from utilities.proxy import resolve_proxies
from .choices import DataSourceStatusChoices, JobIntervalChoices
from .exceptions import SyncError
from .models import DataSource
logger = logging.getLogger(__name__)
@ -23,19 +22,23 @@ class SyncDataSourceJob(JobRunner):
def run(self, *args, **kwargs):
datasource = DataSource.objects.get(pk=self.job.object_id)
self.logger.debug(f"Found DataSource ID {datasource.pk}")
try:
self.logger.info(f"Syncing data source {datasource}")
datasource.sync()
# Update the search cache for DataFiles belonging to this source
self.logger.debug("Updating search cache for data files")
search_backend.cache(datasource.datafiles.iterator())
except Exception as e:
self.logger.error(f"Error syncing data source: {e}")
DataSource.objects.filter(pk=datasource.pk).update(status=DataSourceStatusChoices.FAILED)
if type(e) is SyncError:
logging.error(e)
raise e
self.logger.info("Syncing completed successfully")
@system_job(interval=JobIntervalChoices.INTERVAL_DAILY)
class SystemHousekeepingJob(JobRunner):

View File

@ -2,6 +2,8 @@ import django.contrib.postgres.fields
import django.core.serializers.json
from django.db import migrations, models
import utilities.json
class Migration(migrations.Migration):
@ -15,6 +17,7 @@ class Migration(migrations.Migration):
name='log_entries',
field=django.contrib.postgres.fields.ArrayField(
base_field=models.JSONField(
decoder=utilities.json.JobLogDecoder,
encoder=django.core.serializers.json.DjangoJSONEncoder
),
blank=True,

View File

@ -1,7 +1,9 @@
from django.contrib.contenttypes.models import ContentType, ContentTypeManager
from django.db.models import Q
from netbox.plugins import PluginConfig
from netbox.registry import registry
from utilities.string import title
__all__ = (
'ObjectType',
@ -48,3 +50,29 @@ class ObjectType(ContentType):
class Meta:
proxy = True
@property
def app_labeled_name(self):
# Override ContentType's "app | model" representation style.
return f"{self.app_verbose_name} > {title(self.model_verbose_name)}"
@property
def app_verbose_name(self):
if model := self.model_class():
return model._meta.app_config.verbose_name
@property
def model_verbose_name(self):
if model := self.model_class():
return model._meta.verbose_name
@property
def model_verbose_name_plural(self):
if model := self.model_class():
return model._meta.verbose_name_plural
@property
def is_plugin_model(self):
if not (model := self.model_class()):
return # Return null if model class is invalid
return isinstance(model._meta.app_config, PluginConfig)

View File

@ -20,6 +20,7 @@ from core.choices import JobStatusChoices
from core.dataclasses import JobLogEntry
from core.models import ObjectType
from core.signals import job_end, job_start
from utilities.json import JobLogDecoder
from utilities.querysets import RestrictedQuerySet
from utilities.rqworker import get_queue_for_model
@ -109,9 +110,10 @@ class Job(models.Model):
unique=True
)
log_entries = ArrayField(
verbose_name=_('log entries'),
base_field=models.JSONField(
encoder=DjangoJSONEncoder,
# TODO: Specify a decoder to handle ISO 8601 timestamps
decoder=JobLogDecoder,
),
blank=True,
default=list,
@ -217,6 +219,13 @@ class Job(models.Model):
# Send signal
job_end.send(self)
def log(self, record: logging.LogRecord):
"""
Record a LogRecord from Python's native logging in the job's log.
"""
entry = JobLogEntry.from_logrecord(record)
self.log_entries.append(asdict(entry))
@classmethod
def enqueue(
cls,
@ -283,10 +292,3 @@ class Job(models.Model):
transaction.on_commit(callback)
return job
def log(self, record: logging.LogRecord):
"""
Record a Python LogRecord in the job's log.
"""
entry = JobLogEntry.from_logrecord(record)
self.log_entries.append(asdict(entry))

View File

@ -1,12 +1,11 @@
import django_tables2 as tables
from django.utils.safestring import mark_safe
from core.constants import RQ_TASK_STATUSES
from netbox.registry import registry
__all__ = (
'BackendTypeColumn',
'RQJobStatusColumn',
'BadgeColumn',
)
@ -23,14 +22,21 @@ class BackendTypeColumn(tables.Column):
return value
class RQJobStatusColumn(tables.Column):
class BadgeColumn(tables.Column):
"""
Render a colored label for the status of an RQ job.
Render a colored badge for a value.
Args:
badges: A dictionary mapping of values to core.constants.Badge instances.
"""
def __init__(self, badges, *args, **kwargs):
super().__init__(*args, **kwargs)
self.badges = badges
def render(self, value):
status = RQ_TASK_STATUSES.get(value)
return mark_safe(f'<span class="badge text-bg-{status.color}">{status.label}</span>')
badge = self.badges.get(value)
return mark_safe(f'<span class="badge text-bg-{badge.color}">{badge.label}</span>')
def value(self, value):
status = RQ_TASK_STATUSES.get(value)
return status.label
badge = self.badges.get(value)
return badge.label

View File

@ -2,7 +2,9 @@ import django_tables2 as tables
from django.utils.translation import gettext_lazy as _
from netbox.tables import BaseTable, NetBoxTable, columns
from ..models import Job
from core.constants import JOB_LOG_ENTRY_LEVELS
from core.models import Job
from core.tables.columns import BadgeColumn
class JobTable(NetBoxTable):
@ -62,10 +64,12 @@ class JobTable(NetBoxTable):
class JobLogEntryTable(BaseTable):
timestamp = tables.Column(
timestamp = columns.DateTimeColumn(
timespec='milliseconds',
verbose_name=_('Time'),
)
level = tables.Column(
level = BadgeColumn(
badges=JOB_LOG_ENTRY_LEVELS,
verbose_name=_('Level'),
)
message = tables.Column(

View File

@ -2,7 +2,8 @@ import django_tables2 as tables
from django.utils.translation import gettext_lazy as _
from django_tables2.utils import A
from core.tables.columns import RQJobStatusColumn
from core.constants import RQ_TASK_STATUSES
from core.tables.columns import BadgeColumn
from netbox.tables import BaseTable, columns
@ -84,7 +85,8 @@ class BackgroundTaskTable(BaseTable):
ended_at = columns.DateTimeColumn(
verbose_name=_("Ended")
)
status = RQJobStatusColumn(
status = BadgeColumn(
badges=RQ_TASK_STATUSES,
verbose_name=_("Status"),
accessor='get_status'
)

View File

@ -188,7 +188,7 @@ class JobView(generic.ObjectView):
class JobLogView(generic.ObjectView):
queryset = Job.objects.all()
actions = (DeleteObject,)
template_name = 'core/job_log.html'
template_name = 'core/job/log.html'
tab = ViewTab(
label=_('Log'),
badge=lambda obj: len(obj.log_entries),

View File

@ -1507,7 +1507,7 @@ class InterfaceFilterForm(PathEndpointFilterForm, DeviceComponentFilterForm):
tx_power = forms.IntegerField(
required=False,
label=_('Transmit power (dBm)'),
min_value=0,
min_value=-40,
max_value=127
)
vrf_id = DynamicModelMultipleChoiceField(

View File

@ -0,0 +1,24 @@
import django.core.validators
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('dcim', '0208_platform_manufacturer_uniqueness'),
]
operations = [
migrations.AlterField(
model_name='interface',
name='tx_power',
field=models.SmallIntegerField(
blank=True,
null=True,
validators=[
django.core.validators.MinValueValidator(-40),
django.core.validators.MaxValueValidator(127)
]
),
),
]

View File

@ -719,10 +719,13 @@ class Interface(ModularComponentModel, BaseInterface, CabledObjectModel, PathEnd
verbose_name=('channel width (MHz)'),
help_text=_("Populated by selected channel (if set)")
)
tx_power = models.PositiveSmallIntegerField(
tx_power = models.SmallIntegerField(
blank=True,
null=True,
validators=(MaxValueValidator(127),),
validators=(
MinValueValidator(-40),
MaxValueValidator(127),
),
verbose_name=_('transmit power (dBm)')
)
poe_mode = models.CharField(

View File

@ -1,7 +1,13 @@
import inspect
from django.urls import NoReverseMatch, reverse
from drf_spectacular.types import OpenApiTypes
from drf_spectacular.utils import extend_schema_field
from rest_framework import serializers
from core.models import ObjectType
from netbox.api.serializers import BaseModelSerializer
from utilities.views import get_viewname
__all__ = (
'ObjectTypeSerializer',
@ -10,7 +16,32 @@ __all__ = (
class ObjectTypeSerializer(BaseModelSerializer):
url = serializers.HyperlinkedIdentityField(view_name='extras-api:objecttype-detail')
app_name = serializers.CharField(source='app_verbose_name', read_only=True)
model_name = serializers.CharField(source='model_verbose_name', read_only=True)
model_name_plural = serializers.CharField(source='model_verbose_name_plural', read_only=True)
is_plugin_model = serializers.BooleanField(read_only=True)
rest_api_endpoint = serializers.SerializerMethodField()
description = serializers.SerializerMethodField()
class Meta:
model = ObjectType
fields = ['id', 'url', 'display', 'app_label', 'model']
fields = [
'id', 'url', 'display', 'app_label', 'app_name', 'model', 'model_name', 'model_name_plural',
'is_plugin_model', 'rest_api_endpoint', 'description',
]
@extend_schema_field(OpenApiTypes.STR)
def get_rest_api_endpoint(self, obj):
if not (model := obj.model_class()):
return
if viewname := get_viewname(model, action='list', rest_api=True):
try:
return reverse(viewname)
except NoReverseMatch:
return
@extend_schema_field(OpenApiTypes.STR)
def get_description(self, obj):
if not (model := obj.model_class()):
return
return inspect.getdoc(model)

View File

@ -44,3 +44,10 @@ CENSOR_TOKEN_CHANGED = '***CHANGED***'
# Placeholder text for empty tables
EMPTY_TABLE_TEXT = 'No results found'
# CSV delimiters
CSV_DELIMITERS = {
'comma': ',',
'semicolon': ';',
'pipe': '|',
}

View File

@ -35,7 +35,9 @@ def system_job(interval):
class JobLogHandler(logging.Handler):
"""
A logging handler which records entries on a Job.
"""
def __init__(self, job, *args, **kwargs):
super().__init__(*args, **kwargs)
self.job = job

View File

@ -72,6 +72,16 @@ PREFERENCES = {
),
description=_('The preferred syntax for displaying generic data within the UI')
),
'csv_delimiter': UserPreference(
label=_('CSV delimiter'),
choices=(
('comma', 'Comma (,)'),
('semicolon', 'Semicolon (;)'),
('pipe', 'Pipe (|)'),
),
default='comma',
description=_('The character used to separate fields in CSV data')
),
}

View File

@ -11,7 +11,10 @@ from core.choices import JobStatusChoices
class TestJobRunner(JobRunner):
def run(self, *args, **kwargs):
pass
self.logger.debug("Debug message")
self.logger.info("Info message")
self.logger.warning("Warning message")
self.logger.error("Error message")
class JobRunnerTestCase(TestCase):
@ -47,8 +50,16 @@ class JobRunnerTest(JobRunnerTestCase):
def test_handle(self):
job = TestJobRunner.enqueue(immediate=True)
# Check job status
self.assertEqual(job.status, JobStatusChoices.STATUS_COMPLETED)
# Check logging
self.assertEqual(len(job.log_entries), 4)
self.assertEqual(job.log_entries[0]['message'], "Debug message")
self.assertEqual(job.log_entries[1]['message'], "Info message")
self.assertEqual(job.log_entries[2]['message'], "Warning message")
self.assertEqual(job.log_entries[3]['message'], "Error message")
def test_handle_errored(self):
class ErroredJobRunner(TestJobRunner):
EXP = Exception('Test error')

View File

@ -15,7 +15,6 @@ from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from django.utils.safestring import mark_safe
from django.utils.translation import gettext as _
from django_tables2.export import TableExport
from mptt.models import MPTTModel
from core.models import ObjectType
@ -25,6 +24,7 @@ from extras.models import CustomField, ExportTemplate
from netbox.object_actions import AddObject, BulkDelete, BulkEdit, BulkExport, BulkImport, BulkRename
from utilities.error_handlers import handle_protectederror
from utilities.exceptions import AbortRequest, AbortTransaction, PermissionsViolation
from utilities.export import TableExport
from utilities.forms import BulkRenameForm, ConfirmationForm, restrict_form_fields
from utilities.forms.bulk_import import BulkImportForm
from utilities.htmx import htmx_partial
@ -77,7 +77,7 @@ class ObjectListView(BaseMultiObjectView, ActionsMixin, TableMixin):
return '---\n'.join(yaml_data)
def export_table(self, table, columns=None, filename=None):
def export_table(self, table, columns=None, filename=None, delimiter=None):
"""
Export all table data in CSV format.
@ -86,6 +86,7 @@ class ObjectListView(BaseMultiObjectView, ActionsMixin, TableMixin):
columns: A list of specific columns to include. If None, all columns will be exported.
filename: The name of the file attachment sent to the client. If None, will be determined automatically
from the queryset model name.
delimiter: The character used to separate columns (a comma is used by default)
"""
exclude_columns = {'pk', 'actions'}
if columns:
@ -96,7 +97,8 @@ class ObjectListView(BaseMultiObjectView, ActionsMixin, TableMixin):
exporter = TableExport(
export_format=TableExport.CSV,
table=table,
exclude_columns=exclude_columns
exclude_columns=exclude_columns,
delimiter=delimiter,
)
return exporter.response(
filename=filename or f'netbox_{self.queryset.model._meta.verbose_name_plural}.csv'
@ -159,7 +161,8 @@ class ObjectListView(BaseMultiObjectView, ActionsMixin, TableMixin):
if request.GET['export'] == 'table':
table = self.get_table(self.queryset, request, has_table_actions)
columns = [name for name, _ in table.selected_columns]
return self.export_table(table, columns)
delimiter = request.user.config.get('csv_delimiter')
return self.export_table(table, columns, delimiter=delimiter)
# Render an ExportTemplate
elif request.GET['export']:
@ -176,7 +179,8 @@ class ObjectListView(BaseMultiObjectView, ActionsMixin, TableMixin):
# Fall back to default table/YAML export
else:
table = self.get_table(self.queryset, request, has_table_actions)
return self.export_table(table)
delimiter = request.user.config.get('csv_delimiter')
return self.export_table(table, delimiter=delimiter)
# Render the objects table
table = self.get_table(self.queryset, request, has_table_actions)

View File

@ -1,27 +1,6 @@
{% extends 'generic/object.html' %}
{% load buttons %}
{% load helpers %}
{% load perms %}
{% extends 'core/job/base.html' %}
{% load i18n %}
{% block breadcrumbs %}
{{ block.super }}
{% if object.object %}
<li class="breadcrumb-item">
<a href="{% url 'core:job_list' %}?object_type={{ object.object_type_id }}">{{ object.object|meta:"verbose_name_plural"|bettertitle }}</a>
</li>
{% with parent_jobs_viewname=object.object|viewname:"jobs" %}
<li class="breadcrumb-item">
<a href="{% url parent_jobs_viewname pk=object.object.pk %}">{{ object.object }}</a>
</li>
{% endwith %}
{% else %}
<li class="breadcrumb-item">
<a href="{% url 'core:job_list' %}?name={{ object.name|urlencode }}">{{ object.name }}</a>
</li>
{% endif %}
{% endblock breadcrumbs %}
{% block content %}
<div class="row mb-3">
<div class="col col-12 col-md-6">

View File

@ -3,7 +3,6 @@
{% load helpers %}
{% load perms %}
{% load i18n %}
{% load render_table from django_tables2 %}
{% block breadcrumbs %}
{{ block.super }}
@ -22,13 +21,3 @@
</li>
{% endif %}
{% endblock breadcrumbs %}
{% block content %}
<div class="row mb-3">
<div class="col">
<div class="card">
{% render_table table %}
</div>
</div>
</div>
{% endblock %}

View File

@ -0,0 +1,12 @@
{% extends 'core/job/base.html' %}
{% load render_table from django_tables2 %}
{% block content %}
<div class="row mb-3">
<div class="col">
<div class="card">
{% render_table table %}
</div>
</div>
</div>
{% endblock %}

View File

@ -62,7 +62,7 @@ class UserConfigForm(forms.ModelForm, metaclass=UserConfigFormMetaclass):
'ui.tables.striping',
name=_('User Interface')
),
FieldSet('data_format', name=_('Miscellaneous')),
FieldSet('data_format', 'csv_delimiter', name=_('Miscellaneous')),
)
# List of clearable preferences
pk = forms.MultipleChoiceField(

View File

@ -0,0 +1,26 @@
from django.utils.translation import gettext_lazy as _
from django_tables2.export import TableExport as TableExport_
from netbox.constants import CSV_DELIMITERS
__all__ = (
'TableExport',
)
class TableExport(TableExport_):
"""
A subclass of django-tables2's TableExport class which allows us to specify a delimiting
characters for CSV exports.
"""
def __init__(self, *args, delimiter=None, **kwargs):
if delimiter and delimiter not in CSV_DELIMITERS.keys():
raise ValueError(_("Invalid delimiter name: {name}").format(name=delimiter))
self.delimiter = delimiter or 'comma'
super().__init__(*args, **kwargs)
def export(self):
if self.format == self.CSV and self.delimiter is not None:
delimiter = CSV_DELIMITERS[self.delimiter]
return self.dataset.export(self.format, delimiter=delimiter)
return super().export()

View File

@ -1,10 +1,14 @@
import decimal
import json
from django.core.serializers.json import DjangoJSONEncoder
from utilities.datetime import datetime_from_timestamp
__all__ = (
'ConfigJSONEncoder',
'CustomFieldJSONEncoder',
'JobLogDecoder',
)
@ -29,3 +33,21 @@ class ConfigJSONEncoder(DjangoJSONEncoder):
return type(o).__name__
return super().default(o)
class JobLogDecoder(json.JSONDecoder):
"""
Deserialize JobLogEntry timestamps.
"""
def __init__(self, *args, **kwargs):
kwargs['object_hook'] = self._deserialize_entry
super().__init__(*args, **kwargs)
def _deserialize_entry(self, obj: dict) -> dict:
if obj.get('timestamp'):
# Deserialize a timestamp string to a native datetime object
try:
obj['timestamp'] = datetime_from_timestamp(obj['timestamp'])
except ValueError:
pass
return obj