Build update (#59)

* update dockerfile and docker-compose file to build from alpine 3.13 image
* podman update

.gitignore

@@ -19,7 +19,7 @@ netbox.pid
 .vscode
 .venv
 .tox
-!docker/configuration/gunicorn_config.py
+!docker_old/configuration/gunicorn_config.py
 .idea
 .coverage
 .vscode

.jenkins

@@ -1,7 +1,7 @@
 #!/usr/bin/env groovy
 
 // Include this shared CI repository to load script helpers and libraries.
-library identifier: 'vapor@1.16.0', retriever: modernSCM([
+library identifier: 'vapor@1.20.1', retriever: modernSCM([
   $class: 'GitSCMSource',
   remote: 'https://github.com/vapor-ware/ci-shared.git',
   credentialsId: 'vio-bot-gh',
@@ -9,7 +9,7 @@ library identifier: 'vapor@1.16.0', retriever: modernSCM([
 
 
 pythonPipeline([
-  'image': 'vaporio/netbox',
+  'image': 'docker.io/vaporio/netbox',
   'pythonVersion': '3.7',
   'skipDocs': true,
   'skipLint': true,
@@ -23,7 +23,7 @@ spec:
   - name: dockerhub-vaporvecrobot
   containers:
   - name: python
-    image: vaporio/jenkins-agent-python37:master
+    image: docker.io/vaporio/jenkins-agent-python37:master
     imagePullPolicy: Always
     command:
     - cat
@@ -33,7 +33,7 @@ spec:
         memory: 1Gi
         cpu: 500m
   - name: deploy
-    image: vaporio/deployment-tools:latest
+    image: docker.io/vaporio/deployment-tools:latest
     imagePullPolicy: Always
     command:
     - cat

Dockerfile

@@ -1,78 +1,84 @@
-FROM vaporio/python:3.7 as builder
+FROM alpine:3.13 as builder
 
-RUN apt-get update -qy \
+RUN apk add --no-cache \
- && apt-get install -y \
+      bash \
-      libsasl2-dev \
+      build-base \
+      cargo \
+      ca-certificates \
+      cyrus-sasl-dev \
       graphviz \
-      libjpeg-dev \
+      jpeg-dev \
+      libevent-dev \
       libffi-dev \
-      libxml2-dev \
+      libressl-dev \
-      libxslt1-dev \
+      libxslt-dev \
-      libldap2-dev \
+      musl-dev \
-      libpq-dev \
+      openldap-dev \
-      ttf-ubuntu-font-family \
+      postgresql-dev \
- && rm -rf /var/lib/apt/lists/*
+      py3-pip \
-
+      python3-dev \
-WORKDIR /install
+  && python3 -m venv /opt/netbox/venv \
-
+  && /opt/netbox/venv/bin/python3 -m pip install --upgrade \
-RUN pip install --prefix="/install" --no-warn-script-location \
+      pip \
-# gunicorn is used for launching netbox
+      setuptools \
-      gunicorn \
+      wheel
-      greenlet \
-      eventlet \
-# napalm is used for gathering information from network devices
-      napalm \
-# ruamel is used in startup_scripts
-      'ruamel.yaml>=0.15,<0.16' \
-# django-storages was introduced in 2.7 and is optional
-      django-storages
 
 ARG NETBOX_PATH=.
-COPY ${NETBOX_PATH}/requirements.txt /
+COPY ${NETBOX_PATH}/requirements.txt requirements.extras.txt /
-COPY ${NETBOX_PATH}/requirements.extras.txt /
+RUN /opt/netbox/venv/bin/pip install \
-RUN pip install --prefix="/install" --no-warn-script-location -r /requirements.txt -r /requirements.extras.txt
+      -r /requirements.txt \
+      -r /requirements.extras.txt
 
-FROM vaporio/python:3.7-slim
+###
+# Main stage
+###
 
-RUN apt-get update -qy \
+FROM alpine:3.13 as main
- && apt-get install -y \
+
-      libsasl2-dev \
+
+RUN apk add --no-cache \
+      bash \
+      ca-certificates \
+      curl \
       graphviz \
-      libjpeg-dev \
+      libevent \
-      libffi-dev \
+      libffi \
-      libxml2-dev \
+      libjpeg-turbo \
-      libxslt1-dev \
+      libressl \
-      libldap2-dev \
+      libxslt \
-      libpq-dev \
+      postgresql-libs \
+      python3 \
+      py3-pip \
       ttf-ubuntu-font-family \
- && rm -rf /var/lib/apt/lists/*
+      unit \
+      unit-python3
 
 WORKDIR /opt
 
-COPY --from=builder /install /usr/local
+COPY --from=builder /opt/netbox/venv /opt/netbox/venv
 
 ARG NETBOX_PATH=.
 COPY ${NETBOX_PATH} /opt/netbox
 
 COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
-COPY docker/configuration/gunicorn_config.py /etc/netbox/config/
-COPY docker/nginx.conf /etc/netbox-nginx/nginx.conf
 COPY docker/docker-entrypoint.sh /opt/netbox/docker-entrypoint.sh
+COPY docker/launch-netbox.sh /opt/netbox/launch-netbox.sh
 COPY docker/startup_scripts/ /opt/netbox/startup_scripts/
 COPY docker/initializers/ /opt/netbox/initializers/
-COPY docker/configuration/configuration.py /etc/netbox/config/configuration.py
+COPY docker/configuration/ /etc/netbox/config/
+COPY docker/nginx-unit.json /etc/unit/
 
 WORKDIR /opt/netbox/netbox
 
-RUN mkdir -p static && chmod g+w static media
+# Must set permissions for '/opt/netbox/netbox/media' directory
+# to g+w so that pictures can be uploaded to netbox.
+RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ \
+      && chmod -R g+w media /opt/unit/ \
+      && SECRET_KEY="dummy" /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input
 
 ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ]
 
-CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"]
+CMD [ "/opt/netbox/launch-netbox.sh" ]
-
-ARG BUILD_VERSION
-ARG BUILD_DATE
-ARG VCS_REF
 
 LABEL maintainer="Vapor IO" \
 # See http://label-schema.org/rc1/#build-time-labels
@@ -100,3 +106,13 @@ LABEL maintainer="Vapor IO" \
       org.opencontainers.image.source="https://github.com/vapor-ware/netbox.git" \
       org.opencontainers.image.revision=$VCS_REF \
       org.opencontainers.image.version=$BUILD_VERSION
+
+
+FROM main as ldap
+
+RUN apk add --no-cache \
+      libsasl \
+      libldap \
+      util-linux
+
+COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py

docker-compose.yml

@@ -2,74 +2,61 @@ version: '3.4'
 services:
   netbox: &netbox
     image: vaporio/netbox:develop
-    build:
-      context: .
-      args:
-        BUILD_VERSION: 'local-dev'
-        BUILD_DATE: ''
-        VCS_REF: 'tip'
     depends_on:
     - postgres
     - redis
     - redis-cache
     - netbox-worker
     env_file: docker/env/netbox.env
-    #user: '1000:1000'
+    user: '101'
     volumes:
-    - ./netbox:/opt/netbox/netbox
     - ./docker/startup_scripts:/opt/netbox/startup_scripts:z,ro
     - ./docker/initializers:/opt/netbox/initializers:z,ro
     - ./docker/configuration:/etc/netbox/config:z,ro
     - ./reports:/etc/netbox/reports:z,ro
     - ./scripts:/etc/netbox/scripts:z,ro
-    - netbox-nginx-config:/etc/netbox-nginx:z
-    - netbox-static-files:/opt/netbox/netbox/static:z
     - netbox-media-files:/opt/netbox/netbox/media:z
+    ports:
+    - "8000:8080"
   netbox-worker:
     <<: *netbox
     depends_on:
     - redis
     entrypoint:
-    - python3
+    - /opt/netbox/venv/bin/python
     - /opt/netbox/netbox/manage.py
     command:
     - rqworker
-  nginx:
+    ports: [ ]
-    command: nginx -c /etc/netbox-nginx/nginx.conf
+
-    image: nginx:1.18
-    depends_on:
-    - netbox
-    ports:
-    - 8000:8080
-    volumes:
-    - netbox-static-files:/opt/netbox/netbox/static:ro
-    - netbox-nginx-config:/etc/netbox-nginx/:ro
   postgres:
-    image: postgres:11-alpine
+    image: postgres:12-alpine
-    environment:
+    env_file: docker/env/postgres.env
-      POSTGRES_PASSWORD: "12345"
-      POSTGRES_DB: netbox
     ports:
-    - 5432:5432
+    - "5432:5432"
     volumes:
     - netbox-postgres-data:/var/lib/postgresql/data
+
   redis:
-    image: redis:5-alpine
+    image: redis:6-alpine
     command:
-    - redis-server
+    - sh
-    - --appendonly yes
+    - -c # this is to evaluate the $REDIS_PASSWORD from the env
-    - --requirepass 12345
+    - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
-    ports:
-    - 6379:6379
     volumes:
     - netbox-redis-data:/data
+    ports:
+      - "6379:6379"
+    env_file: docker/env/redis.env
   redis-cache:
-    image: redis:5-alpine
+    image: redis:6-alpine
+    command:
+    - sh
+    - -c # this is to evaluate the $REDIS_PASSWORD from the env
+    - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
+    env_file: docker/env/redis-cache.env
+
 volumes:
-  netbox-static-files:
-    driver: local
-  netbox-nginx-config:
-    driver: local
   netbox-media-files:
     driver: local
   netbox-postgres-data:

docker/configuration.docker.py

@@ -1,10 +1,81 @@
+## Generic Parts
+# These functions are providing the functionality to load
+# arbitrary configuration files.
+#
+# They can be imported by other code (see `ldap_config.py` for an example).
+
 import importlib.util
 import sys
+from os import scandir
+from os.path import abspath, isfile
 
-try:
+
-  spec = importlib.util.spec_from_file_location('configuration', '/etc/netbox/config/configuration.py')
+def _filename(f):
-  module = importlib.util.module_from_spec(spec)
+    return f.name
-  spec.loader.exec_module(module)
+
-  sys.modules['netbox.configuration'] = module
+
-except:
+def _import(module_name, path, loaded_configurations):
-  raise ImportError('')
+    spec = importlib.util.spec_from_file_location("", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    sys.modules[module_name] = module
+
+    loaded_configurations.insert(0, module)
+
+    print(f"🧬 loaded config '{path}'")
+
+
+def read_configurations(config_module, config_dir, main_config):
+    loaded_configurations = []
+
+    main_config_path = abspath(f"{config_dir}/{main_config}.py")
+    if isfile(main_config_path):
+        _import(f"{config_module}.{main_config}", main_config_path, loaded_configurations)
+    else:
+        print(f"⚠️ Main configuration '{main_config_path}' not found.")
+
+    with scandir(config_dir) as it:
+        for f in sorted(it, key=_filename):
+            if not f.is_file():
+                continue
+
+            if f.name.startswith("__"):
+                continue
+
+            if not f.name.endswith(".py"):
+                continue
+
+            if f.name == f"{config_dir}.py":
+                continue
+
+            module_name = f"{config_module}.{f.name[:-len('.py')]}".replace(".", "_")
+            _import(module_name, f.path, loaded_configurations)
+
+    if len(loaded_configurations) == 0:
+        print(f"‼️ No configuration files found in '{config_dir}'.")
+        raise ImportError(f"No configuration files found in '{config_dir}'.")
+
+    return loaded_configurations
+
+
+## Specific Parts
+# This section's code actually loads the various configuration files
+# into the module with the given name.
+# It contains the logic to resolve arbitrary configuration options by
+# levaraging dynamic programming using `__getattr__`.
+
+
+_loaded_configurations = read_configurations(
+    config_dir="/etc/netbox/config/",
+    config_module="netbox.configuration",
+    main_config="configuration",
+)
+
+
+def __getattr__(name):
+    for config in _loaded_configurations:
+        try:
+            return getattr(config, name)
+        except:
+            pass
+    raise AttributeError

docker/configuration/configuration.py

@@ -1,21 +1,27 @@
-import os
+####
-import re
+## We recommend to not edit this file.
-import socket
+## Create separate files to overwrite the settings.
+## See `extra.py` as an example.
+####
 
-# For reference see http://netbox.readthedocs.io/en/latest/configuration/mandatory-settings/
+import re
-# Based on https://github.com/netbox-community/netbox/blob/develop/netbox/netbox/configuration.example.py
+from os import environ
+from os.path import abspath, dirname, join
+
+# For reference see https://netbox.readthedocs.io/en/stable/configuration/
+# Based on https://github.com/netbox-community/netbox/blob/master/netbox/netbox/configuration.example.py
 
 # Read secret from file
-def read_secret(secret_name):
+def _read_secret(secret_name, default = None):
     try:
         f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8')
     except EnvironmentError:
-        return ''
+        return default
     else:
         with f:
             return f.readline().strip()
 
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_BASE_DIR = dirname(dirname(abspath(__file__)))
 
 #########################
 #                       #
@@ -27,47 +33,49 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 # access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name.
 #
 # Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local']
-ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', '*').split(' ')
+ALLOWED_HOSTS = environ.get('ALLOWED_HOSTS', '*').split(' ')
 
-# PostgreSQL database configuration.
+# PostgreSQL database configuration. See the Django documentation for a complete list of available parameters:
+#   https://docs.djangoproject.com/en/stable/ref/settings/#databases
 DATABASE = {
-    'NAME': os.environ.get('DB_NAME', 'netbox'),         # Database name
+    'NAME': environ.get('DB_NAME', 'netbox'),       # Database name
-    'USER': os.environ.get('DB_USER', ''),               # PostgreSQL username
+    'USER': environ.get('DB_USER', ''),             # PostgreSQL username
-    'PASSWORD': os.environ.get('DB_PASSWORD', read_secret('db_password')),
+    'PASSWORD': _read_secret('db_password', environ.get('DB_PASSWORD', '')),
-                                                         # PostgreSQL password
+                                                    # PostgreSQL password
-    'HOST': os.environ.get('DB_HOST', 'localhost'),      # Database server
+    'HOST': environ.get('DB_HOST', 'localhost'),    # Database server
-    'PORT': os.environ.get('DB_PORT', ''),               # Database port (leave blank for default)
+    'PORT': environ.get('DB_PORT', ''),             # Database port (leave blank for default)
-    'OPTIONS': {'sslmode': os.environ.get('DB_SSLMODE', 'prefer')},
+    'OPTIONS': {'sslmode': environ.get('DB_SSLMODE', 'prefer')},
-                                                         # Database connection SSLMODE
+                                                    # Database connection SSLMODE
-    'CONN_MAX_AGE': int(os.environ.get('DB_CONN_MAX_AGE', '300')),
+    'CONN_MAX_AGE': int(environ.get('DB_CONN_MAX_AGE', '300')),
-                                                         # Database connection persistence
+                                                    # Max database connection age
+}
+
+# Redis database settings. Redis is used for caching and for queuing background tasks such as webhook events. A separate
+# configuration exists for each. Full connection details are required in both sections, and it is strongly recommended
+# to use two separate database IDs.
+REDIS = {
+    'tasks': {
+        'HOST': environ.get('REDIS_HOST', 'localhost'),
+        'PORT': int(environ.get('REDIS_PORT', 6379)),
+        'PASSWORD': _read_secret('redis_password', environ.get('REDIS_PASSWORD', '')),
+        'DATABASE': int(environ.get('REDIS_DATABASE', 0)),
+        'SSL': environ.get('REDIS_SSL', 'False').lower() == 'true',
+    },
+    'caching': {
+        'HOST': environ.get('REDIS_CACHE_HOST', environ.get('REDIS_HOST', 'localhost')),
+        'PORT': int(environ.get('REDIS_CACHE_PORT', environ.get('REDIS_PORT', 6379))),
+        'PASSWORD': _read_secret('redis_cache_password', environ.get('REDIS_CACHE_PASSWORD', environ.get('REDIS_PASSWORD', ''))),
+        'DATABASE': int(environ.get('REDIS_CACHE_DATABASE', 1)),
+        'SSL': environ.get('REDIS_CACHE_SSL', environ.get('REDIS_SSL', 'False')).lower() == 'true',
+    },
 }
 
 # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file.
 # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and
 # symbols. NetBox will not run without this defined. For more information, see
-# https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-SECRET_KEY
+# https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY
-SECRET_KEY = os.environ.get('SECRET_KEY', read_secret('secret_key'))
+SECRET_KEY = _read_secret('secret_key', environ.get('SECRET_KEY', ''))
 
-# Redis database settings. The Redis database is used for caching and background processing such as webhooks
-REDIS = {
-    'tasks': {
-        'HOST': os.environ.get('REDIS_HOST', 'localhost'),
-        'PORT': int(os.environ.get('REDIS_PORT', 6379)),
-        'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')),
-        'DATABASE': int(os.environ.get('REDIS_DATABASE', 0)),
-        'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_TIMEOUT', 300)),
-        'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true',
-    },
-    'caching': {
-        'HOST': os.environ.get('REDIS_CACHE_HOST', os.environ.get('REDIS_HOST', 'localhost')),
-        'PORT': int(os.environ.get('REDIS_CACHE_PORT', os.environ.get('REDIS_PORT', 6379))),
-        'PASSWORD': os.environ.get('REDIS_CACHE_PASSWORD', os.environ.get('REDIS_PASSWORD', read_secret('redis_cache_password'))),
-        'DATABASE': int(os.environ.get('REDIS_CACHE_DATABASE', 1)),
-        'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_CACHE_TIMEOUT', os.environ.get('REDIS_TIMEOUT', 300))),
-        'SSL': os.environ.get('REDIS_CACHE_SSL', os.environ.get('REDIS_SSL', 'False')).lower() == 'true',
-    },
-}
 
 #########################
 #                       #
@@ -81,130 +89,162 @@ ADMINS = [
     # ['John Doe', 'jdoe@example.com'],
 ]
 
+# URL schemes that are allowed within links in NetBox
+ALLOWED_URL_SCHEMES = (
+    'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp',
+)
+
 # Optionally display a persistent banner at the top and/or bottom of every page. HTML is allowed. To display the same
 # content in both banners, define BANNER_TOP and set BANNER_BOTTOM = BANNER_TOP.
-BANNER_TOP = os.environ.get('BANNER_TOP', '')
+BANNER_TOP = environ.get('BANNER_TOP', '')
-BANNER_BOTTOM = os.environ.get('BANNER_BOTTOM', '')
+BANNER_BOTTOM = environ.get('BANNER_BOTTOM', '')
 
 # Text to include on the login page above the login form. HTML is allowed.
-BANNER_LOGIN = os.environ.get('BANNER_LOGIN', '')
+BANNER_LOGIN = environ.get('BANNER_LOGIN', '')
 
 # Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set:
 # BASE_PATH = 'netbox/'
-BASE_PATH = os.environ.get('BASE_PATH', '')
+BASE_PATH = environ.get('BASE_PATH', '')
 
 # Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes)
-CACHE_TIMEOUT = int(os.environ.get('CACHE_TIMEOUT', 900))
+CACHE_TIMEOUT = int(environ.get('CACHE_TIMEOUT', 900))
 
 # Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90)
-CHANGELOG_RETENTION = int(os.environ.get('CHANGELOG_RETENTION', 90))
+CHANGELOG_RETENTION = int(environ.get('CHANGELOG_RETENTION', 90))
 
 # API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be
 # allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or
 # CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers
-CORS_ORIGIN_ALLOW_ALL = os.environ.get('CORS_ORIGIN_ALLOW_ALL', 'False').lower() == 'true'
+CORS_ORIGIN_ALLOW_ALL = environ.get('CORS_ORIGIN_ALLOW_ALL', 'False').lower() == 'true'
-CORS_ORIGIN_WHITELIST = list(filter(None, os.environ.get('CORS_ORIGIN_WHITELIST', 'https://localhost').split(' ')))
+CORS_ORIGIN_WHITELIST = list(filter(None, environ.get('CORS_ORIGIN_WHITELIST', 'https://localhost').split(' ')))
-CORS_ORIGIN_REGEX_WHITELIST = [re.compile(r) for r in list(filter(None, os.environ.get('CORS_ORIGIN_REGEX_WHITELIST', '').split(' ')))]
+CORS_ORIGIN_REGEX_WHITELIST = [re.compile(r) for r in list(filter(None, environ.get('CORS_ORIGIN_REGEX_WHITELIST', '').split(' ')))]
 
 # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal
 # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging
 # on a production system.
-DEBUG = os.environ.get('DEBUG', 'False').lower() == 'true'
+DEBUG = environ.get('DEBUG', 'False').lower() == 'true'
 
 # Email settings
 EMAIL = {
-    'SERVER': os.environ.get('EMAIL_SERVER', 'localhost'),
+    'SERVER': environ.get('EMAIL_SERVER', 'localhost'),
-    'PORT': int(os.environ.get('EMAIL_PORT', 25)),
+    'PORT': int(environ.get('EMAIL_PORT', 25)),
-    'USERNAME': os.environ.get('EMAIL_USERNAME', ''),
+    'USERNAME': environ.get('EMAIL_USERNAME', ''),
-    'PASSWORD': os.environ.get('EMAIL_PASSWORD', read_secret('email_password')),
+    'PASSWORD': _read_secret('email_password', environ.get('EMAIL_PASSWORD', '')),
-    'TIMEOUT': int(os.environ.get('EMAIL_TIMEOUT', 10)),  # seconds
+    'USE_SSL': environ.get('EMAIL_USE_SSL', 'False').lower() == 'true',
-    'FROM_EMAIL': os.environ.get('EMAIL_FROM', ''),
+    'USE_TLS': environ.get('EMAIL_USE_TLS', 'False').lower() == 'true',
-    'USE_SSL': os.environ.get('EMAIL_USE_SSL', 'False').lower() == 'true',
+    'SSL_CERTFILE': environ.get('EMAIL_SSL_CERTFILE', ''),
-    'USE_TLS': os.environ.get('EMAIL_USE_TLS', 'False').lower() == 'true',
+    'SSL_KEYFILE': environ.get('EMAIL_SSL_KEYFILE', ''),
-    'SSL_CERTFILE': os.environ.get('EMAIL_SSL_CERTFILE', ''),
+    'TIMEOUT': int(environ.get('EMAIL_TIMEOUT', 10)),  # seconds
-    'SSL_KEYFILE': os.environ.get('EMAIL_SSL_KEYFILE', ''),
+    'FROM_EMAIL': environ.get('EMAIL_FROM', ''),
 }
 
-# Enforcement of unique IP space can be toggled on a per-VRF basis.
+# Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table
-# To enforce unique IP space within the global table (all prefixes and IP addresses not assigned to a VRF),
+# (all prefixes and IP addresses not assigned to a VRF), set ENFORCE_GLOBAL_UNIQUE to True.
-# set ENFORCE_GLOBAL_UNIQUE to True.
+ENFORCE_GLOBAL_UNIQUE = environ.get('ENFORCE_GLOBAL_UNIQUE', 'False').lower() == 'true'
-ENFORCE_GLOBAL_UNIQUE = os.environ.get('ENFORCE_GLOBAL_UNIQUE', 'False').lower() == 'true'
 
 # Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and
 # by anonymous users. List models in the form `<app>.<model>`. Add '*' to this list to exempt all models.
-EXEMPT_VIEW_PERMISSIONS = list(filter(None, os.environ.get('EXEMPT_VIEW_PERMISSIONS', '').split(' ')))
+EXEMPT_VIEW_PERMISSIONS = list(filter(None, environ.get('EXEMPT_VIEW_PERMISSIONS', '').split(' ')))
 
 # Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs:
-#   https://docs.djangoproject.com/en/1.11/topics/logging/
+#   https://docs.djangoproject.com/en/stable/topics/logging/
 LOGGING = {}
 
 # Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users
 # are permitted to access most data in NetBox (excluding secrets) but not make any changes.
-LOGIN_REQUIRED = os.environ.get('LOGIN_REQUIRED', 'False').lower() == 'true'
+LOGIN_REQUIRED = environ.get('LOGIN_REQUIRED', 'False').lower() == 'true'
+
+# The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to
+# re-authenticate. (Default: 1209600 [14 days])
+LOGIN_TIMEOUT = int(environ.get('LOGIN_TIMEOUT', 1209600))
 
 # Setting this to True will display a "maintenance mode" banner at the top of every page.
-MAINTENANCE_MODE = os.environ.get('MAINTENANCE_MODE', 'False').lower() == 'true'
+MAINTENANCE_MODE = environ.get('MAINTENANCE_MODE', 'False').lower() == 'true'
 
 # An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g.
 # "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request
 # all objects by specifying "?limit=0".
-MAX_PAGE_SIZE = int(os.environ.get('MAX_PAGE_SIZE', 1000))
+MAX_PAGE_SIZE = int(environ.get('MAX_PAGE_SIZE', 1000))
 
 # The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that
 # the default value of this setting is derived from the installed location.
-MEDIA_ROOT = os.environ.get('MEDIA_ROOT', os.path.join(BASE_DIR, 'media'))
+MEDIA_ROOT = environ.get('MEDIA_ROOT', join(_BASE_DIR, 'media'))
 
 # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics'
-METRICS_ENABLED = os.environ.get('METRICS_ENABLED', 'False').lower() == 'true'
+METRICS_ENABLED = environ.get('METRICS_ENABLED', 'False').lower() == 'true'
 
-# Credentials that NetBox will use to access live devices.
+# Credentials that NetBox will uses to authenticate to devices when connecting via NAPALM.
-NAPALM_USERNAME = os.environ.get('NAPALM_USERNAME', '')
+NAPALM_USERNAME = environ.get('NAPALM_USERNAME', '')
-NAPALM_PASSWORD = os.environ.get('NAPALM_PASSWORD', read_secret('napalm_password'))
+NAPALM_PASSWORD = _read_secret('napalm_password', environ.get('NAPALM_PASSWORD', ''))
 
 # NAPALM timeout (in seconds). (Default: 30)
-NAPALM_TIMEOUT = int(os.environ.get('NAPALM_TIMEOUT', 30))
+NAPALM_TIMEOUT = int(environ.get('NAPALM_TIMEOUT', 30))
 
 # NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must
 # be provided as a dictionary.
 NAPALM_ARGS = {}
 
 # Determine how many objects to display per page within a list. (Default: 50)
-PAGINATE_COUNT = int(os.environ.get('PAGINATE_COUNT', 50))
+PAGINATE_COUNT = int(environ.get('PAGINATE_COUNT', 50))
 
-# Plugins
+# Enable installed plugins. Add the name of each plugin to the list.
 PLUGINS = [
     'netbox_virtual_circuit_plugin',
     'netbox_bgp',
 ]
 
+# Plugins configuration settings. These settings are used by various plugins that the user may have installed.
+# Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings.
+PLUGINS_CONFIG = {
+}
+
 # When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to
 # prefer IPv4 instead.
-PREFER_IPV4 = os.environ.get('PREFER_IPV4', 'False').lower() == 'true'
+PREFER_IPV4 = environ.get('PREFER_IPV4', 'False').lower() == 'true'
 
-# This determines how often the GitHub API is called to check the latest release of NetBox in seconds. Must be at least 1 hour.
+# Rack elevation size defaults, in pixels. For best results, the ratio of width to height should be roughly 10:1.
-RELEASE_CHECK_TIMEOUT = os.environ.get('RELEASE_CHECK_TIMEOUT', 24 * 3600)
+RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = int(environ.get('RACK_ELEVATION_DEFAULT_UNIT_HEIGHT', 22))
+RACK_ELEVATION_DEFAULT_UNIT_WIDTH = int(environ.get('RACK_ELEVATION_DEFAULT_UNIT_WIDTH', 220))
+
+# Remote authentication support
+REMOTE_AUTH_ENABLED = environ.get('REMOTE_AUTH_ENABLED', 'False').lower() == 'true'
+REMOTE_AUTH_BACKEND = environ.get('REMOTE_AUTH_BACKEND', 'netbox.authentication.RemoteUserBackend')
+REMOTE_AUTH_HEADER = environ.get('REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER')
+REMOTE_AUTH_AUTO_CREATE_USER = environ.get('REMOTE_AUTH_AUTO_CREATE_USER', 'True').lower() == 'true'
+REMOTE_AUTH_DEFAULT_GROUPS = list(filter(None, environ.get('REMOTE_AUTH_DEFAULT_GROUPS', '').split(' ')))
+
+# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour.
+RELEASE_CHECK_TIMEOUT = int(environ.get('RELEASE_CHECK_TIMEOUT', 24 * 3600))
 
 # This repository is used to check whether there is a new release of NetBox available. Set to None to disable the
 # version check or use the URL below to check for release in the official NetBox repository.
 # https://api.github.com/repos/netbox-community/netbox/releases
-RELEASE_CHECK_URL = os.environ.get('RELEASE_CHECK_URL', None)
+RELEASE_CHECK_URL = environ.get('RELEASE_CHECK_URL', None)
 
 # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of
 # this setting is derived from the installed location.
-REPORTS_ROOT = os.environ.get('REPORTS_ROOT', '/etc/netbox/reports')
+REPORTS_ROOT = environ.get('REPORTS_ROOT', '/etc/netbox/reports')
+
+# Maximum execution time for background tasks, in seconds.
+RQ_DEFAULT_TIMEOUT = int(environ.get('RQ_DEFAULT_TIMEOUT', 300))
 
 # The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of
 # this setting is derived from the installed location.
-SCRIPTS_ROOT = os.environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts')
+SCRIPTS_ROOT = environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts')
+
+# By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use
+# local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only
+# database access.) Note that the user as which NetBox runs must have read and write permissions to this path.
+SESSION_FILE_PATH = environ.get('SESSIONS_ROOT', None)
 
 # Time zone (default: UTC)
-TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC')
+TIME_ZONE = environ.get('TIME_ZONE', 'UTC')
 
 # Date/time formatting. See the following link for supported formats:
-# https://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
+# https://docs.djangoproject.com/en/stable/ref/templates/builtins/#date
-DATE_FORMAT = os.environ.get('DATE_FORMAT', 'N j, Y')
+DATE_FORMAT = environ.get('DATE_FORMAT', 'N j, Y')
-SHORT_DATE_FORMAT = os.environ.get('SHORT_DATE_FORMAT', 'Y-m-d')
+SHORT_DATE_FORMAT = environ.get('SHORT_DATE_FORMAT', 'Y-m-d')
-TIME_FORMAT = os.environ.get('TIME_FORMAT', 'g:i a')
+TIME_FORMAT = environ.get('TIME_FORMAT', 'g:i a')
-SHORT_TIME_FORMAT = os.environ.get('SHORT_TIME_FORMAT', 'H:i:s')
+SHORT_TIME_FORMAT = environ.get('SHORT_TIME_FORMAT', 'H:i:s')
-DATETIME_FORMAT = os.environ.get('DATETIME_FORMAT', 'N j, Y g:i a')
+DATETIME_FORMAT = environ.get('DATETIME_FORMAT', 'N j, Y g:i a')
-SHORT_DATETIME_FORMAT = os.environ.get('SHORT_DATETIME_FORMAT', 'Y-m-d H:i')
+SHORT_DATETIME_FORMAT = environ.get('SHORT_DATETIME_FORMAT', 'Y-m-d H:i')

docker/configuration/extra.py

@@ -0,0 +1,55 @@
+####
+## This file contains extra configuration options that can't be configured
+## directly through environment variables.
+####
+
+## Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of
+## application errors (assuming correct email settings are provided).
+# ADMINS = [
+#     # ['John Doe', 'jdoe@example.com'],
+# ]
+
+
+## URL schemes that are allowed within links in NetBox
+# ALLOWED_URL_SCHEMES = (
+#     'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp',
+# )
+
+
+## NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must
+## be provided as a dictionary.
+# NAPALM_ARGS = {}
+
+
+## Enable installed plugins. Add the name of each plugin to the list.
+# from netbox.configuration.configuration import PLUGINS
+# PLUGINS.append('my_plugin')
+
+## Plugins configuration settings. These settings are used by various plugins that the user may have installed.
+## Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings.
+# from netbox.configuration.configuration import PLUGINS_CONFIG
+# PLUGINS_CONFIG['my_plugin'] = {
+#   'foo': 'bar',
+#   'buzz': 'bazz'
+# }
+
+
+## Remote authentication support
+# REMOTE_AUTH_DEFAULT_PERMISSIONS = {}
+
+
+## By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the
+## class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example:
+# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage'
+# STORAGE_CONFIG = {
+#     'AWS_ACCESS_KEY_ID': 'Key ID',
+#     'AWS_SECRET_ACCESS_KEY': 'Secret',
+#     'AWS_STORAGE_BUCKET_NAME': 'netbox',
+#     'AWS_S3_REGION_NAME': 'eu-west-1',
+# }
+
+
+## This file can contain arbitrary Python code, e.g.:
+# from datetime import datetime
+# now = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
+# BANNER_TOP = f'<marquee width="200px">This instance started on {now}.</marquee>'

docker/configuration/gunicorn_config.py

@@ -1,8 +0,0 @@
-command = '/usr/bin/gunicorn'
-pythonpath = '/opt/netbox/netbox'
-bind = '0.0.0.0:8001'
-workers = 3
-errorlog = '-'
-accesslog = '-'
-capture_output = False
-loglevel = 'debug'

docker/configuration/ldap/ldap_config.py

@@ -0,0 +1,90 @@
+from importlib import import_module
+from os import environ
+
+import ldap
+from django_auth_ldap.config import LDAPSearch
+
+
+# Read secret from file
+def _read_secret(secret_name, default=None):
+    try:
+        f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8')
+    except EnvironmentError:
+        return default
+    else:
+        with f:
+            return f.readline().strip()
+
+# Import and return the group type based on string name
+def _import_group_type(group_type_name):
+    mod = import_module('django_auth_ldap.config')
+    try:
+        return getattr(mod, group_type_name)()
+    except:
+        return None
+
+# Server URI
+AUTH_LDAP_SERVER_URI = environ.get('AUTH_LDAP_SERVER_URI', '')
+
+# The following may be needed if you are binding to Active Directory.
+AUTH_LDAP_CONNECTION_OPTIONS = {
+    ldap.OPT_REFERRALS: 0
+}
+
+# Set the DN and password for the NetBox service account.
+AUTH_LDAP_BIND_DN = environ.get('AUTH_LDAP_BIND_DN', '')
+AUTH_LDAP_BIND_PASSWORD = _read_secret('auth_ldap_bind_password', environ.get('AUTH_LDAP_BIND_PASSWORD', ''))
+
+# Set a string template that describes any user’s distinguished name based on the username.
+AUTH_LDAP_USER_DN_TEMPLATE = environ.get('AUTH_LDAP_USER_DN_TEMPLATE', None)
+
+# Enable STARTTLS for ldap authentication.
+AUTH_LDAP_START_TLS = environ.get('AUTH_LDAP_START_TLS', 'False').lower() == 'true'
+
+# Include this setting if you want to ignore certificate errors. This might be needed to accept a self-signed cert.
+# Note that this is a NetBox-specific setting which sets:
+#     ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+LDAP_IGNORE_CERT_ERRORS = environ.get('LDAP_IGNORE_CERT_ERRORS', 'False').lower() == 'true'
+
+AUTH_LDAP_USER_SEARCH_BASEDN = environ.get('AUTH_LDAP_USER_SEARCH_BASEDN', '')
+AUTH_LDAP_USER_SEARCH_ATTR = environ.get('AUTH_LDAP_USER_SEARCH_ATTR', 'sAMAccountName')
+AUTH_LDAP_USER_SEARCH = LDAPSearch(
+    AUTH_LDAP_USER_SEARCH_BASEDN,
+    ldap.SCOPE_SUBTREE,
+    "(" + AUTH_LDAP_USER_SEARCH_ATTR + "=%(user)s)"
+)
+
+# This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group
+# heirarchy.
+AUTH_LDAP_GROUP_SEARCH_BASEDN = environ.get('AUTH_LDAP_GROUP_SEARCH_BASEDN', '')
+AUTH_LDAP_GROUP_SEARCH_CLASS = environ.get('AUTH_LDAP_GROUP_SEARCH_CLASS', 'group')
+AUTH_LDAP_GROUP_SEARCH = LDAPSearch(AUTH_LDAP_GROUP_SEARCH_BASEDN, ldap.SCOPE_SUBTREE,
+                                    "(objectClass=" + AUTH_LDAP_GROUP_SEARCH_CLASS + ")")
+AUTH_LDAP_GROUP_TYPE = _import_group_type(environ.get('AUTH_LDAP_GROUP_TYPE', 'GroupOfNamesType'))
+
+# Define a group required to login.
+AUTH_LDAP_REQUIRE_GROUP = environ.get('AUTH_LDAP_REQUIRE_GROUP_DN')
+
+# Define special user types using groups. Exercise great caution when assigning superuser status.
+AUTH_LDAP_USER_FLAGS_BY_GROUP = {}
+
+if AUTH_LDAP_REQUIRE_GROUP is not None:
+    AUTH_LDAP_USER_FLAGS_BY_GROUP = {
+        "is_active": environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', ''),
+        "is_staff": environ.get('AUTH_LDAP_IS_ADMIN_DN', ''),
+        "is_superuser": environ.get('AUTH_LDAP_IS_SUPERUSER_DN', '')
+    }
+
+# For more granular permissions, we can map LDAP groups to Django groups.
+AUTH_LDAP_FIND_GROUP_PERMS = environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true'
+AUTH_LDAP_MIRROR_GROUPS = environ.get('AUTH_LDAP_MIRROR_GROUPS', '').lower() == 'true'
+
+# Cache groups for one hour to reduce LDAP traffic
+AUTH_LDAP_CACHE_TIMEOUT = int(environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600))
+
+# Populate the Django user from the LDAP directory.
+AUTH_LDAP_USER_ATTR_MAP = {
+    "first_name": environ.get('AUTH_LDAP_ATTR_FIRSTNAME', 'givenName'),
+    "last_name": environ.get('AUTH_LDAP_ATTR_LASTNAME', 'sn'),
+    "email": environ.get('AUTH_LDAP_ATTR_MAIL', 'mail')
+}

docker/configuration/ldap_config.py

@@ -1,80 +0,0 @@
-import ldap
-import os
-
-from django_auth_ldap.config import LDAPSearch
-from importlib import import_module
-
-# Read secret from file
-def read_secret(secret_name):
-    try:
-        f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8')
-    except EnvironmentError:
-        return ''
-    else:
-        with f:
-            return f.readline().strip()
-
-# Import and return the group type based on string name
-def import_group_type(group_type_name):
-    mod = import_module('django_auth_ldap.config')
-    try:
-        return getattr(mod, group_type_name)()
-    except:
-        return None
-
-# Server URI
-AUTH_LDAP_SERVER_URI = os.environ.get('AUTH_LDAP_SERVER_URI', '')
-
-# The following may be needed if you are binding to Active Directory.
-AUTH_LDAP_CONNECTION_OPTIONS = {
-    ldap.OPT_REFERRALS: 0
-}
-
-# Set the DN and password for the NetBox service account.
-AUTH_LDAP_BIND_DN = os.environ.get('AUTH_LDAP_BIND_DN', '')
-AUTH_LDAP_BIND_PASSWORD = os.environ.get('AUTH_LDAP_BIND_PASSWORD', read_secret('auth_ldap_bind_password'))
-
-# Set a string template that describes any user’s distinguished name based on the username.
-AUTH_LDAP_USER_DN_TEMPLATE = os.environ.get('AUTH_LDAP_USER_DN_TEMPLATE', None)
-
-# Include this setting if you want to ignore certificate errors. This might be needed to accept a self-signed cert.
-# Note that this is a NetBox-specific setting which sets:
-#     ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
-LDAP_IGNORE_CERT_ERRORS = os.environ.get('LDAP_IGNORE_CERT_ERRORS', 'False').lower() == 'true'
-
-AUTH_LDAP_USER_SEARCH_BASEDN = os.environ.get('AUTH_LDAP_USER_SEARCH_BASEDN', '')
-AUTH_LDAP_USER_SEARCH_ATTR = os.environ.get('AUTH_LDAP_USER_SEARCH_ATTR', 'sAMAccountName')
-AUTH_LDAP_USER_SEARCH = LDAPSearch(AUTH_LDAP_USER_SEARCH_BASEDN,
-                                   ldap.SCOPE_SUBTREE,
-                                   "(" + AUTH_LDAP_USER_SEARCH_ATTR + "=%(user)s)")
-
-# This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group
-# heirarchy.
-AUTH_LDAP_GROUP_SEARCH_BASEDN = os.environ.get('AUTH_LDAP_GROUP_SEARCH_BASEDN', '')
-AUTH_LDAP_GROUP_SEARCH_CLASS = os.environ.get('AUTH_LDAP_GROUP_SEARCH_CLASS', 'group')
-AUTH_LDAP_GROUP_SEARCH = LDAPSearch(AUTH_LDAP_GROUP_SEARCH_BASEDN, ldap.SCOPE_SUBTREE,
-                                    "(objectClass=" + AUTH_LDAP_GROUP_SEARCH_CLASS + ")")
-AUTH_LDAP_GROUP_TYPE = import_group_type(os.environ.get('AUTH_LDAP_GROUP_TYPE', 'GroupOfNamesType'))
-
-# Define a group required to login.
-AUTH_LDAP_REQUIRE_GROUP = os.environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', '')
-
-# Define special user types using groups. Exercise great caution when assigning superuser status.
-AUTH_LDAP_USER_FLAGS_BY_GROUP = {
-    "is_active": os.environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', ''),
-    "is_staff": os.environ.get('AUTH_LDAP_IS_ADMIN_DN', ''),
-    "is_superuser": os.environ.get('AUTH_LDAP_IS_SUPERUSER_DN', '')
-}
-
-# For more granular permissions, we can map LDAP groups to Django groups.
-AUTH_LDAP_FIND_GROUP_PERMS = os.environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true'
-
-# Cache groups for one hour to reduce LDAP traffic
-AUTH_LDAP_CACHE_TIMEOUT = int(os.environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600))
-
-# Populate the Django user from the LDAP directory.
-AUTH_LDAP_USER_ATTR_MAP = {
-    "first_name": os.environ.get('AUTH_LDAP_ATTR_FIRSTNAME', 'givenName'),
-    "last_name": os.environ.get('AUTH_LDAP_ATTR_LASTNAME', 'sn'),
-    "email": os.environ.get('AUTH_LDAP_ATTR_MAIL', 'mail')
-}

docker/docker-entrypoint.sh

@@ -1,9 +1,16 @@
 #!/bin/bash
+# Runs on every start of the NetBox Docker container
+
+# Stop when an error occures
 set -e
 
-# Allows Netbox to be run as non-root users
+# Allows NetBox to be run as non-root users
 umask 002
 
+# Load correct Python3 env
+# shellcheck disable=SC1091
+source /opt/netbox/venv/bin/activate
+
 # Try to connect to the DB
 DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3}
 MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30}
@@ -11,7 +18,7 @@ CUR_DB_WAIT_TIME=0
 while ! ./manage.py migrate 2>&1 && [ "${CUR_DB_WAIT_TIME}" -lt "${MAX_DB_WAIT_TIME}" ]; do
   echo "⏳ Waiting on DB... (${CUR_DB_WAIT_TIME}s / ${MAX_DB_WAIT_TIME}s)"
   sleep "${DB_WAIT_TIMEOUT}"
-  CUR_DB_WAIT_TIME=$(( CUR_DB_WAIT_TIME + DB_WAIT_TIMEOUT ))
+  CUR_DB_WAIT_TIME=$((CUR_DB_WAIT_TIME + DB_WAIT_TIMEOUT))
 done
 if [ "${CUR_DB_WAIT_TIME}" -ge "${MAX_DB_WAIT_TIME}" ]; then
   echo "❌ Waited ${MAX_DB_WAIT_TIME}s or more for the DB to become ready."
@@ -29,17 +36,17 @@ else
     SUPERUSER_EMAIL='admin@example.com'
   fi
   if [ -f "/run/secrets/superuser_password" ]; then
-    SUPERUSER_PASSWORD="$(< /run/secrets/superuser_password)"
+    SUPERUSER_PASSWORD="$(</run/secrets/superuser_password)"
   elif [ -z ${SUPERUSER_PASSWORD+x} ]; then
     SUPERUSER_PASSWORD='admin'
   fi
   if [ -f "/run/secrets/superuser_api_token" ]; then
-    SUPERUSER_API_TOKEN="$(< /run/secrets/superuser_api_token)"
+    SUPERUSER_API_TOKEN="$(</run/secrets/superuser_api_token)"
   elif [ -z ${SUPERUSER_API_TOKEN+x} ]; then
     SUPERUSER_API_TOKEN='0123456789abcdef0123456789abcdef01234567'
   fi
 
-  ./manage.py shell --interface python << END
+  ./manage.py shell --interface python <<END
 from django.contrib.auth.models import User
 from users.models import Token
 if not User.objects.filter(username='${SUPERUSER_NAME}'):
@@ -53,16 +60,14 @@ fi
 # Run the startup scripts (and initializers)
 if [ "$SKIP_STARTUP_SCRIPTS" == "true" ]; then
   echo "↩️ Skipping startup scripts"
+else
+  echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python
 fi
 
-echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python
-
-
-# copy static files
-./manage.py collectstatic --no-input
-
 echo "✅ Initialisation is done."
 
-# launch whatever is passed by docker
+# Launch whatever is passed by docker
 # (i.e. the RUN instruction in the Dockerfile)
+#
+# shellcheck disable=SC2068
 exec $@

docker/env/netbox.env

@@ -1,30 +1,42 @@
 CORS_ORIGIN_ALLOW_ALL=True
-DB_NAME=netbox
-DB_USER=postgres
-DB_PASSWORD=12345
 DB_HOST=postgres
-EMAIL_SERVER=localhost
+DB_NAME=netbox
-EMAIL_PORT=25
+DB_PASSWORD=12345
-EMAIL_USERNAME=netbox
+DB_USER=postgres
-EMAIL_PASSWORD=
-EMAIL_TIMEOUT=5
 EMAIL_FROM=netbox@bar.com
+EMAIL_PASSWORD=
+EMAIL_PORT=25
+EMAIL_SERVER=localhost
+EMAIL_SSL_CERTFILE=
+EMAIL_SSL_KEYFILE=
+EMAIL_TIMEOUT=5
+EMAIL_USERNAME=netbox
+# EMAIL_USE_SSL and EMAIL_USE_TLS are mutually exclusive, i.e. they can't both be `true`!
+EMAIL_USE_SSL=false
+EMAIL_USE_TLS=false
+HOUSEKEEPING_INTERVAL=86400
+MAX_PAGE_SIZE=1000
 MEDIA_ROOT=/opt/netbox/netbox/media
-NAPALM_USERNAME=
+METRICS_ENABLED=false
 NAPALM_PASSWORD=
 NAPALM_TIMEOUT=10
-MAX_PAGE_SIZE=1000
+NAPALM_USERNAME=
-REDIS_HOST=redis
+REDIS_CACHE_DATABASE=1
-REDIS_DATABASE=0
-REDIS_SSL=false
 REDIS_CACHE_HOST=redis-cache
-REDIS_CACHE_DATABASE=0
+REDIS_CACHE_INSECURE_SKIP_TLS_VERIFY=false
+REDIS_CACHE_PASSWORD=t4Ph722qJ5QHeQ1qfu36
 REDIS_CACHE_SSL=false
+REDIS_DATABASE=0
+REDIS_HOST=redis
+REDIS_INSECURE_SKIP_TLS_VERIFY=false
+REDIS_PASSWORD=H733Kdjndks81
+REDIS_SSL=false
+RELEASE_CHECK_URL=https://api.github.com/repos/netbox-community/netbox/releases
 SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj
 SKIP_STARTUP_SCRIPTS=true
 SKIP_SUPERUSER=false
-SUPERUSER_NAME=admin
-SUPERUSER_EMAIL=admin@example.com
-SUPERUSER_PASSWORD=admin
 SUPERUSER_API_TOKEN=0123456789abcdef0123456789abcdef01234567
+SUPERUSER_EMAIL=admin@example.com
+SUPERUSER_NAME=admin
+SUPERUSER_PASSWORD=admin
 WEBHOOKS_ENABLED=true

docker/env/postgres.env

@@ -0,0 +1,3 @@
+POSTGRES_DB=netbox
+POSTGRES_PASSWORD=12345
+POSTGRES_USER=postgres

docker/env/redis-cache.env

@@ -0,0 +1 @@
+REDIS_PASSWORD=t4Ph722qJ5QHeQ1qfu36

docker/env/redis.env

@@ -0,0 +1 @@
+REDIS_PASSWORD=H733Kdjndks81

docker/launch-netbox.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+UNIT_CONFIG="${UNIT_CONFIG-/etc/unit/nginx-unit.json}"
+UNIT_SOCKET="/opt/unit/unit.sock"
+
+load_configuration() {
+  MAX_WAIT=10
+  WAIT_COUNT=0
+  while [ ! -S $UNIT_SOCKET ]; do
+    if [ $WAIT_COUNT -ge $MAX_WAIT ]; then
+      echo "⚠️ No control socket found; configuration will not be loaded."
+      return 1
+    fi
+
+    WAIT_COUNT=$((WAIT_COUNT + 1))
+    echo "⏳ Waiting for control socket to be created... (${WAIT_COUNT}/${MAX_WAIT})"
+
+    sleep 1
+  done
+
+  # even when the control socket exists, it does not mean unit has finished initialisation
+  # this curl call will get a reply once unit is fully launched
+  curl --silent --output /dev/null --request GET --unix-socket $UNIT_SOCKET http://localhost/
+
+  echo "⚙️ Applying configuration from $UNIT_CONFIG"
+
+  RESP_CODE=$(
+    curl \
+      --silent \
+      --output /dev/null \
+      --write-out '%{http_code}' \
+      --request PUT \
+      --data-binary "@${UNIT_CONFIG}" \
+      --unix-socket $UNIT_SOCKET \
+      http://localhost/config
+  )
+  if [ "$RESP_CODE" != "200" ]; then
+    echo "⚠️ Could no load Unit configuration"
+    kill "$(cat /opt/unit/unit.pid)"
+    return 1
+  fi
+
+  echo "✅ Unit configuration loaded successfully"
+}
+
+load_configuration &
+
+exec unitd \
+  --no-daemon \
+  --control unix:$UNIT_SOCKET \
+  --pid /opt/unit/unit.pid \
+  --log /dev/stdout \
+  --state /opt/unit/state/ \
+  --tmp /opt/unit/tmp/

docker/ldap_config.docker.py

@@ -0,0 +1,23 @@
+from .configuration import read_configurations
+
+_loaded_configurations = read_configurations(
+    config_dir="/etc/netbox/config/ldap/",
+    config_module="netbox.configuration.ldap",
+    main_config="ldap_config",
+)
+
+
+def __getattr__(name):
+    for config in _loaded_configurations:
+        try:
+            return getattr(config, name)
+        except:
+            pass
+    raise AttributeError
+
+
+def __dir__():
+    names = []
+    for config in _loaded_configurations:
+        names.extend(config.__dir__())
+    return names

docker/nginx-unit.json

@@ -0,0 +1,40 @@
+{
+  "listeners": {
+    "*:8080": {
+      "pass": "routes"
+    }
+  },
+
+  "routes": [
+    {
+      "match": {
+        "uri": "/static/*"
+      },
+      "action": {
+        "share": "/opt/netbox/netbox"
+      }
+    },
+
+    {
+      "action": {
+        "pass": "applications/netbox"
+      }
+    }
+  ],
+
+  "applications": {
+    "netbox": {
+      "type": "python 3",
+      "path": "/opt/netbox/netbox/",
+      "module": "netbox.wsgi",
+      "home": "/opt/netbox/venv",
+      "processes": {
+        "max": 4,
+        "spare": 1,
+        "idle_timeout": 120
+      }
+    }
+  },
+
+  "access_log": "/dev/stdout"
+}

docker/nginx.conf

@@ -1,36 +0,0 @@
-daemon off;
-worker_processes 1;
-
-error_log /dev/stderr info;
-
-events {
-    worker_connections 1024;
-}
-
-http {
-    include              /etc/nginx/mime.types;
-    default_type         application/octet-stream;
-    sendfile             on;
-    tcp_nopush           on;
-    keepalive_timeout    65;
-    gzip                 on;
-    server_tokens        off;
-    client_max_body_size 10M;
-
-    server {
-        listen      8080;
-        access_log  off;
-
-        location /static/ {
-            alias /opt/netbox/netbox/static/;
-        }
-
-        location / {
-            proxy_pass http://netbox:8001;
-            proxy_set_header X-Forwarded-Host $http_host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-Proto $scheme;
-            add_header P3P 'CP="ALL DSP COR PSAa PSDa OUR NOR ONL UNI COM NAV"';
-        }
-    }
-}

reports/devices.py.example

@@ -0,0 +1,46 @@
+from dcim.choices import DeviceStatusChoices
+from dcim.models import ConsolePort, Device, PowerPort
+from extras.reports import Report
+
+
+class DeviceConnectionsReport(Report):
+    description = "Validate the minimum physical connections for each device"
+
+    def test_console_connection(self):
+
+        # Check that every console port for every active device has a connection defined.
+        active = DeviceStatusChoices.STATUS_ACTIVE
+        for console_port in ConsolePort.objects.prefetch_related('device').filter(device__status=active):
+            if console_port.connected_endpoint is None:
+                self.log_failure(
+                    console_port.device,
+                    "No console connection defined for {}".format(console_port.name)
+                )
+            elif not console_port.connection_status:
+                self.log_warning(
+                    console_port.device,
+                    "Console connection for {} marked as planned".format(console_port.name)
+                )
+            else:
+                self.log_success(console_port.device)
+
+    def test_power_connections(self):
+
+        # Check that every active device has at least two connected power supplies.
+        for device in Device.objects.filter(status=DeviceStatusChoices.STATUS_ACTIVE):
+            connected_ports = 0
+            for power_port in PowerPort.objects.filter(device=device):
+                if power_port.connected_endpoint is not None:
+                    connected_ports += 1
+                    if not power_port.connection_status:
+                        self.log_warning(
+                            device,
+                            "Power connection for {} marked as planned".format(power_port.name)
+                        )
+            if connected_ports < 2:
+                self.log_failure(
+                    device,
+                    "{} connected power supplies found (2 needed)".format(connected_ports)
+                )
+            else:
+                self.log_success(device)

requirements.extras.txt

@@ -1,4 +1,4 @@
-django-allauth==0.42.0
-netbox-virtual-circuit-plugin==1.6.2
 django-storages[google]==1.11.1
-netbox-bgp==0.3.7
+netbox-virtual-circuit-plugin==1.6.2
+django-allauth==0.42.0
+netbox-bgp==0.3.7

scripts/export-to-initializers

@@ -1,28 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import csv
-import sys
-import yaml
-import argparse
-
-parser = argparse.ArgumentParser(description='Netbox csv export to initializer yaml')
-parser.add_argument(
-    'input',
-    type=argparse.FileType('r'),
-    metavar='export.csv',
-    help='netbox csv export'
-)
-
-parser.add_argument(
-    'output',
-    type=argparse.FileType('w'),
-    help='parsed yaml output'
-)
-
-args = parser.parse_args()
-
-data = [r for r in csv.reader(args.input)]
-header = data.pop(0)
-output = [dict(zip(header, r)) for r in data]
-args.output.write(yaml.dump(output, default_flow_style=False))

scripts/git-hooks/pre-commit

@@ -1,41 +0,0 @@
-#!/bin/sh
-# Create a link to this file at .git/hooks/pre-commit to
-# force PEP8 validation prior to committing
-#
-# Ignored violations:
-#
-#   W504: Line break after binary operator
-#   E501: Line too long
-
-exec 1>&2
-
-EXIT=0
-RED='\033[0;31m'
-NOCOLOR='\033[0m'
-
-if [ -d ./venv/ ]; then
-    VENV="$PWD/venv"
-    if [ -e $VENV/bin/python ]; then
-        PATH=$VENV/bin:$PATH
-    elif [ -e $VENV/Scripts/python.exe ]; then
-        PATH=$VENV/Scripts:$PATH
-    fi
-fi
-
-echo "Validating PEP8 compliance..."
-pycodestyle --ignore=W504,E501 netbox/
-if [ $? != 0 ]; then
-	EXIT=1
-fi
-
-echo "Checking for missing migrations..."
-python netbox/manage.py makemigrations --dry-run --check
-if [ $? != 0 ]; then
-	EXIT=1
-fi
-
-if [ $EXIT != 0 ]; then
-  printf "${RED}COMMIT FAILED${NOCOLOR}\n"
-fi
-
-exit $EXIT

scripts/management-api-to-initializers

@@ -1,156 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import csv
-import sys
-import yaml
-import argparse
-
-from pathlib import Path
-
-here = Path(__file__).parent
-
-def sku_parser(uri):
-    resource = uri.lower().replace('.vio.sh', '').replace('-', '.')
-
-    data = {
-        'market': None,
-        'site': None,
-        'cluster': None,
-        'rack': None,
-        'locker': None,
-    }
-
-    parts = resource.split('.')
-    parts.reverse()
-
-    data.update(dict(zip(data.keys(), parts)))
-
-    return data
-
-def sku_builder(*, market=None, site=None, cluster=None, rack=None, locker=None, base='vio.ke', sep='.'):
-    uri = base or ''
-
-    if not market:
-        return uri
-
-    uri = f'{market}{sep}{uri}'
-
-    if uri.endswith(sep):
-        uri = uri[:-1]
-
-    if not site:
-        return uri
-
-    uri = f'{site}{sep}{uri}'
-
-    if not cluster:
-        return uri
-
-    uri = f'{cluster}{sep}{uri}'
-
-    if not rack:
-        return uri
-
-    uri = f'{rack}{sep}{uri}'
-
-    if not locker:
-        return uri
-
-    return f'{locker}{sep}{uri}'
-
-
-parser = argparse.ArgumentParser(description='Managemenet API csv export to initializer yaml')
-parser.add_argument(
-    'lockers',
-    type=argparse.FileType('r'),
-    metavar='export.csv',
-    help='lockers csv export'
-)
-
-lockers = []
-racks = {}
-rack_groups = {}
-
-args = parser.parse_args()
-
-data = [r for r in csv.reader(args.lockers)]
-header = data.pop(0)
-locker_data = [dict(zip(header, r)) for r in data]
-
-lockers = []
-
-rack_roles = {
-    'f': 'Facility',
-    'a': 'Customer',
-    'b': 'Utility',
-}
-
-for locker in locker_data:
-    rack_group = locker['name'].rsplit(' ', 1)[0]
-    uri = sku_parser(locker['sku'])
-
-    if 'r' in uri['rack']:
-        continue
-
-    zone_id = locker['zone_id']
-
-    if 'w' in uri['rack']:
-        rack_name = '{} {}'.format(rack_group, 'Facility' if locker['locker_id'].lower() == 'f' else 'Zone {}'.format(locker['locker_id'][-1]))
-
-        if not rack_group in rack_groups:
-            rack_uri = uri.copy()
-            del rack_uri['locker']
-            rack_groups[rack_group] = {
-                'name': rack_group,
-                'site': locker['site_location'],
-                'slug': sku_builder(sep='-', base='', **rack_uri).lower(),
-            }
-
-    if not rack_name in racks:
-        rack_uri = uri.copy()
-        rack_uri['locker'] = locker['locker_id'][-1]
-        racks[rack_name] = {
-            'name': rack_name,
-            'group': rack_group,
-            'role': rack_roles.get(locker['locker_id'][-1].lower()),
-            'type': '4-post cabinet',
-            'u_height': '36',
-            'width': '19',
-            'site': locker['site_location'],
-            'facility_id': sku_builder(base='', **rack_uri).lower(),
-        }
-
-    if 'f' == uri['locker']:
-        continue
-
-    l = {
-        'name': locker['name'],
-        'face': 'Front',
-        'device_type': '9U Locker',
-        'rack': rack_name,
-        'position': 1 if not zone_id.isnumeric() else 9 * (int(zone_id) - 1) + 1,
-        'site': locker['site_location'],
-        'device_role': 'Customer Network Locker' if locker['locker_id'][-1] == 'B' else 'Customer Compute Locker',
-        'custom_fields': {
-            'sf_id': locker['id'],
-        },
-    }
-
-    if locker['account']:
-        l['tenant'] = locker['account']
-
-    lockers.append(l)
-
-initializers_dir = here.parent / 'docker' / 'initializers'
-
-with open(initializers_dir / 'devices.yml', 'w+') as f:
-    f.write(yaml.dump(lockers, default_flow_style=False))
-
-with open(initializers_dir / 'racks.yml', 'w+') as f:
-    f.write(yaml.dump(list(racks.values()), default_flow_style=False))
-
-with open(initializers_dir / 'rack_groups.yml', 'w+') as f:
-    f.write(yaml.dump(list(rack_groups.values()), default_flow_style=False))
-
-#args.output.write(yaml.dump(output, default_flow_style=False))