diff --git a/.gitignore b/.gitignore index 30c001253..3ec36e925 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,7 @@ netbox.pid .vscode .venv .tox -!docker/configuration/gunicorn_config.py +!docker_old/configuration/gunicorn_config.py .idea .coverage .vscode diff --git a/.jenkins b/.jenkins index d7273fa3f..3b12d0aa7 100644 --- a/.jenkins +++ b/.jenkins @@ -1,7 +1,7 @@ #!/usr/bin/env groovy // Include this shared CI repository to load script helpers and libraries. -library identifier: 'vapor@1.16.0', retriever: modernSCM([ +library identifier: 'vapor@1.20.1', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/vapor-ware/ci-shared.git', credentialsId: 'vio-bot-gh', @@ -9,7 +9,7 @@ library identifier: 'vapor@1.16.0', retriever: modernSCM([ pythonPipeline([ - 'image': 'vaporio/netbox', + 'image': 'docker.io/vaporio/netbox', 'pythonVersion': '3.7', 'skipDocs': true, 'skipLint': true, @@ -23,7 +23,7 @@ spec: - name: dockerhub-vaporvecrobot containers: - name: python - image: vaporio/jenkins-agent-python37:master + image: docker.io/vaporio/jenkins-agent-python37:master imagePullPolicy: Always command: - cat @@ -33,7 +33,7 @@ spec: memory: 1Gi cpu: 500m - name: deploy - image: vaporio/deployment-tools:latest + image: docker.io/vaporio/deployment-tools:latest imagePullPolicy: Always command: - cat diff --git a/Dockerfile b/Dockerfile index a8cfb449b..afcd6c641 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,78 +1,84 @@ -FROM vaporio/python:3.7 as builder +FROM alpine:3.13 as builder -RUN apt-get update -qy \ - && apt-get install -y \ - libsasl2-dev \ +RUN apk add --no-cache \ + bash \ + build-base \ + cargo \ + ca-certificates \ + cyrus-sasl-dev \ graphviz \ - libjpeg-dev \ + jpeg-dev \ + libevent-dev \ libffi-dev \ - libxml2-dev \ - libxslt1-dev \ - libldap2-dev \ - libpq-dev \ - ttf-ubuntu-font-family \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /install - -RUN pip install --prefix="/install" --no-warn-script-location \ -# gunicorn is used for launching netbox - gunicorn \ - greenlet \ - eventlet \ -# napalm is used for gathering information from network devices - napalm \ -# ruamel is used in startup_scripts - 'ruamel.yaml>=0.15,<0.16' \ -# django-storages was introduced in 2.7 and is optional - django-storages + libressl-dev \ + libxslt-dev \ + musl-dev \ + openldap-dev \ + postgresql-dev \ + py3-pip \ + python3-dev \ + && python3 -m venv /opt/netbox/venv \ + && /opt/netbox/venv/bin/python3 -m pip install --upgrade \ + pip \ + setuptools \ + wheel ARG NETBOX_PATH=. -COPY ${NETBOX_PATH}/requirements.txt / -COPY ${NETBOX_PATH}/requirements.extras.txt / -RUN pip install --prefix="/install" --no-warn-script-location -r /requirements.txt -r /requirements.extras.txt +COPY ${NETBOX_PATH}/requirements.txt requirements.extras.txt / +RUN /opt/netbox/venv/bin/pip install \ + -r /requirements.txt \ + -r /requirements.extras.txt -FROM vaporio/python:3.7-slim +### +# Main stage +### -RUN apt-get update -qy \ - && apt-get install -y \ - libsasl2-dev \ +FROM alpine:3.13 as main + + +RUN apk add --no-cache \ + bash \ + ca-certificates \ + curl \ graphviz \ - libjpeg-dev \ - libffi-dev \ - libxml2-dev \ - libxslt1-dev \ - libldap2-dev \ - libpq-dev \ + libevent \ + libffi \ + libjpeg-turbo \ + libressl \ + libxslt \ + postgresql-libs \ + python3 \ + py3-pip \ ttf-ubuntu-font-family \ - && rm -rf /var/lib/apt/lists/* + unit \ + unit-python3 WORKDIR /opt -COPY --from=builder /install /usr/local +COPY --from=builder /opt/netbox/venv /opt/netbox/venv ARG NETBOX_PATH=. COPY ${NETBOX_PATH} /opt/netbox COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py -COPY docker/configuration/gunicorn_config.py /etc/netbox/config/ -COPY docker/nginx.conf /etc/netbox-nginx/nginx.conf COPY docker/docker-entrypoint.sh /opt/netbox/docker-entrypoint.sh +COPY docker/launch-netbox.sh /opt/netbox/launch-netbox.sh COPY docker/startup_scripts/ /opt/netbox/startup_scripts/ COPY docker/initializers/ /opt/netbox/initializers/ -COPY docker/configuration/configuration.py /etc/netbox/config/configuration.py +COPY docker/configuration/ /etc/netbox/config/ +COPY docker/nginx-unit.json /etc/unit/ WORKDIR /opt/netbox/netbox -RUN mkdir -p static && chmod g+w static media +# Must set permissions for '/opt/netbox/netbox/media' directory +# to g+w so that pictures can be uploaded to netbox. +RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ \ + && chmod -R g+w media /opt/unit/ \ + && SECRET_KEY="dummy" /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ] -CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"] - -ARG BUILD_VERSION -ARG BUILD_DATE -ARG VCS_REF +CMD [ "/opt/netbox/launch-netbox.sh" ] LABEL maintainer="Vapor IO" \ # See http://label-schema.org/rc1/#build-time-labels @@ -100,3 +106,13 @@ LABEL maintainer="Vapor IO" \ org.opencontainers.image.source="https://github.com/vapor-ware/netbox.git" \ org.opencontainers.image.revision=$VCS_REF \ org.opencontainers.image.version=$BUILD_VERSION + + +FROM main as ldap + +RUN apk add --no-cache \ + libsasl \ + libldap \ + util-linux + +COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 3d5cff207..9f4f578dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,74 +2,61 @@ version: '3.4' services: netbox: &netbox image: vaporio/netbox:develop - build: - context: . - args: - BUILD_VERSION: 'local-dev' - BUILD_DATE: '' - VCS_REF: 'tip' depends_on: - postgres - redis - redis-cache - netbox-worker env_file: docker/env/netbox.env - #user: '1000:1000' + user: '101' volumes: - - ./netbox:/opt/netbox/netbox - ./docker/startup_scripts:/opt/netbox/startup_scripts:z,ro - ./docker/initializers:/opt/netbox/initializers:z,ro - ./docker/configuration:/etc/netbox/config:z,ro - ./reports:/etc/netbox/reports:z,ro - ./scripts:/etc/netbox/scripts:z,ro - - netbox-nginx-config:/etc/netbox-nginx:z - - netbox-static-files:/opt/netbox/netbox/static:z - netbox-media-files:/opt/netbox/netbox/media:z + ports: + - "8000:8080" netbox-worker: <<: *netbox depends_on: - redis entrypoint: - - python3 + - /opt/netbox/venv/bin/python - /opt/netbox/netbox/manage.py command: - rqworker - nginx: - command: nginx -c /etc/netbox-nginx/nginx.conf - image: nginx:1.18 - depends_on: - - netbox - ports: - - 8000:8080 - volumes: - - netbox-static-files:/opt/netbox/netbox/static:ro - - netbox-nginx-config:/etc/netbox-nginx/:ro + ports: [ ] + postgres: - image: postgres:11-alpine - environment: - POSTGRES_PASSWORD: "12345" - POSTGRES_DB: netbox + image: postgres:12-alpine + env_file: docker/env/postgres.env ports: - - 5432:5432 + - "5432:5432" volumes: - netbox-postgres-data:/var/lib/postgresql/data + redis: - image: redis:5-alpine + image: redis:6-alpine command: - - redis-server - - --appendonly yes - - --requirepass 12345 - ports: - - 6379:6379 + - sh + - -c # this is to evaluate the $REDIS_PASSWORD from the env + - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose volumes: - netbox-redis-data:/data + ports: + - "6379:6379" + env_file: docker/env/redis.env redis-cache: - image: redis:5-alpine + image: redis:6-alpine + command: + - sh + - -c # this is to evaluate the $REDIS_PASSWORD from the env + - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose + env_file: docker/env/redis-cache.env + volumes: - netbox-static-files: - driver: local - netbox-nginx-config: - driver: local netbox-media-files: driver: local netbox-postgres-data: diff --git a/docker/configuration.docker.py b/docker/configuration.docker.py index 733887fc1..3cd5f7ba1 100644 --- a/docker/configuration.docker.py +++ b/docker/configuration.docker.py @@ -1,10 +1,81 @@ +## Generic Parts +# These functions are providing the functionality to load +# arbitrary configuration files. +# +# They can be imported by other code (see `ldap_config.py` for an example). + import importlib.util import sys +from os import scandir +from os.path import abspath, isfile -try: - spec = importlib.util.spec_from_file_location('configuration', '/etc/netbox/config/configuration.py') - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - sys.modules['netbox.configuration'] = module -except: - raise ImportError('') + +def _filename(f): + return f.name + + +def _import(module_name, path, loaded_configurations): + spec = importlib.util.spec_from_file_location("", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[module_name] = module + + loaded_configurations.insert(0, module) + + print(f"🧬 loaded config '{path}'") + + +def read_configurations(config_module, config_dir, main_config): + loaded_configurations = [] + + main_config_path = abspath(f"{config_dir}/{main_config}.py") + if isfile(main_config_path): + _import(f"{config_module}.{main_config}", main_config_path, loaded_configurations) + else: + print(f"⚠️ Main configuration '{main_config_path}' not found.") + + with scandir(config_dir) as it: + for f in sorted(it, key=_filename): + if not f.is_file(): + continue + + if f.name.startswith("__"): + continue + + if not f.name.endswith(".py"): + continue + + if f.name == f"{config_dir}.py": + continue + + module_name = f"{config_module}.{f.name[:-len('.py')]}".replace(".", "_") + _import(module_name, f.path, loaded_configurations) + + if len(loaded_configurations) == 0: + print(f"‼️ No configuration files found in '{config_dir}'.") + raise ImportError(f"No configuration files found in '{config_dir}'.") + + return loaded_configurations + + +## Specific Parts +# This section's code actually loads the various configuration files +# into the module with the given name. +# It contains the logic to resolve arbitrary configuration options by +# levaraging dynamic programming using `__getattr__`. + + +_loaded_configurations = read_configurations( + config_dir="/etc/netbox/config/", + config_module="netbox.configuration", + main_config="configuration", +) + + +def __getattr__(name): + for config in _loaded_configurations: + try: + return getattr(config, name) + except: + pass + raise AttributeError diff --git a/docker/configuration/configuration.py b/docker/configuration/configuration.py index 25a2afcd4..554286a10 100644 --- a/docker/configuration/configuration.py +++ b/docker/configuration/configuration.py @@ -1,21 +1,27 @@ -import os -import re -import socket +#### +## We recommend to not edit this file. +## Create separate files to overwrite the settings. +## See `extra.py` as an example. +#### -# For reference see http://netbox.readthedocs.io/en/latest/configuration/mandatory-settings/ -# Based on https://github.com/netbox-community/netbox/blob/develop/netbox/netbox/configuration.example.py +import re +from os import environ +from os.path import abspath, dirname, join + +# For reference see https://netbox.readthedocs.io/en/stable/configuration/ +# Based on https://github.com/netbox-community/netbox/blob/master/netbox/netbox/configuration.example.py # Read secret from file -def read_secret(secret_name): +def _read_secret(secret_name, default = None): try: f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8') except EnvironmentError: - return '' + return default else: with f: return f.readline().strip() -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_BASE_DIR = dirname(dirname(abspath(__file__))) ######################### # # @@ -27,47 +33,49 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. # # Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] -ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', '*').split(' ') +ALLOWED_HOSTS = environ.get('ALLOWED_HOSTS', '*').split(' ') -# PostgreSQL database configuration. +# PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: +# https://docs.djangoproject.com/en/stable/ref/settings/#databases DATABASE = { - 'NAME': os.environ.get('DB_NAME', 'netbox'), # Database name - 'USER': os.environ.get('DB_USER', ''), # PostgreSQL username - 'PASSWORD': os.environ.get('DB_PASSWORD', read_secret('db_password')), - # PostgreSQL password - 'HOST': os.environ.get('DB_HOST', 'localhost'), # Database server - 'PORT': os.environ.get('DB_PORT', ''), # Database port (leave blank for default) - 'OPTIONS': {'sslmode': os.environ.get('DB_SSLMODE', 'prefer')}, - # Database connection SSLMODE - 'CONN_MAX_AGE': int(os.environ.get('DB_CONN_MAX_AGE', '300')), - # Database connection persistence + 'NAME': environ.get('DB_NAME', 'netbox'), # Database name + 'USER': environ.get('DB_USER', ''), # PostgreSQL username + 'PASSWORD': _read_secret('db_password', environ.get('DB_PASSWORD', '')), + # PostgreSQL password + 'HOST': environ.get('DB_HOST', 'localhost'), # Database server + 'PORT': environ.get('DB_PORT', ''), # Database port (leave blank for default) + 'OPTIONS': {'sslmode': environ.get('DB_SSLMODE', 'prefer')}, + # Database connection SSLMODE + 'CONN_MAX_AGE': int(environ.get('DB_CONN_MAX_AGE', '300')), + # Max database connection age +} + +# Redis database settings. Redis is used for caching and for queuing background tasks such as webhook events. A separate +# configuration exists for each. Full connection details are required in both sections, and it is strongly recommended +# to use two separate database IDs. +REDIS = { + 'tasks': { + 'HOST': environ.get('REDIS_HOST', 'localhost'), + 'PORT': int(environ.get('REDIS_PORT', 6379)), + 'PASSWORD': _read_secret('redis_password', environ.get('REDIS_PASSWORD', '')), + 'DATABASE': int(environ.get('REDIS_DATABASE', 0)), + 'SSL': environ.get('REDIS_SSL', 'False').lower() == 'true', + }, + 'caching': { + 'HOST': environ.get('REDIS_CACHE_HOST', environ.get('REDIS_HOST', 'localhost')), + 'PORT': int(environ.get('REDIS_CACHE_PORT', environ.get('REDIS_PORT', 6379))), + 'PASSWORD': _read_secret('redis_cache_password', environ.get('REDIS_CACHE_PASSWORD', environ.get('REDIS_PASSWORD', ''))), + 'DATABASE': int(environ.get('REDIS_CACHE_DATABASE', 1)), + 'SSL': environ.get('REDIS_CACHE_SSL', environ.get('REDIS_SSL', 'False')).lower() == 'true', + }, } # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and # symbols. NetBox will not run without this defined. For more information, see -# https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-SECRET_KEY -SECRET_KEY = os.environ.get('SECRET_KEY', read_secret('secret_key')) +# https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY +SECRET_KEY = _read_secret('secret_key', environ.get('SECRET_KEY', '')) -# Redis database settings. The Redis database is used for caching and background processing such as webhooks -REDIS = { - 'tasks': { - 'HOST': os.environ.get('REDIS_HOST', 'localhost'), - 'PORT': int(os.environ.get('REDIS_PORT', 6379)), - 'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')), - 'DATABASE': int(os.environ.get('REDIS_DATABASE', 0)), - 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_TIMEOUT', 300)), - 'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true', - }, - 'caching': { - 'HOST': os.environ.get('REDIS_CACHE_HOST', os.environ.get('REDIS_HOST', 'localhost')), - 'PORT': int(os.environ.get('REDIS_CACHE_PORT', os.environ.get('REDIS_PORT', 6379))), - 'PASSWORD': os.environ.get('REDIS_CACHE_PASSWORD', os.environ.get('REDIS_PASSWORD', read_secret('redis_cache_password'))), - 'DATABASE': int(os.environ.get('REDIS_CACHE_DATABASE', 1)), - 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_CACHE_TIMEOUT', os.environ.get('REDIS_TIMEOUT', 300))), - 'SSL': os.environ.get('REDIS_CACHE_SSL', os.environ.get('REDIS_SSL', 'False')).lower() == 'true', - }, -} ######################### # # @@ -81,130 +89,162 @@ ADMINS = [ # ['John Doe', 'jdoe@example.com'], ] +# URL schemes that are allowed within links in NetBox +ALLOWED_URL_SCHEMES = ( + 'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp', +) + # Optionally display a persistent banner at the top and/or bottom of every page. HTML is allowed. To display the same # content in both banners, define BANNER_TOP and set BANNER_BOTTOM = BANNER_TOP. -BANNER_TOP = os.environ.get('BANNER_TOP', '') -BANNER_BOTTOM = os.environ.get('BANNER_BOTTOM', '') +BANNER_TOP = environ.get('BANNER_TOP', '') +BANNER_BOTTOM = environ.get('BANNER_BOTTOM', '') # Text to include on the login page above the login form. HTML is allowed. -BANNER_LOGIN = os.environ.get('BANNER_LOGIN', '') +BANNER_LOGIN = environ.get('BANNER_LOGIN', '') # Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set: # BASE_PATH = 'netbox/' -BASE_PATH = os.environ.get('BASE_PATH', '') +BASE_PATH = environ.get('BASE_PATH', '') # Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes) -CACHE_TIMEOUT = int(os.environ.get('CACHE_TIMEOUT', 900)) +CACHE_TIMEOUT = int(environ.get('CACHE_TIMEOUT', 900)) # Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90) -CHANGELOG_RETENTION = int(os.environ.get('CHANGELOG_RETENTION', 90)) +CHANGELOG_RETENTION = int(environ.get('CHANGELOG_RETENTION', 90)) # API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be # allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or # CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers -CORS_ORIGIN_ALLOW_ALL = os.environ.get('CORS_ORIGIN_ALLOW_ALL', 'False').lower() == 'true' -CORS_ORIGIN_WHITELIST = list(filter(None, os.environ.get('CORS_ORIGIN_WHITELIST', 'https://localhost').split(' '))) -CORS_ORIGIN_REGEX_WHITELIST = [re.compile(r) for r in list(filter(None, os.environ.get('CORS_ORIGIN_REGEX_WHITELIST', '').split(' ')))] +CORS_ORIGIN_ALLOW_ALL = environ.get('CORS_ORIGIN_ALLOW_ALL', 'False').lower() == 'true' +CORS_ORIGIN_WHITELIST = list(filter(None, environ.get('CORS_ORIGIN_WHITELIST', 'https://localhost').split(' '))) +CORS_ORIGIN_REGEX_WHITELIST = [re.compile(r) for r in list(filter(None, environ.get('CORS_ORIGIN_REGEX_WHITELIST', '').split(' ')))] # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging # on a production system. -DEBUG = os.environ.get('DEBUG', 'False').lower() == 'true' +DEBUG = environ.get('DEBUG', 'False').lower() == 'true' # Email settings EMAIL = { - 'SERVER': os.environ.get('EMAIL_SERVER', 'localhost'), - 'PORT': int(os.environ.get('EMAIL_PORT', 25)), - 'USERNAME': os.environ.get('EMAIL_USERNAME', ''), - 'PASSWORD': os.environ.get('EMAIL_PASSWORD', read_secret('email_password')), - 'TIMEOUT': int(os.environ.get('EMAIL_TIMEOUT', 10)), # seconds - 'FROM_EMAIL': os.environ.get('EMAIL_FROM', ''), - 'USE_SSL': os.environ.get('EMAIL_USE_SSL', 'False').lower() == 'true', - 'USE_TLS': os.environ.get('EMAIL_USE_TLS', 'False').lower() == 'true', - 'SSL_CERTFILE': os.environ.get('EMAIL_SSL_CERTFILE', ''), - 'SSL_KEYFILE': os.environ.get('EMAIL_SSL_KEYFILE', ''), + 'SERVER': environ.get('EMAIL_SERVER', 'localhost'), + 'PORT': int(environ.get('EMAIL_PORT', 25)), + 'USERNAME': environ.get('EMAIL_USERNAME', ''), + 'PASSWORD': _read_secret('email_password', environ.get('EMAIL_PASSWORD', '')), + 'USE_SSL': environ.get('EMAIL_USE_SSL', 'False').lower() == 'true', + 'USE_TLS': environ.get('EMAIL_USE_TLS', 'False').lower() == 'true', + 'SSL_CERTFILE': environ.get('EMAIL_SSL_CERTFILE', ''), + 'SSL_KEYFILE': environ.get('EMAIL_SSL_KEYFILE', ''), + 'TIMEOUT': int(environ.get('EMAIL_TIMEOUT', 10)), # seconds + 'FROM_EMAIL': environ.get('EMAIL_FROM', ''), } -# Enforcement of unique IP space can be toggled on a per-VRF basis. -# To enforce unique IP space within the global table (all prefixes and IP addresses not assigned to a VRF), -# set ENFORCE_GLOBAL_UNIQUE to True. -ENFORCE_GLOBAL_UNIQUE = os.environ.get('ENFORCE_GLOBAL_UNIQUE', 'False').lower() == 'true' +# Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table +# (all prefixes and IP addresses not assigned to a VRF), set ENFORCE_GLOBAL_UNIQUE to True. +ENFORCE_GLOBAL_UNIQUE = environ.get('ENFORCE_GLOBAL_UNIQUE', 'False').lower() == 'true' # Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and # by anonymous users. List models in the form `.`. Add '*' to this list to exempt all models. -EXEMPT_VIEW_PERMISSIONS = list(filter(None, os.environ.get('EXEMPT_VIEW_PERMISSIONS', '').split(' '))) +EXEMPT_VIEW_PERMISSIONS = list(filter(None, environ.get('EXEMPT_VIEW_PERMISSIONS', '').split(' '))) # Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs: -# https://docs.djangoproject.com/en/1.11/topics/logging/ +# https://docs.djangoproject.com/en/stable/topics/logging/ LOGGING = {} # Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users # are permitted to access most data in NetBox (excluding secrets) but not make any changes. -LOGIN_REQUIRED = os.environ.get('LOGIN_REQUIRED', 'False').lower() == 'true' +LOGIN_REQUIRED = environ.get('LOGIN_REQUIRED', 'False').lower() == 'true' + +# The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to +# re-authenticate. (Default: 1209600 [14 days]) +LOGIN_TIMEOUT = int(environ.get('LOGIN_TIMEOUT', 1209600)) # Setting this to True will display a "maintenance mode" banner at the top of every page. -MAINTENANCE_MODE = os.environ.get('MAINTENANCE_MODE', 'False').lower() == 'true' +MAINTENANCE_MODE = environ.get('MAINTENANCE_MODE', 'False').lower() == 'true' # An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g. # "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request # all objects by specifying "?limit=0". -MAX_PAGE_SIZE = int(os.environ.get('MAX_PAGE_SIZE', 1000)) +MAX_PAGE_SIZE = int(environ.get('MAX_PAGE_SIZE', 1000)) # The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that # the default value of this setting is derived from the installed location. -MEDIA_ROOT = os.environ.get('MEDIA_ROOT', os.path.join(BASE_DIR, 'media')) +MEDIA_ROOT = environ.get('MEDIA_ROOT', join(_BASE_DIR, 'media')) # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' -METRICS_ENABLED = os.environ.get('METRICS_ENABLED', 'False').lower() == 'true' +METRICS_ENABLED = environ.get('METRICS_ENABLED', 'False').lower() == 'true' -# Credentials that NetBox will use to access live devices. -NAPALM_USERNAME = os.environ.get('NAPALM_USERNAME', '') -NAPALM_PASSWORD = os.environ.get('NAPALM_PASSWORD', read_secret('napalm_password')) +# Credentials that NetBox will uses to authenticate to devices when connecting via NAPALM. +NAPALM_USERNAME = environ.get('NAPALM_USERNAME', '') +NAPALM_PASSWORD = _read_secret('napalm_password', environ.get('NAPALM_PASSWORD', '')) # NAPALM timeout (in seconds). (Default: 30) -NAPALM_TIMEOUT = int(os.environ.get('NAPALM_TIMEOUT', 30)) +NAPALM_TIMEOUT = int(environ.get('NAPALM_TIMEOUT', 30)) # NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must # be provided as a dictionary. NAPALM_ARGS = {} # Determine how many objects to display per page within a list. (Default: 50) -PAGINATE_COUNT = int(os.environ.get('PAGINATE_COUNT', 50)) +PAGINATE_COUNT = int(environ.get('PAGINATE_COUNT', 50)) -# Plugins +# Enable installed plugins. Add the name of each plugin to the list. PLUGINS = [ 'netbox_virtual_circuit_plugin', 'netbox_bgp', ] +# Plugins configuration settings. These settings are used by various plugins that the user may have installed. +# Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +PLUGINS_CONFIG = { +} + # When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to # prefer IPv4 instead. -PREFER_IPV4 = os.environ.get('PREFER_IPV4', 'False').lower() == 'true' +PREFER_IPV4 = environ.get('PREFER_IPV4', 'False').lower() == 'true' -# This determines how often the GitHub API is called to check the latest release of NetBox in seconds. Must be at least 1 hour. -RELEASE_CHECK_TIMEOUT = os.environ.get('RELEASE_CHECK_TIMEOUT', 24 * 3600) +# Rack elevation size defaults, in pixels. For best results, the ratio of width to height should be roughly 10:1. +RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = int(environ.get('RACK_ELEVATION_DEFAULT_UNIT_HEIGHT', 22)) +RACK_ELEVATION_DEFAULT_UNIT_WIDTH = int(environ.get('RACK_ELEVATION_DEFAULT_UNIT_WIDTH', 220)) + +# Remote authentication support +REMOTE_AUTH_ENABLED = environ.get('REMOTE_AUTH_ENABLED', 'False').lower() == 'true' +REMOTE_AUTH_BACKEND = environ.get('REMOTE_AUTH_BACKEND', 'netbox.authentication.RemoteUserBackend') +REMOTE_AUTH_HEADER = environ.get('REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER') +REMOTE_AUTH_AUTO_CREATE_USER = environ.get('REMOTE_AUTH_AUTO_CREATE_USER', 'True').lower() == 'true' +REMOTE_AUTH_DEFAULT_GROUPS = list(filter(None, environ.get('REMOTE_AUTH_DEFAULT_GROUPS', '').split(' '))) + +# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour. +RELEASE_CHECK_TIMEOUT = int(environ.get('RELEASE_CHECK_TIMEOUT', 24 * 3600)) # This repository is used to check whether there is a new release of NetBox available. Set to None to disable the # version check or use the URL below to check for release in the official NetBox repository. # https://api.github.com/repos/netbox-community/netbox/releases -RELEASE_CHECK_URL = os.environ.get('RELEASE_CHECK_URL', None) +RELEASE_CHECK_URL = environ.get('RELEASE_CHECK_URL', None) # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of # this setting is derived from the installed location. -REPORTS_ROOT = os.environ.get('REPORTS_ROOT', '/etc/netbox/reports') +REPORTS_ROOT = environ.get('REPORTS_ROOT', '/etc/netbox/reports') + +# Maximum execution time for background tasks, in seconds. +RQ_DEFAULT_TIMEOUT = int(environ.get('RQ_DEFAULT_TIMEOUT', 300)) # The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of # this setting is derived from the installed location. -SCRIPTS_ROOT = os.environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts') +SCRIPTS_ROOT = environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts') + +# By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use +# local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only +# database access.) Note that the user as which NetBox runs must have read and write permissions to this path. +SESSION_FILE_PATH = environ.get('SESSIONS_ROOT', None) # Time zone (default: UTC) -TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC') +TIME_ZONE = environ.get('TIME_ZONE', 'UTC') # Date/time formatting. See the following link for supported formats: -# https://docs.djangoproject.com/en/dev/ref/templates/builtins/#date -DATE_FORMAT = os.environ.get('DATE_FORMAT', 'N j, Y') -SHORT_DATE_FORMAT = os.environ.get('SHORT_DATE_FORMAT', 'Y-m-d') -TIME_FORMAT = os.environ.get('TIME_FORMAT', 'g:i a') -SHORT_TIME_FORMAT = os.environ.get('SHORT_TIME_FORMAT', 'H:i:s') -DATETIME_FORMAT = os.environ.get('DATETIME_FORMAT', 'N j, Y g:i a') -SHORT_DATETIME_FORMAT = os.environ.get('SHORT_DATETIME_FORMAT', 'Y-m-d H:i') +# https://docs.djangoproject.com/en/stable/ref/templates/builtins/#date +DATE_FORMAT = environ.get('DATE_FORMAT', 'N j, Y') +SHORT_DATE_FORMAT = environ.get('SHORT_DATE_FORMAT', 'Y-m-d') +TIME_FORMAT = environ.get('TIME_FORMAT', 'g:i a') +SHORT_TIME_FORMAT = environ.get('SHORT_TIME_FORMAT', 'H:i:s') +DATETIME_FORMAT = environ.get('DATETIME_FORMAT', 'N j, Y g:i a') +SHORT_DATETIME_FORMAT = environ.get('SHORT_DATETIME_FORMAT', 'Y-m-d H:i') diff --git a/docker/configuration/extra.py b/docker/configuration/extra.py new file mode 100644 index 000000000..46f1877ed --- /dev/null +++ b/docker/configuration/extra.py @@ -0,0 +1,55 @@ +#### +## This file contains extra configuration options that can't be configured +## directly through environment variables. +#### + +## Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of +## application errors (assuming correct email settings are provided). +# ADMINS = [ +# # ['John Doe', 'jdoe@example.com'], +# ] + + +## URL schemes that are allowed within links in NetBox +# ALLOWED_URL_SCHEMES = ( +# 'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp', +# ) + + +## NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must +## be provided as a dictionary. +# NAPALM_ARGS = {} + + +## Enable installed plugins. Add the name of each plugin to the list. +# from netbox.configuration.configuration import PLUGINS +# PLUGINS.append('my_plugin') + +## Plugins configuration settings. These settings are used by various plugins that the user may have installed. +## Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +# from netbox.configuration.configuration import PLUGINS_CONFIG +# PLUGINS_CONFIG['my_plugin'] = { +# 'foo': 'bar', +# 'buzz': 'bazz' +# } + + +## Remote authentication support +# REMOTE_AUTH_DEFAULT_PERMISSIONS = {} + + +## By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +## class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', +# } + + +## This file can contain arbitrary Python code, e.g.: +# from datetime import datetime +# now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") +# BANNER_TOP = f'This instance started on {now}.' diff --git a/docker/configuration/gunicorn_config.py b/docker/configuration/gunicorn_config.py deleted file mode 100644 index 063d8224c..000000000 --- a/docker/configuration/gunicorn_config.py +++ /dev/null @@ -1,8 +0,0 @@ -command = '/usr/bin/gunicorn' -pythonpath = '/opt/netbox/netbox' -bind = '0.0.0.0:8001' -workers = 3 -errorlog = '-' -accesslog = '-' -capture_output = False -loglevel = 'debug' diff --git a/docker/configuration/ldap/ldap_config.py b/docker/configuration/ldap/ldap_config.py new file mode 100644 index 000000000..3071b4581 --- /dev/null +++ b/docker/configuration/ldap/ldap_config.py @@ -0,0 +1,90 @@ +from importlib import import_module +from os import environ + +import ldap +from django_auth_ldap.config import LDAPSearch + + +# Read secret from file +def _read_secret(secret_name, default=None): + try: + f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8') + except EnvironmentError: + return default + else: + with f: + return f.readline().strip() + +# Import and return the group type based on string name +def _import_group_type(group_type_name): + mod = import_module('django_auth_ldap.config') + try: + return getattr(mod, group_type_name)() + except: + return None + +# Server URI +AUTH_LDAP_SERVER_URI = environ.get('AUTH_LDAP_SERVER_URI', '') + +# The following may be needed if you are binding to Active Directory. +AUTH_LDAP_CONNECTION_OPTIONS = { + ldap.OPT_REFERRALS: 0 +} + +# Set the DN and password for the NetBox service account. +AUTH_LDAP_BIND_DN = environ.get('AUTH_LDAP_BIND_DN', '') +AUTH_LDAP_BIND_PASSWORD = _read_secret('auth_ldap_bind_password', environ.get('AUTH_LDAP_BIND_PASSWORD', '')) + +# Set a string template that describes any user’s distinguished name based on the username. +AUTH_LDAP_USER_DN_TEMPLATE = environ.get('AUTH_LDAP_USER_DN_TEMPLATE', None) + +# Enable STARTTLS for ldap authentication. +AUTH_LDAP_START_TLS = environ.get('AUTH_LDAP_START_TLS', 'False').lower() == 'true' + +# Include this setting if you want to ignore certificate errors. This might be needed to accept a self-signed cert. +# Note that this is a NetBox-specific setting which sets: +# ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER) +LDAP_IGNORE_CERT_ERRORS = environ.get('LDAP_IGNORE_CERT_ERRORS', 'False').lower() == 'true' + +AUTH_LDAP_USER_SEARCH_BASEDN = environ.get('AUTH_LDAP_USER_SEARCH_BASEDN', '') +AUTH_LDAP_USER_SEARCH_ATTR = environ.get('AUTH_LDAP_USER_SEARCH_ATTR', 'sAMAccountName') +AUTH_LDAP_USER_SEARCH = LDAPSearch( + AUTH_LDAP_USER_SEARCH_BASEDN, + ldap.SCOPE_SUBTREE, + "(" + AUTH_LDAP_USER_SEARCH_ATTR + "=%(user)s)" +) + +# This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group +# heirarchy. +AUTH_LDAP_GROUP_SEARCH_BASEDN = environ.get('AUTH_LDAP_GROUP_SEARCH_BASEDN', '') +AUTH_LDAP_GROUP_SEARCH_CLASS = environ.get('AUTH_LDAP_GROUP_SEARCH_CLASS', 'group') +AUTH_LDAP_GROUP_SEARCH = LDAPSearch(AUTH_LDAP_GROUP_SEARCH_BASEDN, ldap.SCOPE_SUBTREE, + "(objectClass=" + AUTH_LDAP_GROUP_SEARCH_CLASS + ")") +AUTH_LDAP_GROUP_TYPE = _import_group_type(environ.get('AUTH_LDAP_GROUP_TYPE', 'GroupOfNamesType')) + +# Define a group required to login. +AUTH_LDAP_REQUIRE_GROUP = environ.get('AUTH_LDAP_REQUIRE_GROUP_DN') + +# Define special user types using groups. Exercise great caution when assigning superuser status. +AUTH_LDAP_USER_FLAGS_BY_GROUP = {} + +if AUTH_LDAP_REQUIRE_GROUP is not None: + AUTH_LDAP_USER_FLAGS_BY_GROUP = { + "is_active": environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', ''), + "is_staff": environ.get('AUTH_LDAP_IS_ADMIN_DN', ''), + "is_superuser": environ.get('AUTH_LDAP_IS_SUPERUSER_DN', '') + } + +# For more granular permissions, we can map LDAP groups to Django groups. +AUTH_LDAP_FIND_GROUP_PERMS = environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true' +AUTH_LDAP_MIRROR_GROUPS = environ.get('AUTH_LDAP_MIRROR_GROUPS', '').lower() == 'true' + +# Cache groups for one hour to reduce LDAP traffic +AUTH_LDAP_CACHE_TIMEOUT = int(environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600)) + +# Populate the Django user from the LDAP directory. +AUTH_LDAP_USER_ATTR_MAP = { + "first_name": environ.get('AUTH_LDAP_ATTR_FIRSTNAME', 'givenName'), + "last_name": environ.get('AUTH_LDAP_ATTR_LASTNAME', 'sn'), + "email": environ.get('AUTH_LDAP_ATTR_MAIL', 'mail') +} diff --git a/docker/configuration/ldap_config.py b/docker/configuration/ldap_config.py deleted file mode 100644 index ba2067c42..000000000 --- a/docker/configuration/ldap_config.py +++ /dev/null @@ -1,80 +0,0 @@ -import ldap -import os - -from django_auth_ldap.config import LDAPSearch -from importlib import import_module - -# Read secret from file -def read_secret(secret_name): - try: - f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8') - except EnvironmentError: - return '' - else: - with f: - return f.readline().strip() - -# Import and return the group type based on string name -def import_group_type(group_type_name): - mod = import_module('django_auth_ldap.config') - try: - return getattr(mod, group_type_name)() - except: - return None - -# Server URI -AUTH_LDAP_SERVER_URI = os.environ.get('AUTH_LDAP_SERVER_URI', '') - -# The following may be needed if you are binding to Active Directory. -AUTH_LDAP_CONNECTION_OPTIONS = { - ldap.OPT_REFERRALS: 0 -} - -# Set the DN and password for the NetBox service account. -AUTH_LDAP_BIND_DN = os.environ.get('AUTH_LDAP_BIND_DN', '') -AUTH_LDAP_BIND_PASSWORD = os.environ.get('AUTH_LDAP_BIND_PASSWORD', read_secret('auth_ldap_bind_password')) - -# Set a string template that describes any user’s distinguished name based on the username. -AUTH_LDAP_USER_DN_TEMPLATE = os.environ.get('AUTH_LDAP_USER_DN_TEMPLATE', None) - -# Include this setting if you want to ignore certificate errors. This might be needed to accept a self-signed cert. -# Note that this is a NetBox-specific setting which sets: -# ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER) -LDAP_IGNORE_CERT_ERRORS = os.environ.get('LDAP_IGNORE_CERT_ERRORS', 'False').lower() == 'true' - -AUTH_LDAP_USER_SEARCH_BASEDN = os.environ.get('AUTH_LDAP_USER_SEARCH_BASEDN', '') -AUTH_LDAP_USER_SEARCH_ATTR = os.environ.get('AUTH_LDAP_USER_SEARCH_ATTR', 'sAMAccountName') -AUTH_LDAP_USER_SEARCH = LDAPSearch(AUTH_LDAP_USER_SEARCH_BASEDN, - ldap.SCOPE_SUBTREE, - "(" + AUTH_LDAP_USER_SEARCH_ATTR + "=%(user)s)") - -# This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group -# heirarchy. -AUTH_LDAP_GROUP_SEARCH_BASEDN = os.environ.get('AUTH_LDAP_GROUP_SEARCH_BASEDN', '') -AUTH_LDAP_GROUP_SEARCH_CLASS = os.environ.get('AUTH_LDAP_GROUP_SEARCH_CLASS', 'group') -AUTH_LDAP_GROUP_SEARCH = LDAPSearch(AUTH_LDAP_GROUP_SEARCH_BASEDN, ldap.SCOPE_SUBTREE, - "(objectClass=" + AUTH_LDAP_GROUP_SEARCH_CLASS + ")") -AUTH_LDAP_GROUP_TYPE = import_group_type(os.environ.get('AUTH_LDAP_GROUP_TYPE', 'GroupOfNamesType')) - -# Define a group required to login. -AUTH_LDAP_REQUIRE_GROUP = os.environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', '') - -# Define special user types using groups. Exercise great caution when assigning superuser status. -AUTH_LDAP_USER_FLAGS_BY_GROUP = { - "is_active": os.environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', ''), - "is_staff": os.environ.get('AUTH_LDAP_IS_ADMIN_DN', ''), - "is_superuser": os.environ.get('AUTH_LDAP_IS_SUPERUSER_DN', '') -} - -# For more granular permissions, we can map LDAP groups to Django groups. -AUTH_LDAP_FIND_GROUP_PERMS = os.environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true' - -# Cache groups for one hour to reduce LDAP traffic -AUTH_LDAP_CACHE_TIMEOUT = int(os.environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600)) - -# Populate the Django user from the LDAP directory. -AUTH_LDAP_USER_ATTR_MAP = { - "first_name": os.environ.get('AUTH_LDAP_ATTR_FIRSTNAME', 'givenName'), - "last_name": os.environ.get('AUTH_LDAP_ATTR_LASTNAME', 'sn'), - "email": os.environ.get('AUTH_LDAP_ATTR_MAIL', 'mail') -} diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index b4479040b..c795d6a6f 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,9 +1,16 @@ #!/bin/bash +# Runs on every start of the NetBox Docker container + +# Stop when an error occures set -e -# Allows Netbox to be run as non-root users +# Allows NetBox to be run as non-root users umask 002 +# Load correct Python3 env +# shellcheck disable=SC1091 +source /opt/netbox/venv/bin/activate + # Try to connect to the DB DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3} MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30} @@ -11,7 +18,7 @@ CUR_DB_WAIT_TIME=0 while ! ./manage.py migrate 2>&1 && [ "${CUR_DB_WAIT_TIME}" -lt "${MAX_DB_WAIT_TIME}" ]; do echo "⏳ Waiting on DB... (${CUR_DB_WAIT_TIME}s / ${MAX_DB_WAIT_TIME}s)" sleep "${DB_WAIT_TIMEOUT}" - CUR_DB_WAIT_TIME=$(( CUR_DB_WAIT_TIME + DB_WAIT_TIMEOUT )) + CUR_DB_WAIT_TIME=$((CUR_DB_WAIT_TIME + DB_WAIT_TIMEOUT)) done if [ "${CUR_DB_WAIT_TIME}" -ge "${MAX_DB_WAIT_TIME}" ]; then echo "❌ Waited ${MAX_DB_WAIT_TIME}s or more for the DB to become ready." @@ -29,17 +36,17 @@ else SUPERUSER_EMAIL='admin@example.com' fi if [ -f "/run/secrets/superuser_password" ]; then - SUPERUSER_PASSWORD="$(< /run/secrets/superuser_password)" + SUPERUSER_PASSWORD="$(&2 - -EXIT=0 -RED='\033[0;31m' -NOCOLOR='\033[0m' - -if [ -d ./venv/ ]; then - VENV="$PWD/venv" - if [ -e $VENV/bin/python ]; then - PATH=$VENV/bin:$PATH - elif [ -e $VENV/Scripts/python.exe ]; then - PATH=$VENV/Scripts:$PATH - fi -fi - -echo "Validating PEP8 compliance..." -pycodestyle --ignore=W504,E501 netbox/ -if [ $? != 0 ]; then - EXIT=1 -fi - -echo "Checking for missing migrations..." -python netbox/manage.py makemigrations --dry-run --check -if [ $? != 0 ]; then - EXIT=1 -fi - -if [ $EXIT != 0 ]; then - printf "${RED}COMMIT FAILED${NOCOLOR}\n" -fi - -exit $EXIT diff --git a/scripts/management-api-to-initializers b/scripts/management-api-to-initializers deleted file mode 100755 index 3a840ae03..000000000 --- a/scripts/management-api-to-initializers +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python3 - -import os -import csv -import sys -import yaml -import argparse - -from pathlib import Path - -here = Path(__file__).parent - -def sku_parser(uri): - resource = uri.lower().replace('.vio.sh', '').replace('-', '.') - - data = { - 'market': None, - 'site': None, - 'cluster': None, - 'rack': None, - 'locker': None, - } - - parts = resource.split('.') - parts.reverse() - - data.update(dict(zip(data.keys(), parts))) - - return data - -def sku_builder(*, market=None, site=None, cluster=None, rack=None, locker=None, base='vio.ke', sep='.'): - uri = base or '' - - if not market: - return uri - - uri = f'{market}{sep}{uri}' - - if uri.endswith(sep): - uri = uri[:-1] - - if not site: - return uri - - uri = f'{site}{sep}{uri}' - - if not cluster: - return uri - - uri = f'{cluster}{sep}{uri}' - - if not rack: - return uri - - uri = f'{rack}{sep}{uri}' - - if not locker: - return uri - - return f'{locker}{sep}{uri}' - - -parser = argparse.ArgumentParser(description='Managemenet API csv export to initializer yaml') -parser.add_argument( - 'lockers', - type=argparse.FileType('r'), - metavar='export.csv', - help='lockers csv export' -) - -lockers = [] -racks = {} -rack_groups = {} - -args = parser.parse_args() - -data = [r for r in csv.reader(args.lockers)] -header = data.pop(0) -locker_data = [dict(zip(header, r)) for r in data] - -lockers = [] - -rack_roles = { - 'f': 'Facility', - 'a': 'Customer', - 'b': 'Utility', -} - -for locker in locker_data: - rack_group = locker['name'].rsplit(' ', 1)[0] - uri = sku_parser(locker['sku']) - - if 'r' in uri['rack']: - continue - - zone_id = locker['zone_id'] - - if 'w' in uri['rack']: - rack_name = '{} {}'.format(rack_group, 'Facility' if locker['locker_id'].lower() == 'f' else 'Zone {}'.format(locker['locker_id'][-1])) - - if not rack_group in rack_groups: - rack_uri = uri.copy() - del rack_uri['locker'] - rack_groups[rack_group] = { - 'name': rack_group, - 'site': locker['site_location'], - 'slug': sku_builder(sep='-', base='', **rack_uri).lower(), - } - - if not rack_name in racks: - rack_uri = uri.copy() - rack_uri['locker'] = locker['locker_id'][-1] - racks[rack_name] = { - 'name': rack_name, - 'group': rack_group, - 'role': rack_roles.get(locker['locker_id'][-1].lower()), - 'type': '4-post cabinet', - 'u_height': '36', - 'width': '19', - 'site': locker['site_location'], - 'facility_id': sku_builder(base='', **rack_uri).lower(), - } - - if 'f' == uri['locker']: - continue - - l = { - 'name': locker['name'], - 'face': 'Front', - 'device_type': '9U Locker', - 'rack': rack_name, - 'position': 1 if not zone_id.isnumeric() else 9 * (int(zone_id) - 1) + 1, - 'site': locker['site_location'], - 'device_role': 'Customer Network Locker' if locker['locker_id'][-1] == 'B' else 'Customer Compute Locker', - 'custom_fields': { - 'sf_id': locker['id'], - }, - } - - if locker['account']: - l['tenant'] = locker['account'] - - lockers.append(l) - -initializers_dir = here.parent / 'docker' / 'initializers' - -with open(initializers_dir / 'devices.yml', 'w+') as f: - f.write(yaml.dump(lockers, default_flow_style=False)) - -with open(initializers_dir / 'racks.yml', 'w+') as f: - f.write(yaml.dump(list(racks.values()), default_flow_style=False)) - -with open(initializers_dir / 'rack_groups.yml', 'w+') as f: - f.write(yaml.dump(list(rack_groups.values()), default_flow_style=False)) - -#args.output.write(yaml.dump(output, default_flow_style=False))