Commit 9708c4fb authored by BAIRE Anthony's avatar BAIRE Anthony

formalise the definition, documentation and loading of env variables

- env variables are defined in 'config/env.py', this includes:
  - their default value
  - their documentation

- the loaded variables are stored as attributes of this module
  (eg: config.env.ALLGO_DEBUG)

- all variables are guaranteed to be set once the module is loaded
  thus, the config.settings just has to use env.ALLGO_XXXXX

- the sphinx documentation of the variables is generated directly
  from the source code

- the plumbing is done by the config.env_loader module (while loads
  the variables, and patches the docstring of the config.env module)
parent ee5c85fd
import os
from . import env_loader
with env_loader.EnvironmentVarLoader(__name__, "ALLGO_",
{"{ENV}": os.getenv("ENV", "ENV_IS_UNSET")}) as env_var:
#
# core django config
#
env_var("ALLGO_SECRET_KEY", help="secret key for django")
env_var("ALLGO_DEBUG",
default = "False",
help="enable debugging")
env_var("ALLGO_ALLOWED_HOSTS",
default="localhost,127.0.0.1,dev-django",
help="""list of hostnames that this server can serve over HTTP
In production, this value must be set to the fully qualified domain
name where this allgo instance is reachable.
""")
env_var("ALLGO_ADDITIONAL_APPS", default="",
help="comma-separated list of additional django apps to be enabled")
env_var("ALLGO_STATIC_PATH", fixed=True,
default="/var/www/html/static",
help="path when the static files are stored for deployment")
env_var("ALLGO_MEDIA_PATH", fixed=True,
default="/vol/rw/media",
help="path where the user-uploaded files are stored")
#
# email
#
env_var("ALLGO_EMAIL_BACKEND",
default ="django.core.mail.backends.smtp.EmailBackend",
help="django backend for sending emails")
env_var("ALLGO_EMAIL_FROM", default="no-reply@allgo.inria.fr",
help="sender e-mail address for outgoing mails")
env_var("ALLGO_EMAIL_HOST", default="smtp.inria.fr", help="host name of the SMTP relay")
env_var("ALLGO_EMAIL_PORT", default="25", help="tcp port of the SMTP relay")
env_var("ALLGO_EMAIL_USER", default="", help="user name for the SMTP relay")
env_var("ALLGO_EMAIL_PASSWORD", default="", help="password for the SMTP relay")
env_var("ALLGO_EMAIL_TLS", default="False", help="use SMTP over TLS")
#
# database
#
env_var("ALLGO_DATABASE_ENGINE", fixed=True,
default="django.db.backends.mysql",
help="django database engine")
env_var("ALLGO_DATABASE_NAME", fixed=True,
default="allgo",
help="database name")
env_var("ALLGO_DATABASE_USER", fixed=True,
default="allgo",
help="database user name")
env_var("ALLGO_DATABASE_PASSWORD",
default="allgo",
help="database password")
env_var("ALLGO_DATABASE_HOST", fixed=True,
default="{ENV}-mysql",
help="database host name")
env_var("ALLGO_DATABASE_MODE", fixed=True,
default="STRICT_ALL_TABLES",
help="""
database sql mode
see: https://mariadb.com/kb/en/library/sql-mode/
""")
#
# allgo-specific variables
#
env_var("ALLGO_CONTROLLER_HOST", fixed=True,
default="{ENV}-controller",
help="Hostname of the allgo controller")
env_var("ALLGO_CONTROLLER_PORT", fixed=True,
default="4567",
help="TCP port of the allgo controller (for the notifications)")
import json
import logging
import os
import re
import sys
import textwrap
log = logging.getLogger('allgo')
class EnvironmentVarLoader:
class UNSET:
pass
def __init__(self, module_name, prefix, replace={}):
"""new environment var loader
`module_name` is the name of the module where the environment
variables will be stored
`prefix` is the common prefix of all env variables
`replace` is a mapping of strings that must be replaced in the
value of loaded variables
"""
self.module = sys.modules[module_name]
self.prefix = prefix
self.replace = replace
# list of env vars for doc generation
# [(name, fixed, default, help), ...]
self.lst = []
# set of unseen variables to issue a warning if the user sets an
# unknown env variable)
self.unseen = {v for v in os.environ if v.startswith(prefix)}
def __call__(self, name, default=UNSET, *, fixed=False, help):
"""load the env var `name`
`default` is the default value for this variable if not present in
the environment
`fixed` boolean
- False: this variable is tunable by the user
- True: this is a "fixed" variable. It is not recommended
to change its value because it may break something.
Notes:
- occurences of "{ENV}" are expanded with os.environ["ENV"]
- the result as a attribute of self.module
- an unset value
"""
assert name.startswith(self.prefix) and re.match(r"[A-Z0-9_]+\Z", name)
self.unseen.discard(name)
# store the info for the doc generation
self.lst.append((name, fixed, default, help))
# read the environment variable
value = os.getenv(name, default)
if value is self.UNSET:
raise RuntimeError("environment variable %r must be set" % name)
# apply the substitutions
for orig, repl in self.replace.items():
value = value.replace(orig, repl)
# store the value as a module attribute
setattr(self.module, name, value)
def __enter__(self):
return self
def __exit__(self, *exc_info):
"""finish the loader task
- log the current config
- warn about unknown variables
- update the docstring (for the generation of the sphinx doc)
"""
# log the current configuration
for name, *_ in self.lst:
#FIXME: replace with "info"
log.warning("%s=%r", name, getattr(self.module, name))
# warn if we have unknown variables
for name in sorted(self.unseen):
log.warning("unexpected environment variable: %s", name)
# generate the doc
self._patch_docstring()
def _patch_docstring(self):
doc = [self.module.__doc__ or "", "\n"]
fmt_value = lambda v: "**(unset)**" if v is self.UNSET else (
"``'%s'``" % v)
# summary tables
def output_table(title, iterator):
nonlocal doc
doc += [ "\n\n.. csv-table :: %s" % title,
'\n :header: "name", "default value"',
'\n']
for name, fixed, default, help in iterator:
doc.append("\n %s, %s" % (name, json.dumps(fmt_value(default))))
doc += ["\n\n\n"]
output_table("list of allgo env variables that are tunable",
filter(lambda x: not x[1], self.lst))
output_table("list of allgo variables that should not be modified",
filter(lambda x: x[1], self.lst))
# detailed doc
for name, fixed, default, help in self.lst:
doc.append("\n\n.. envvar:: %s\n\n default: %s\n\n%s\n\n"
% (name, fmt_value(default),
textwrap.indent(textwrap.dedent(help).strip(), " ")))
self.module.__doc__ = "".join(doc)
import os
import sys
from . import env
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
APPS_DIR = os.path.join(ROOT_DIR, 'allgo')
def parse_bool(value):
if value.lower() in (1 ,"true"):
return True
if value.lower() in (0, "false"):
return False
raise ValueError("invalid value %r (expected 'true' or 'false')" % value)
# REQUIRED SETTINGS
# ------------------------------------------------------------------------------
# TODO
# if ALLGO_SECRET_KEY is empty:
# load .env
SECRET_KEY = os.environ['ALLGO_SECRET_KEY']
SECRET_KEY = env.ALLGO_SECRET_KEY
# GENERAL
# ------------------------------------------------------------------------------
DEBUG = os.environ.get('ALLGO_DEBUG', default=False)
ALLOWED_HOSTS = os.environ.get('ALLGO_ALLOWED_HOSTS', default='localhost').split(",")
DEBUG = env.ALLGO_DEBUG
ALLOWED_HOSTS = env.ALLGO_ALLOWED_HOSTS.split(",")
TIME_ZONE = 'UTC'
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
......@@ -29,17 +40,12 @@ USE_TZ = True
# ------------------------------------------------------------------------------
DATABASES = {
'default': {
'ENGINE': os.environ.get(
'ALLGO_DATABASE_ENGINE', default='django.db.backends.mysql'),
'NAME': os.environ.get('ALLGO_DATABASE_NAME', default='allgo'),
'USER': os.environ.get('ALLGO_DATABASE_USER', default='allgo'),
'PASSWORD': os.environ['ALLGO_DATABASE_PASSWORD'],
'HOST': os.environ.get('ALLGO_DATABASE_HOST', default='dev-mysql'),
'OPTIONS': {
'sql_mode': os.environ.get(
'ALLGO_DATABASE_MODE',
default='STRICT_ALL_TABLES'),
}
'ENGINE': env.ALLGO_DATABASE_ENGINE,
'NAME': env.ALLGO_DATABASE_NAME,
'USER': env.ALLGO_DATABASE_USER,
'PASSWORD': env.ALLGO_DATABASE_PASSWORD,
'HOST': env.ALLGO_DATABASE_HOST,
'OPTIONS': {'sql_mode': env.ALLGO_DATABASE_MODE }
}
}
DATABASES['default']['ATOMIC_REQUESTS'] = True
......@@ -53,6 +59,7 @@ WSGI_APPLICATION = 'config.wsgi.application'
# APPS
# ------------------------------------------------------------------------------
DJANGO_APPS = [
'django.contrib.admin',
'django.contrib.auth',
......@@ -73,11 +80,8 @@ LOCAL_APPS = [
'main',
]
if "ALLGO_ADDITIONAL_APPS" in os.environ:
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS + \
os.environ['ALLGO_ADDITIONAL_APPS'].split(",")
else:
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS + list(
filter(bool, env.ALLGO_ADDITIONAL_APPS.split(",")))
# MIGRATIONS
......@@ -136,9 +140,7 @@ MIDDLEWARE = [
# STATIC
# ------------------------------------------------------------------------------
STATIC_ROOT = os.environ.get(
'ALLGO_STATIC_PATH',
os.path.join(ROOT_DIR, 'static'))
STATIC_ROOT = env.ALLGO_STATIC_PATH
STATIC_URL = '/static/'
STATICFILES_DIRS = [
os.path.join(APPS_DIR, 'static'),
......@@ -151,9 +153,7 @@ STATICFILES_FINDERS = [
# MEDIA
# ------------------------------------------------------------------------------
MEDIA_ROOT = os.environ.get(
'ALLGO_MEDIA_PATH',
os.path.join(APPS_DIR, 'media'))
MEDIA_ROOT = env.ALLGO_MEDIA_PATH
MEDIA_URL = '/media/'
......@@ -188,17 +188,13 @@ TEMPLATES = [
# EMAIL
# ------------------------------------------------------------------------------
EMAIL_BACKEND = os.environ.get(
'ALLGO_EMAIL_BACKEND',
default='django.core.mail.backends.smtp.EmailBackend')
DEFAULT_FROM_EMAIL = os.environ.get(
'ALLGO_EMAIL_FROM',
default='no-reply@allgo.inria.fr')
EMAIL_HOST = os.environ.get('ALLGO_EMAIL_HOST', default='smtp.inria.fr')
EMAIL_PORT = os.environ.get('ALLGO_EMAIL_PORT', default=25)
EMAIL_HOST_USER = os.environ.get('ALLGO_EMAIL_USER', default='')
EMAIL_HOST_PASSWORD = os.environ.get('ALLGO_EMAIL_PASSWORD', default='')
EMAIL_USE_TLS = os.environ.get('ALLGO_EMAIL_TLS', default=False)
EMAIL_BACKEND = env.ALLGO_EMAIL_BACKEND
DEFAULT_FROM_EMAIL = env.ALLGO_EMAIL_FROM
EMAIL_HOST = env.ALLGO_EMAIL_HOST
EMAIL_PORT = env.ALLGO_EMAIL_PORT
EMAIL_HOST_USER = env.ALLGO_EMAIL_USER
EMAIL_HOST_PASSWORD = env.ALLGO_EMAIL_PASSWORD
EMAIL_USE_TLS = parse_bool(env.ALLGO_EMAIL_TLS)
# ADMIN
......
......@@ -152,30 +152,5 @@ Environment variables
All environment variables **must** be prefixed by `ALLGO`.
+-------------------------+-----------------------------------------------+
| VARIABLE | DEFAULT |
+=========================+===============================================+
| ALLGO_SECRET_KEY | None |
+-------------------------+-----------------------------------------------+
| ALLGO_DEBUG | False |
+-------------------------+-----------------------------------------------+
| ALLGO_ALLOWED_HOSTS | 'localhost' |
+-------------------------+-----------------------------------------------+
| ALLGO_DATABASE_ENGINE | 'django.db.backends.mysql' |
+-------------------------+-----------------------------------------------+
| ALLGO_DATABASE_NAME | 'allgo' |
+-------------------------+-----------------------------------------------+
| ALLGO_DATABASE_USER | 'allgo' |
+-------------------------+-----------------------------------------------+
| ALLGO_DATABASE_PASSWORD | None |
+-------------------------+-----------------------------------------------+
| ALLGO_DATABASE_MODE | 'STRICT_ALL_TABLES' |
+-------------------------+-----------------------------------------------+
| ALLGO_ADDITIONAL_APPS | None |
+-------------------------+-----------------------------------------------+
| ALLGO_EMAIL_BACKEND | 'django.core.mail.backends.smtp.EmailBackend' |
+-------------------------+-----------------------------------------------+
| ALLGO_STATIC_PATH | '<app_dir>/static' |
+-------------------------+-----------------------------------------------+
| ALLGO_MEDIA_PATH | '<app_dir>/media' |
+-------------------------+-----------------------------------------------+
.. automodule:: config.env
......@@ -33,15 +33,11 @@ services:
environment:
ENV: dev
PYTHONUNBUFFERED: 1
ALLGO_ALLOWED_HOSTS: 0.0.0.0,dev-django,localhost
DJANGO_DEBUG: 1
DJANGO_LOG_LEVEL: "DEBUG"
ALLGO_DEBUG: "True"
ALLGO_EMAIL_BACKEND: "django.core.mail.backends.console.EmailBackend"
ALLGO_SECRET_KEY: "nFgLEiedSJfYKyJA6WjkiGs8c23vokcVoM4DDLi9GsCX36TdsR"
ALLGO_DATABASE_PASSWORD: "allgo"
ALLGO_CONTROLLER_HOST: "dev-controller"
ALLGO_CONTROLLER_PORT: "4567"
SIGNING_KEY_PATH: "/certs/server.key"
SIGNING_KEY_TYPE: "RSA"
SIGNING_KEY_ALG: "RS256"
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment