formalise the definition, documentation and loading of env variables

- env variables are defined in 'config/env.py', this includes:
  - their default value
  - their documentation

- the loaded variables are stored as attributes of this module
  (eg: config.env.ALLGO_DEBUG)

- all variables are guaranteed to be set once the module is loaded
  thus, the config.settings just has to use env.ALLGO_XXXXX

- the sphinx documentation of the variables is generated directly
  from the source code

- the plumbing is done by the config.env_loader module (while loads
  the variables, and patches the docstring of the config.env module)

django/config/env.py

+import os
+from . import env_loader
+
+with env_loader.EnvironmentVarLoader(__name__, "ALLGO_",
+        {"{ENV}": os.getenv("ENV", "ENV_IS_UNSET")}) as env_var:
+
+    #
+    # core django config
+    #
+
+    env_var("ALLGO_SECRET_KEY", help="secret key for django")
+
+    env_var("ALLGO_DEBUG",
+            default = "False",
+            help="enable debugging")
+
+    env_var("ALLGO_ALLOWED_HOSTS",
+            default="localhost,127.0.0.1,dev-django",
+            help="""list of hostnames that this server can serve over HTTP
+            
+            In production, this value must be set to the fully qualified domain
+            name where this allgo instance is reachable.
+            """)
+
+    env_var("ALLGO_ADDITIONAL_APPS", default="",
+            help="comma-separated list of additional django apps to be enabled")
+
+    env_var("ALLGO_STATIC_PATH", fixed=True,
+            default="/var/www/html/static",
+            help="path when the static files are stored for deployment")
+
+    env_var("ALLGO_MEDIA_PATH", fixed=True,
+            default="/vol/rw/media",
+            help="path where the user-uploaded files are stored")
+
+    #
+    # email
+    #
+
+    env_var("ALLGO_EMAIL_BACKEND",
+            default ="django.core.mail.backends.smtp.EmailBackend",
+            help="django backend for sending emails")
+
+    env_var("ALLGO_EMAIL_FROM", default="no-reply@allgo.inria.fr",
+            help="sender e-mail address for outgoing mails")
+
+    env_var("ALLGO_EMAIL_HOST", default="smtp.inria.fr", help="host name of the SMTP relay")
+    env_var("ALLGO_EMAIL_PORT", default="25",            help="tcp port of the SMTP relay")
+    env_var("ALLGO_EMAIL_USER", default="",              help="user name for the SMTP relay")
+    env_var("ALLGO_EMAIL_PASSWORD", default="",          help="password for the SMTP relay")
+    env_var("ALLGO_EMAIL_TLS",  default="False",         help="use SMTP over TLS")
+
+    #
+    # database
+    #
+
+    env_var("ALLGO_DATABASE_ENGINE", fixed=True,
+            default="django.db.backends.mysql",
+            help="django database engine")
+
+    env_var("ALLGO_DATABASE_NAME", fixed=True,
+            default="allgo",
+            help="database name")
+
+    env_var("ALLGO_DATABASE_USER", fixed=True,
+            default="allgo",
+            help="database user name")
+
+    env_var("ALLGO_DATABASE_PASSWORD",
+            default="allgo",
+            help="database password")
+
+    env_var("ALLGO_DATABASE_HOST", fixed=True,
+            default="{ENV}-mysql",
+            help="database host name")
+
+    env_var("ALLGO_DATABASE_MODE", fixed=True,
+            default="STRICT_ALL_TABLES",
+            help="""
+            database sql mode
+            
+            see: https://mariadb.com/kb/en/library/sql-mode/
+            """)
+
+
+    #
+    # allgo-specific variables
+    #
+
+    env_var("ALLGO_CONTROLLER_HOST", fixed=True,
+            default="{ENV}-controller",
+            help="Hostname of the allgo controller")
+
+    env_var("ALLGO_CONTROLLER_PORT", fixed=True,
+            default="4567",
+            help="TCP port of the allgo controller (for the notifications)")
+

django/config/env_loader.py

+import json
+import logging
+import os
+import re
+import sys
+import textwrap
+
+log = logging.getLogger('allgo')
+
+class EnvironmentVarLoader:
+    class UNSET:
+        pass
+
+    def __init__(self, module_name, prefix, replace={}):
+        """new environment var loader
+
+        `module_name`   is the name of the module where the environment
+                        variables will be stored
+        `prefix`        is the common prefix of all env variables
+        `replace`       is a mapping of strings that must be replaced in the
+                        value of loaded variables
+        """
+        self.module = sys.modules[module_name]
+        self.prefix = prefix
+        self.replace = replace
+
+        # list of env vars for doc generation
+        # [(name, fixed, default, help), ...]
+        self.lst = []
+
+        # set of unseen variables to issue a warning if the user sets an
+        # unknown env variable)
+        self.unseen = {v for v in os.environ if v.startswith(prefix)}
+
+    def __call__(self, name, default=UNSET, *, fixed=False, help):
+        """load the env var `name`
+        
+        `default`   is the default value for this variable if not present in
+                    the environment
+        `fixed`     boolean
+                    - False: this variable is tunable by the user 
+                    - True:  this is a "fixed" variable. It is not recommended
+                            to change its value because it may break something.
+        Notes:
+        - occurences of "{ENV}" are expanded with os.environ["ENV"]
+        - the result as a attribute of self.module
+        - an unset value 
+        """
+
+        assert name.startswith(self.prefix) and re.match(r"[A-Z0-9_]+\Z", name)
+        self.unseen.discard(name)
+
+        # store the info for the doc generation
+        self.lst.append((name, fixed, default, help))
+
+        # read the environment variable
+        value = os.getenv(name, default)
+        if value is self.UNSET:
+            raise RuntimeError("environment variable %r must be set" % name)
+
+        # apply the substitutions
+        for orig, repl in self.replace.items():
+            value = value.replace(orig, repl)
+
+        # store the value as a module attribute
+        setattr(self.module, name, value)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_info):
+        """finish the loader task
+        
+        - log the current config
+        - warn about unknown variables
+        - update the docstring (for the generation of the sphinx doc)
+        """       
+
+        # log the current configuration
+        for name, *_ in self.lst:
+            #FIXME: replace with "info"
+            log.warning("%s=%r", name, getattr(self.module, name))
+
+        # warn if we have unknown variables
+        for name in sorted(self.unseen):
+            log.warning("unexpected environment variable: %s", name)
+
+        # generate the doc
+        self._patch_docstring()
+
+    def _patch_docstring(self):
+        doc = [self.module.__doc__ or "", "\n"]
+
+        fmt_value = lambda v: "**(unset)**" if v is self.UNSET else (
+                "``'%s'``" % v)
+    
+        # summary tables
+        def output_table(title, iterator):
+            nonlocal doc
+            doc += [ "\n\n.. csv-table :: %s" % title,
+                    '\n :header: "name", "default value"',
+                    '\n']
+            for name, fixed, default, help in iterator:
+                doc.append("\n %s, %s" % (name, json.dumps(fmt_value(default))))
+            doc += ["\n\n\n"]
+
+        output_table("list of allgo env variables that are tunable",
+                filter(lambda x: not x[1], self.lst))
+        output_table("list of allgo variables that should not be modified",
+                filter(lambda x: x[1], self.lst))
+
+        # detailed doc
+        for name, fixed, default, help in self.lst:
+            doc.append("\n\n.. envvar:: %s\n\n default: %s\n\n%s\n\n"
+                    % (name, fmt_value(default),
+                        textwrap.indent(textwrap.dedent(help).strip(), " ")))
+
+        self.module.__doc__ = "".join(doc)
+

django/config/settings.py

 import os
 import sys
 
+from . import env
+
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 APPS_DIR = os.path.join(ROOT_DIR, 'allgo')
 
+
+def parse_bool(value):
+    if value.lower() in (1 ,"true"):
+        return True
+    if value.lower() in (0, "false"):
+        return False
+    raise ValueError("invalid value %r (expected 'true' or 'false')" % value)
+
+
+
 # REQUIRED SETTINGS
 # ------------------------------------------------------------------------------
 # TODO
 # if ALLGO_SECRET_KEY is empty:
 #   load .env
-SECRET_KEY = os.environ['ALLGO_SECRET_KEY']
+SECRET_KEY = env.ALLGO_SECRET_KEY
 
 
 # GENERAL
 # ------------------------------------------------------------------------------
-DEBUG = os.environ.get('ALLGO_DEBUG', default=False)
-ALLOWED_HOSTS = os.environ.get('ALLGO_ALLOWED_HOSTS', default='localhost').split(",")
-
+DEBUG = env.ALLGO_DEBUG
+ALLOWED_HOSTS = env.ALLGO_ALLOWED_HOSTS.split(",")
 TIME_ZONE = 'UTC'
 LANGUAGE_CODE = 'en-us'
 SITE_ID = 1
@@ -29,17 +40,12 @@ USE_TZ = True
 # ------------------------------------------------------------------------------
 DATABASES = {
         'default': {
-            'ENGINE': os.environ.get(
-                'ALLGO_DATABASE_ENGINE', default='django.db.backends.mysql'),
-            'NAME': os.environ.get('ALLGO_DATABASE_NAME', default='allgo'),
-            'USER': os.environ.get('ALLGO_DATABASE_USER', default='allgo'),
-            'PASSWORD': os.environ['ALLGO_DATABASE_PASSWORD'],
-            'HOST': os.environ.get('ALLGO_DATABASE_HOST', default='dev-mysql'),
-            'OPTIONS': {
-                'sql_mode': os.environ.get(
-                    'ALLGO_DATABASE_MODE',
-                    default='STRICT_ALL_TABLES'),
-            }
+            'ENGINE':   env.ALLGO_DATABASE_ENGINE,
+            'NAME':     env.ALLGO_DATABASE_NAME,
+            'USER':     env.ALLGO_DATABASE_USER,
+            'PASSWORD': env.ALLGO_DATABASE_PASSWORD,
+            'HOST':     env.ALLGO_DATABASE_HOST,
+            'OPTIONS': {'sql_mode': env.ALLGO_DATABASE_MODE }
         }
 }
 DATABASES['default']['ATOMIC_REQUESTS'] = True
@@ -53,6 +59,7 @@ WSGI_APPLICATION = 'config.wsgi.application'
 
 # APPS
 # ------------------------------------------------------------------------------
+
 DJANGO_APPS = [
     'django.contrib.admin',
     'django.contrib.auth',
@@ -73,11 +80,8 @@ LOCAL_APPS = [
     'main',
 ]
 
-if "ALLGO_ADDITIONAL_APPS" in os.environ:
-    INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS + \
-            os.environ['ALLGO_ADDITIONAL_APPS'].split(",")
-else:
-    INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
+INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS + list(
+        filter(bool, env.ALLGO_ADDITIONAL_APPS.split(",")))
 
 
 # MIGRATIONS
@@ -136,9 +140,7 @@ MIDDLEWARE = [
 
 # STATIC
 # ------------------------------------------------------------------------------
-STATIC_ROOT = os.environ.get(
-        'ALLGO_STATIC_PATH',
-        os.path.join(ROOT_DIR, 'static'))
+STATIC_ROOT = env.ALLGO_STATIC_PATH
 STATIC_URL = '/static/'
 STATICFILES_DIRS = [
     os.path.join(APPS_DIR, 'static'),
@@ -151,9 +153,7 @@ STATICFILES_FINDERS = [
 
 # MEDIA
 # ------------------------------------------------------------------------------
-MEDIA_ROOT = os.environ.get(
-        'ALLGO_MEDIA_PATH',
-        os.path.join(APPS_DIR, 'media'))
+MEDIA_ROOT = env.ALLGO_MEDIA_PATH
 MEDIA_URL = '/media/'
 
 
@@ -188,17 +188,13 @@ TEMPLATES = [
 
 # EMAIL
 # ------------------------------------------------------------------------------
-EMAIL_BACKEND = os.environ.get(
-        'ALLGO_EMAIL_BACKEND',
-        default='django.core.mail.backends.smtp.EmailBackend')
-DEFAULT_FROM_EMAIL = os.environ.get(
-        'ALLGO_EMAIL_FROM',
-        default='no-reply@allgo.inria.fr')
-EMAIL_HOST = os.environ.get('ALLGO_EMAIL_HOST', default='smtp.inria.fr')
-EMAIL_PORT = os.environ.get('ALLGO_EMAIL_PORT', default=25)
-EMAIL_HOST_USER = os.environ.get('ALLGO_EMAIL_USER', default='')
-EMAIL_HOST_PASSWORD = os.environ.get('ALLGO_EMAIL_PASSWORD', default='')
-EMAIL_USE_TLS = os.environ.get('ALLGO_EMAIL_TLS', default=False)
+EMAIL_BACKEND       = env.ALLGO_EMAIL_BACKEND
+DEFAULT_FROM_EMAIL  = env.ALLGO_EMAIL_FROM
+EMAIL_HOST          = env.ALLGO_EMAIL_HOST
+EMAIL_PORT          = env.ALLGO_EMAIL_PORT
+EMAIL_HOST_USER     = env.ALLGO_EMAIL_USER
+EMAIL_HOST_PASSWORD = env.ALLGO_EMAIL_PASSWORD
+EMAIL_USE_TLS       = parse_bool(env.ALLGO_EMAIL_TLS)
 
 
 # ADMIN

django/docs/installation.rst

@@ -152,30 +152,5 @@ Environment variables
 
   All environment variables **must** be prefixed by `ALLGO`.
 
-+-------------------------+-----------------------------------------------+
-| VARIABLE                | DEFAULT                                       |
-+=========================+===============================================+
-| ALLGO_SECRET_KEY        | None                                          |
-+-------------------------+-----------------------------------------------+
-| ALLGO_DEBUG             | False                                         |
-+-------------------------+-----------------------------------------------+
-| ALLGO_ALLOWED_HOSTS     | 'localhost'                                   |
-+-------------------------+-----------------------------------------------+
-| ALLGO_DATABASE_ENGINE   | 'django.db.backends.mysql'                    |
-+-------------------------+-----------------------------------------------+
-| ALLGO_DATABASE_NAME     | 'allgo'                                       |
-+-------------------------+-----------------------------------------------+
-| ALLGO_DATABASE_USER     | 'allgo'                                       |
-+-------------------------+-----------------------------------------------+
-| ALLGO_DATABASE_PASSWORD | None                                          |
-+-------------------------+-----------------------------------------------+
-| ALLGO_DATABASE_MODE     | 'STRICT_ALL_TABLES'                           |
-+-------------------------+-----------------------------------------------+
-| ALLGO_ADDITIONAL_APPS   | None                                          |
-+-------------------------+-----------------------------------------------+
-| ALLGO_EMAIL_BACKEND     | 'django.core.mail.backends.smtp.EmailBackend' |
-+-------------------------+-----------------------------------------------+
-| ALLGO_STATIC_PATH       | '<app_dir>/static'                            |
-+-------------------------+-----------------------------------------------+
-| ALLGO_MEDIA_PATH        | '<app_dir>/media'                             |
-+-------------------------+-----------------------------------------------+
+.. automodule:: config.env
+

docker-compose.yml

@@ -33,15 +33,11 @@ services:
     environment:
         ENV: dev
         PYTHONUNBUFFERED: 1
-        ALLGO_ALLOWED_HOSTS: 0.0.0.0,dev-django,localhost
         DJANGO_DEBUG: 1
         DJANGO_LOG_LEVEL: "DEBUG"
         ALLGO_DEBUG: "True"
         ALLGO_EMAIL_BACKEND: "django.core.mail.backends.console.EmailBackend"
         ALLGO_SECRET_KEY: "nFgLEiedSJfYKyJA6WjkiGs8c23vokcVoM4DDLi9GsCX36TdsR"
-        ALLGO_DATABASE_PASSWORD: "allgo"
-        ALLGO_CONTROLLER_HOST: "dev-controller"
-        ALLGO_CONTROLLER_PORT: "4567"
         SIGNING_KEY_PATH: "/certs/server.key"
         SIGNING_KEY_TYPE: "RSA"
         SIGNING_KEY_ALG: "RS256"