Commit 2b0fd6f5 authored by CAMPION Sebastien's avatar CAMPION Sebastien

Merge branch 'django' into runner

parents 19794f82 50f55edd
......@@ -4,7 +4,7 @@ ADD apt-getq /usr/local/bin/
RUN sed -i 's/deb.debian.org/miroir.irisa.fr/; s/main *$/main contrib non-free/' /etc/apt/sources.list &&\
apt-getq update &&\
apt-getq dist-upgrade &&\
apt-getq install vim-tiny locales patch realpath logrotate python3 less
apt-getq install vim-tiny locales patch realpath logrotate python3 less procps
ENV LANG en_US.UTF-8
......
#!/bin/bash
set -e
CONTAINERS="dev-mysql dev-controller dev-ssh dev-rails dev-django dev-nginx dev-smtpsink dev-registry"
die()
{
echo "error: $*" >&2
exit 1
}
# generate the .env file for docker-compose
#
# DOCKERUSER is set to the uid:gid of the current user
#
# If the file already exists, it will never be replaced. The function just
# prints a warning if it differs from the would-be generated file.
generate_env_file()
{
cat >".env.tmp" <<EOF
DOCKERUSER=`id -u`:`id -g`
EOF
if [ ! -e .env ] ; then
mv .env.tmp .env
else
if cmp --quiet .env .env.tmp ; then
rm .env.tmp
else
echo "warning: config file '.env' already exists and is different from the generate config. Remove it if you want it to be overwritten."
diff -u .env .env.tmp || true
fi
fi
}
# generate the certificate & key for signing the tokens
generate_secrets()
{
mkdir -p certs
if [ ! -f cert/server.key ] & [ ! -f certs/server.crt ]; then
openssl req -subj '/CN=localhost/O=Registry Demo/C=US' -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout certs/server.key -out certs/server.crt
fi
}
# generate the factories
#
# this function is run after the rails container is initialised because the
# list of factories is read from the database
build_factories()
{
# build the factories
for name in `echo 'select concat(name, ":", version) as img from docker_os order by img;' | docker exec -i dev-mysql mysql -uroot allgo | grep -v '^img$'`
do
tag="localhost:8002/allgo/dev/factory/$name"
if docker inspect --type image -- "$name" >/dev/null 2>/dev/null
then
echo "$name: already pulled"
(set -x
docker tag -- "$name" "$tag"
docker push -- "$tag"
docker rmi -- "$name" || true
)
else
echo "$name:"
(set -x
docker pull -- "$name"
docker tag -- "$name" "$tag"
docker push -- "$tag"
docker rmi -- "$name"
)
fi
done
}
# remove container and its data
purge_container()
{
local name="$1"
(set -x ; docker-compose rm -f "$name")
# ensure $name is well formatted (to avoid disasters with rm -rf)
[[ "$name" =~ ^dev-[a-z][a-z0-9-]*$ ]] || die "bad container name: $name"
local data_dir="data/${name/dev-/}"
if [ -e "$data_dir" ] ; then
echo "warning: data dir '$data_dir' already exists"
echo -n "remove it [y/N]? "
read confirm
[ "$confirm" = "y" ] || die "aborted"
(set -x ; rm -rf -- "$data_dir")
fi
}
# init a container
init_container()
{
local name="$1"
local data_dir="data/${name/dev-/}"
(set -x
# create the directory before running the container
# so that it is owned by the current user (not by root)
mkdir -p -- "$data_dir"
# run the /dk/container_init script if present
#
# FIXME: the "sleep 1" is because of a race condition in
# docker-compose (if the command finishes too quickly, then it
# is run twice)
docker-compose run --rm "$name" sh -e -c \
'if [ -f /dk/container_init ] ; then sleep 1; /dk/container_init ; fi'
# start the container
docker-compose up -d "$name"
)
}
. prepare.sh
##################################################################
mkdir -p certs
if [ ! -f cert/server.key ] & [ ! -f certs/server.crt ]; then
openssl req -subj '/CN=localhost/O=Registry Demo/C=US' -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout certs/server.key -out certs/server.crt
if [ ! -f docker-compose.yml ] || [ ! -f bootstrap ] ; then
die "the 'bootstrap' script must be run from the root of the allgo repository"
fi
# build base image (if not present)
(set -x ; make base-debian)
if [ "$1" == "-h" ] ; then
cat <<EOF
usage: $0 [CONTAINER ...]
The bootstrap script initialises the environment and the selected containers
(or by default all the containers).
If a container is already initialised, then the script asks for confirmation
before purging its data (to bootstrap it again).
EOF
fi
# selection of the containers to be generated
if [ -n "$*" ] ; then
TODO="$*"
else
docker-compose down
TODO="dev-mysql dev-controller dev-ssh dev-rails dev-django dev-nginx"
TODO="$CONTAINERS"
fi
(set -xe
for name in $TODO
do
# ensure $name does not start with '-'
[[ ! "$name" =~ ^- ]]
./init-container "$name"
docker-compose up -d "$name"
done
docker-compose up -d
docker-compose ps
)
set -e
# build the factories
for name in `echo 'select concat(name, ":", version) as img from docker_os order by img;' | docker exec -i dev-mysql mysql -uroot allgo | grep -v '^img$'`
# purge containers (container + external volumes) that will be bootstraped
for name in $TODO
do
tag="localhost:8002/allgo/dev/factory/$name"
if docker inspect --type image -- "$name" >/dev/null 2>/dev/null
then
echo "$name: already pulled"
(set -x
docker tag -- "$name" "$tag"
docker push -- "$tag"
docker rmi -- "$name" || true
)
else
echo "$name:"
(set -x
docker pull -- "$name"
docker tag -- "$name" "$tag"
docker push -- "$tag"
docker rmi -- "$name"
)
fi
purge_container "$name"
done
generate_env_file
generate_secrets
# build base image (if not present)
(set -x ; make base-debian)
# build the requested images images
docker-compose build $TODO
# initialise the containers
for name in $TODO
do
init_container "$name"
done
# build the factories (if the rail container was initialised)
if echo "$TODO" | grep -q dev-rails ; then
build_factories
fi
# display running containers
docker-compose ps
......@@ -288,19 +288,6 @@ Session.vim
tags
### VirtualEnv template
# Virtualenv
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
[Bb]in
[Ii]nclude
[Ll]ib
[Ll]ib64
[Ll]ocal
[Ss]cripts
pyvenv.cfg
pip-selfcheck.json
allgo/media/
......
FROM allgo/base-debian
# install system packages
# Installation of gunicorn through pip because the debian package
# python3-gunicorn doesn't install any binary
# configure stretch backports
COPY setup/backports/. /
RUN apt-getq update && apt-getq install mysql-server default-libmysqlclient-dev \
RUN apt-getq update && apt-getq install \
default-mysql-client default-libmysqlclient-dev \
python3-django python3-django-allauth python3-misaka \
nginx-light zip gcc python3-dev python3-pip python3-wheel python3-mysqldb \
python-mysqldb python3-crypto supervisor python3-redis python-mysqldb \
python-mysqldb python3-crypto gunicorn3 python3-redis python-mysqldb \
python3-crypto python3-djangorestframework supervisor \
&& pip3 install gunicorn
COPY requirements.txt /tmp/
RUN cd /tmp && pip3 install -r requirements.txt && rm requirements.txt
......@@ -24,3 +20,4 @@ WORKDIR /opt/allgo
LABEL dk.migrate_always=1
ENV PYTHONUNBUFFERED 1
CMD run-allgo
HEALTHCHECK CMD healthcheck
......@@ -7,14 +7,16 @@ import subprocess
from Crypto.PublicKey import RSA
from jose import jwt
SIGNING_KEY_PATH = os.environ.get('SIGNING_KEY_PATH')
SIGNING_KEY_TYPE = os.environ.get('SIGNING_KEY_TYPE')
SIGNING_KEY_ALG = os.environ.get('SIGNING_KEY_ALG')
import config
SIGNING_KEY_PATH = config.env.ALLGO_TOKEN_SIGNING_KEY_PATH
SIGNING_KEY_TYPE = config.env.ALLGO_TOKEN_SIGNING_KEY_TYPE
SIGNING_KEY_ALG = config.env.ALLGO_TOKEN_SIGNING_KEY_ALG
SIGNING_KEY = open(SIGNING_KEY_PATH).read()
ISSUER = os.environ.get('ISSUER')
TOKEN_EXPIRATION = os.environ.get('TOKEN_EXPIRATION')
TOKEN_TYPE = os.environ.get('TOKEN_TYPE')
ISSUER = config.env.ALLGO_TOKEN_ISSUER
TOKEN_EXPIRATION = config.env.ALLGO_TOKEN_EXPIRATION
TOKEN_TYPE = config.env.ALLGO_TOKEN_TYPE
def run_command(command):
......
......@@ -21,11 +21,15 @@ from django.views.decorators.csrf import csrf_exempt
from django.views.generic import ListView, DetailView, UpdateView, FormView
from rest_framework.authtoken.models import Token as DjangoToken
from .forms import UserForm, HomeSignupForm, JobForm, SSHForm, TokenForm
from .forms import JobForm, SSHForm, TokenForm
from .models import Runner
from .models import User
from .models import Webapp, Job, AllgoUser, WebappVersion
from .tokens import Token
from .models import Webapp, Job, User, AllgoUser, WebappVersion
from .forms import UserForm, HomeSignupForm
import config.env
CONTROLLER_ADDR = config.env.ALLGO_CONTROLLER_HOST, int(config.env.ALLGO_CONTROLLER_PORT)
log = logging.getLogger('allgo')
......@@ -72,7 +76,7 @@ def check_token_and_jobid(func): # Check token decorator
@check_token_and_jobid
@csrf_exempt
def runner_dw(request, jobid, filename):
datastore = getattr(settings, "ALLGO_DATASTORE")
datastore = config.env.ALLGO_DATASTORE
filepath = os.path.join(datastore, jobid, filename)
assert ".." not in filepath, "filepath unsecure"
return FileResponse(open(filepath, 'rb'))
......@@ -81,7 +85,7 @@ def runner_dw(request, jobid, filename):
@check_token_and_jobid
@csrf_exempt
def runner_up(request, jobid, digest, nbofchunk, chunkid):
datastore = getattr(settings, "ALLGO_DATASTORE")
datastore = config.env.ALLGO_DATASTORE
outputdir = os.path.join(datastore, jobid, ".%s" % digest)
if not os.path.exists(outputdir):
os.mkdir(outputdir)
......@@ -126,7 +130,7 @@ def concatenate_chunks_in_onefiletar(odir, tarfilepath):
@check_token_and_jobid
@csrf_exempt
def runner_log(request, jobid):
redis_host = getattr(settings, "ALLGO_DJANGO_REDIS_HOST")
redis_host = config.env.ALLGO_REDIS_HOST
r = redis.StrictRedis(host=redis_host, port=6379, db=0)
username, runner = get_token_cred(request)
job = Job.objects.get(runner=runner, id=jobid)
......@@ -169,15 +173,15 @@ def runner_jobs(runner):
job = Job.objects.filter(state=0, runner=runner.id).first()
if job:
log.debug("Send job %s to runner %s", job.id, runner.token)
datastore = getattr(settings, "ALLGO_DATASTORE")
datastore = config.env.ALLGO_DATASTORE
jdir = os.path.join(datastore, str(job.id))
files = {f: sha1file(os.path.join(jdir, f)) for f in os.listdir(jdir) if os.path.isfile(os.path.join(jdir, f))}
job.state = 1
job.save()
return json.dumps({"jobid": job.id,
"registry": getattr(settings, "ALLGO_DJANGO_REGISTRY"),
"registry": config.env.ALLGO_DJANGO_REGISTRY,
"image": job.webapp.name.lower(),
"chunksize": getattr(settings, "ALLGO_DJANGO_MAXUPLOADSIZE"),
"chunksize": config.env.ALLGO_DJANGO_MAXUPLOADSIZE,
"files": files})
else:
return ""
......@@ -195,7 +199,7 @@ def runner_cmd(request):
def get_allowed_actions(user, scope, actions, request):
resource_type, resource_name, resource_actions = scope.split(":")
if isinstance(user, Runner):
if user.account.is_superuser and get_client_ip(request) in getattr(settings, "ALLGO_ALLOWED_IP_ADMIN").split(','):
if user.account.is_superuser and get_client_ip(request) in config.env.ALLGO_ALLOWED_IP_ADMIN.split(','):
return ['*']
elif resource_type == "repository" and resource_name in [w.name for w in user.webapps]:
return ['pull']
......@@ -216,10 +220,7 @@ def notify_controler():
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5)
controller_host = getattr(settings, "ALLGO_CONTROLLER_HOST")
controller_port = getattr(settings, "ALLGO_CONTROLLER_PORT")
sock.connect((controller_host), int(controller_port))
sock.connect(CONTROLLER_ADDR)
sock.settimeout(None)
sock.makefile('rb', 0)
sock.shutdown(socket.SHUT_RDWR)
......@@ -228,7 +229,6 @@ def notify_controler():
except Exception as e:
log.error("Controller notification failed !!! %s", str(e))
def index(request):
"""
Do nothing specific just return a specific template
......@@ -248,7 +248,7 @@ def index(request):
@csrf_exempt
def registryhook(request):
redis_host = getattr(settings, "ALLGO_DJANGO_REDIS_HOST")
redis_host = config.env.ALLGO_REDIS_HOST
r = redis.StrictRedis(host=redis_host, port=6379, db=0)
body_unicode = request.body.decode('utf-8')
......@@ -269,7 +269,7 @@ def jupyter(request):
encoded_token = token.encode_token()
user = User.objects.get(username=user)
next = "/user/%s/git-pull?" % user + urlencode({'repo': request.GET.get("repo")})
jupyter_url = getattr(settings, "ALLGO_JUPYTER_URL")
jupyter_url = config.env.ALLGO_JUPYTER_URL
return redirect(jupyter_url + "?" + urlencode({'bearer': encoded_token, "next": next}))
......
import os
from . import env_loader
with env_loader.EnvironmentVarLoader(__name__, "ALLGO_",
{"{ENV}": os.getenv("ENV", "ENV_IS_UNSET")}) as env_var:
#
# core django config
#
env_var("ALLGO_SECRET_KEY_PATH", protected=True,
default="/vol/cache/allgo/secret_key",
help="""path where the django secret key is stored
This key is generated automatically at startup and rotated every
`ALLGO_SECRET_KEY_DAYS` days
""")
env_var("ALLGO_SECRET_KEY_DAYS",
default="30",
help="""lifetime of the django secret key in days
Note: the regeneration of the key happens only at django startup
time (i.e. django needs to be restarted to regenerate the key)
""")
env_var("ALLGO_DEBUG",
default="False",
help="enable debugging")
env_var("ALLGO_ALLOWED_HOSTS",
default="localhost,127.0.0.1,dev-django",
help="""list of hostnames that this server can serve over HTTP
In production, this value must be set to the fully qualified domain
name where this allgo instance is reachable.
""")
env_var("ALLGO_ADDITIONAL_APPS", default="",
help="comma-separated list of additional django apps to be enabled")
env_var("ALLGO_STATIC_PATH", protected=True,
default="/var/www/html/static",
help="path when the static files are stored for deployment")
env_var("ALLGO_MEDIA_PATH", protected=True,
default="/vol/rw/media",
help="path where the user-uploaded files are stored")
env_var("ALLGO_DATASTORE", protected=True,
default="/vol/rw/datastore",
help="path where the jobs files are stored")
env_var("ALLGO_ALLOWED_IP_ADMIN", protected=True,
default="127.0.0.1,0.0.0.0",
help="Admin token's can be used only if request comes from one of this ip address (comma separated list)")
#
# runner
#
env_var("ALLGO_DJANGO_REGISTRY", protected=False,
default="localhost:5000",
help="registry address sent to the runner in order to pull docker images")
env_var("ALLGO_DJANGO_MAXUPLOADSIZE", protected=False,
default="16384",
help="max upload size of chunk sended by runner as results")
env_var("ALLGO_DJANGO_MAXUPLOADSIZE", protected=False,
default="16384",
help="max upload size of chunk sended by runner as results")
#
# jupyter
#
env_var("ALLGO_JUPYTER_URL", protected=False,
default="http://0.0.0.0:8000/hub/login",
help="Url user to redirect allgo user to jupyter notebook")
env_var("ALLGO_REDIS_HOST", protected=True,
default="dev-redis",
help="redis host")
#
# email
#
env_var("ALLGO_EMAIL_BACKEND",
default="django.core.mail.backends.smtp.EmailBackend",
help="django backend for sending emails")
env_var("ALLGO_EMAIL_FROM", default="no-reply@allgo.inria.fr",
help="sender e-mail address for outgoing mails")
env_var("ALLGO_EMAIL_HOST", default="smtp.inria.fr", help="host name of the SMTP relay")
env_var("ALLGO_EMAIL_PORT", default="25", help="tcp port of the SMTP relay")
env_var("ALLGO_EMAIL_USER", default="", help="user name for the SMTP relay")
env_var("ALLGO_EMAIL_PASSWORD", default="", help="password for the SMTP relay")
env_var("ALLGO_EMAIL_TLS", default="False", help="use SMTP over TLS")
#
# database
#
env_var("ALLGO_DATABASE_ENGINE", protected=True,
default="django.db.backends.mysql",
help="django database engine")
env_var("ALLGO_DATABASE_NAME", protected=True,
default="allgo",
help="database name")
env_var("ALLGO_DATABASE_USER", protected=True,
default="allgo",
help="database user name")
env_var("ALLGO_DATABASE_PASSWORD",
default="allgo",
help="database password")
env_var("ALLGO_DATABASE_HOST", protected=True,
default="{ENV}-mysql",
help="database host name")
env_var("ALLGO_DATABASE_MODE", protected=True,
default="STRICT_ALL_TABLES",
help="""
database sql mode
see: https://mariadb.com/kb/en/library/sql-mode/
""")
#
# allgo-specific variables
#
# note: this variable is not used inside django. It is listed just for
# documentation purpose
env_var("ALLGO_HTTP_SERVER", protected=True,
default="gunicorn",
help="""selection of the HTTP server running allgo
Possible values are ``gunicorn`` and ``django``.
* ``gunicorn`` runs the gunicorn server, with the logs sent into
`/vol/log/django/`.
* ``django`` runs django's native server (`django-admin
runserver`), with logs sent to stdout/stderr. It should never be
used in production.
""")
env_var("ALLGO_CONTROLLER_HOST", protected=True,
default="{ENV}-controller",
help="Hostname of the allgo controller")
env_var("ALLGO_CONTROLLER_PORT", protected=True,
default="4567",
help="TCP port of the allgo controller (for the notifications)")
#
# allgo authentication tokens
#
# TODO: decide a default location in the container
env_var("ALLGO_TOKEN_SIGNING_KEY_PATH",
default="/certs/server.key",
help="path of the secret key (PEM file) for signing authentication tokens")
env_var("ALLGO_TOKEN_SIGNING_KEY_TYPE", protected=True,
default="RSA",
help="""type of the secret key for signing authentication tokens
For the moment, only 'RSA' is supported.
""")
env_var("ALLGO_TOKEN_SIGNING_KEY_ALG",
default="RS256",
help='RFC 7515 "alg" parameter (signature algorithm)')
env_var("ALLGO_TOKEN_ISSUER",
default="allgo_oauth",
help='RFC 7519 "iss" parameter (identifies the principal that issuset the token)')
env_var("ALLGO_TOKEN_EXPIRATION",
default="3600",
help='RFC 7519 "exp" parameter (lifetime of authentication tokens in seconds')
env_var("ALLGO_TOKEN_TYPE", protected=True,
default="JWT",
help='RFC 7519 "typ" parameter (token type)')
import json
import logging
import os
import re
import sys
import textwrap
log = logging.getLogger('allgo')
class EnvironmentVarLoader:
class UNSET:
pass
def __init__(self, module_name, prefix, replace={}):
"""new environment var loader
`module_name` is the name of the module where the environment
variables will be stored
`prefix` is the common prefix of all env variables
`replace` is a mapping of strings that must be replaced in the
value of loaded variables
"""
self.module = sys.modules[module_name]
self.prefix = prefix
self.replace = replace
# list of env vars for doc generation
# [(name, protected, default, help), ...]
self.lst = []
# set of unseen variables to issue a warning if the user sets an
# unknown env variable)
self.unseen = {v for v in os.environ if v.startswith(prefix)}
def __call__(self, name, default=UNSET, *, protected=False, help):
"""load the env var `name`
`default` is the default value for this variable if not present in
the environment
`protected` boolean
- False: this variable is tunable by the user
- True: this is a "protected" variable. It is not
recommended to change its value because it may break
something.
Notes:
- occurences of "{ENV}" are expanded with os.environ["ENV"]
- the result as a attribute of self.module
- an unset value
"""
assert name.startswith(self.prefix) and re.match(r"[A-Z0-9_]+\Z", name)
self.unseen.discard(name)
# store the info for the doc generation
self.lst.append((name, protected, default, help))
# read the environment variable
value = os.getenv(name, default)
if value is self.UNSET:
raise RuntimeError("environment variable %r must be set" % name)
# apply the substitutions
for orig, repl in self.replace.items():
value = value.replace(orig, repl)
# store the value as a module attribute
setattr(self.module, name, value)
def __enter__(self):
return self
def __exit__(self, *exc_info):
"""finish the loader task
- log the current config
- warn about unknown variables
- update the docstring (for the generation of the sphinx doc)
"""
# log the current configuration
for name, *_ in self.lst:
log.info("%s=%r", name, getattr(self.module, name))