Commit af131c2a authored by CAMPION Sebastien's avatar CAMPION Sebastien

Merge branch 'stretch' into runner

parents 3879d53d eef4c015
......@@ -10,5 +10,4 @@ docker/cover.copy
__pycache__
.coverage
/controller/htmlcov
/toolbox/openssh-*.deb
/.env
PREFIX = allgo/
USER = $(shell id -u)
GROUP = $(shell id -g)
include docker.mk
.stamp.build: .stamp.builder
#
# build
#
docker run --rm -t -i -v '$(PWD):/src' -e USER='$(USER)' -e GROUP='$(GROUP)' allgo/builder sh /src/builder/build.sh
touch '$@'
build: FORCE
rm -f '.stamp.$@'
$(MAKE) '.stamp.$@'
FROM debian:jessie
FROM debian:stretch
ADD apt-getq /usr/local/bin/
RUN sed -i 's/deb.debian.org/miroir.irisa.fr/; s/main *$/main contrib non-free/' /etc/apt/sources.list &&\
......@@ -8,5 +8,5 @@ RUN sed -i 's/deb.debian.org/miroir.irisa.fr/; s/main *$/main contrib non-free/'
ENV LANG en_US.UTF-8
COPY . /tmp/context
RUN ["sh", "/tmp/context/setup.sh"]
COPY files/. /
RUN ["sh", "/context/setup.sh"]
#!/bin/sh
set -x -e
cat >/etc/locale.gen <<EOF
en_US.UTF-8 UTF-8
fr_FR.UTF-8 UTF-8
EOF
locale-gen
cd /
apply-patches /context/patches/*.diff
rm -rf /context
#!/usr/bin/python3
#
# poll a socket until it is ready to accept connection
import argparse
import re
import socket
import sys
import time
def die(msg, *k):
sys.stderr.write("error: %s\n" % (msg % k))
sys.exit(1)
def address(val):
if val.startswith("/"):
# UNIX socket
return val
else:
# TCP socket
host, port = val.split(":")
return host, int(port)
parser = argparse.ArgumentParser()
parser.add_argument("-p", "--period", type=float, default=1.0,
help="delay between each connection attempt (default=1s)")
parser.add_argument("-t", "--timeout", type=int, default=60,
help="global timeout (default=60s)")
parser.add_argument("target", metavar="HOST:PORT|/SOCKET/PATH", type=address,
help="address of the target socket")
args=parser.parse_args()
if args.period < 0:
die("period must not be negative")
if args.timeout <= 0:
die("timeout must be positive")
limit = time.time() + args.timeout
def connect(target, timeout):
family = socket.AF_UNIX if isinstance(target, str) else socket.AF_INET
sock = socket.socket(family)
sock.settimeout(timeout)
sock.connect(target)
return sock
while True:
t0 = time.time()
try:
connect(args.target, args.period)
sys.exit(0)
except socket.error as e:
if t0 > limit or not args.period:
die("unable to connect to %s (%s)", args.target, e)
delay = t0 + args.period - time.time()
if delay > 0:
time.sleep(delay)
#!/bin/sh
set -x -e
cat >/etc/locale.gen <<EOF
en_US.UTF-8 UTF-8
fr_FR.UTF-8 UTF-8
EOF
locale-gen
cd /tmp/context
cp -r dk /dk
mkdir /usr/lib/python3/dist-packages/dk/
cp dk_migration.py /usr/lib/python3/dist-packages/dk/migration.py
install -m 0755 apply-patches /usr/local/bin
install -m 0755 diversions /usr/local/sbin
cd /
apply-patches /tmp/context/patches/*.diff
rm -rf /tmp/context
......@@ -13,15 +13,12 @@ fi
# build base image (if not present)
(set -x ; make base-debian)
# build the binaries (for the 'toolbox' and 'ssh' images)
(set -x ; make .stamp.build)
if [ -n "$*" ] ; then
TODO="$*"
else
docker-compose down
TODO="dev-mysql dev-smtpsink dev-ssh dev-rails dev-django dev-nginx"
TODO="dev-mysql dev-controller dev-ssh dev-rails dev-nginx"
fi
(set -xe
......@@ -29,10 +26,8 @@ fi
do
# ensure $name does not start with '-'
[[ ! "$name" =~ ^- ]]
# FIXME: race condition in docker-compose (if the command
# finishes too quickly, then it is run twice)
docker-compose run --rm "$name" sh -c "sleep 1 ; /dk/container_init"
./init-container "$name"
docker-compose up -d "$name"
done
docker-compose up -d
......@@ -43,10 +38,11 @@ fi
for name in `echo 'select concat(name, ":", version) as img from docker_os order by img;' | docker exec -i dev-mysql mysql -uroot allgo | grep -v '^img$'`
do
tag="localhost:8002/allgo/dev/factory/$name"
if docker inspect --type image -- "$tag" >/dev/null 2>/dev/null
if docker inspect --type image -- "$name" >/dev/null 2>/dev/null
then
echo "$name: already pulled"
(set -x
docker tag -- "$name" "$tag"
docker push -- "$tag"
docker rmi -- "$name" || true
)
......
FROM debian:jessie
RUN sed -i 's/deb.debian.org/miroir.irisa.fr/' /etc/apt/sources.list &&\
sed 's/^deb[^s]/deb-src /' /etc/apt/sources.list > /etc/apt/sources.list.d/src.list &&\
apt-get update -y -qq
RUN apt-get build-dep -y -qq --no-install-recommends openssh &&\
apt-get install -y -qq --no-install-recommends libmysqlclient-dev
RUN mkdir /pkg && cd /pkg && apt-get source -d openssh
#!/bin/sh
set -ex
cat <<EOF
###########################################
## build allgo-authorized-keys-command ##
###########################################
EOF
cd /src/ssh
make
cat <<EOF
###########################################
## build openssh ##
###########################################
EOF
cd /tmp
dpkg-source -x /pkg/openssh_*.dsc
cd openssh-*
patch -p1 < /src/toolbox/ssh_override_paths.diff
dpkg-buildpackage -b
for pkg in openssh-client openssh-server
do
install -m 0644 -o "$USER" -g "$GROUP" ../"$pkg"_*.deb /src/toolbox/"$pkg".deb
done
FROM allgo/base-debian
RUN echo deb http://miroir.irisa.fr/debian/ jessie-backports main >> /etc/apt/sources.list && apt-getq update
RUN apt-getq install python3-websocket python3-six python3-requests \
python3-mysqldb python3-sqlalchemy python3-fasteners \
python3-nose python3-coverage libjs-jquery python3-yaml \
python3-termcolor python3-iso8601
python3-termcolor python3-iso8601 python3-docker
COPY *.deb /tmp/
RUN dpkg -i /tmp/*.deb
COPY files/. /
COPY dk/. /dk/
RUN mkdir -p /opt/allgo-docker
RUN ln -s /vol/host/run/docker.sock /run/docker.sock &&\
mkdir -p /opt/allgo-docker
COPY docker-controller *.py install-examples config-example.yml /opt/allgo-docker/
RUN ln -s /vol/host/run/docker.sock /run/docker.sock
WORKDIR /opt/allgo-docker
CMD ["/opt/allgo-docker/docker-controller"]
......
......@@ -1499,6 +1499,8 @@ class DockerController:
docker.utils.parse_bytes(txt)
except Exception as e:
raise ValueError("%s: %s" % (cfg.path(key), e))
if cfg is None:
cfg = config_reader.Mapping({})
self.cpu_shares = cfg.get("cpus", None, int)
......
......@@ -12,6 +12,7 @@ import fasteners
import config_reader
import controller
# TODO: deploy to prod
log = controller.log
......
......@@ -4,5 +4,7 @@ CONFIG="/vol/ro/config.yml"
set -e
mkdir -p /vol/ro
[ -e "$CONFIG" ] || (set -x ; install -m 0640 config-example.yml "$CONFIG" )
#!/bin/sh
if [ $# -ne 1 ] ; then
echo "error: usage: $0 CONTAINER"
exit 1
fi
set -x
# FIXME: race condition in docker-compose (if the command
# finishes too quickly, then it is run twice)
docker-compose run --rm "$1" sh -c "sleep 1 ; /dk/container_init"
......@@ -4,8 +4,9 @@ FROM allgo/base-debian
RUN useradd --home /nonexistent mysql
RUN apt-getq install mariadb-server
COPY . /tmp/context
RUN sh /tmp/context/setup.sh
COPY files/. /
RUN mkdir -m 0777 /run/mysqld
USER mysql
CMD ["/usr/sbin/mysqld"]
......
......@@ -13,13 +13,15 @@ mysql_install_db >/dev/null
/usr/sbin/mysqld &
pid=$!
sleep 5
mysql -u root <<EOF
wait-socket /var/run/mysqld/mysqld.sock
mysql -uroot mysql <<EOF
GRANT ALL ON allgo.* TO allgo@'%' IDENTIFIED BY 'allgo';
GRANT ALL ON allgo_test.* TO allgo@'%';
GRANT SELECT ON allgo.* TO ssh@'%';
GRANT SELECT ON allgo.* TO ssh@'%' IDENTIFIED BY 'ssh';
UPDATE user SET Password='' WHERE Host='%' AND User='ssh';
EOF
kill "$pid"
......
#
# mariadb config for allgo
#
[mysqld]
bind-address = 0.0.0.0
# disabled because mysql_install_db fails if not run as root
# (note: the container is directly run as the mysql user)
user =
datadir = /vol/rw
log_error = /vol/log/error.log
--- etc/mysql/my.cnf
+++ etc/mysql/my.cnf
@@ -32,19 +32,19 @@
#
# * Basic Settings
#
-user = mysql
+#user = mysql # disabled because mysql_install_db fails if not run as root
pid-file = /var/run/mysqld/mysqld.pid
socket = /var/run/mysqld/mysqld.sock
port = 3306
basedir = /usr
-datadir = /var/lib/mysql
+datadir = /vol/rw
tmpdir = /tmp
lc-messages-dir = /usr/share/mysql
skip-external-locking
#
# Instead of skip-networking the default is now to listen only on
# localhost which is more compatible and is not less secure.
-bind-address = 127.0.0.1
+bind-address = 0.0.0.0
#
# * Fine Tuning
#
@@ -74,7 +74,7 @@
#
# Error log - should be very few entries.
#
-log_error = /var/log/mysql/error.log
+log_error = /vol/log/error.log
#
# Here you can see queries with especially long duration
#slow_query_log_file = /var/log/mysql/mysql-slow.log
#!/bin/sh
set -e -x
mkdir -m 0777 /run/mysqld
apply-patches /tmp/context/patches/*.diff
cp -r /tmp/context/dk/. /dk/
rm -rf /tmp/context
......@@ -3,13 +3,11 @@ FROM allgo/base-debian
RUN apt-getq install nginx openssl
RUN apt-getq install nginx-extras
COPY . /tmp/context
RUN sh /tmp/context/setup.sh
COPY files/. /
ENTRYPOINT ["/usr/local/sbin/entrypoint"]
RUN apply-patches /context/patches/*.diff && rm -rf /context
# ensure the configuration is valid before starting the server
CMD ["/bin/sh", "-c", "nginx -t && exec nginx -g 'daemon off;'"]
CMD ["run-nginx"]
LABEL dk.ready_match_stderr="configuration file .* test is successful"
......
......@@ -3,9 +3,9 @@
set -e
# preprocess the config
entrypoint true
update-nginx-config
# test it
# ensure the new configuration is valid before signalling nginx
nginx -t
# signal nginx
......
# hardened TLS config ('modern' profile generated at:
# https://mozilla.github.io/server-side-tls/ssl-config-generator/)
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_session_tickets off;
ssl_protocols TLSv1.2;
ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
ssl_prefer_server_ciphers on;
more_set_headers 'Strict-Transport-Security: max-age=15768000';
# additional configs taken from:
# - https://www.ssllabs.com/projects/best-practices/index.html
# - https://weakdh.org/sysadmin.html
more_set_headers 'X-Frame-Options: SAMEORIGIN';
more_set_headers 'X-Content-Type-Options: nosniff';
ssl_dhparam /vol/rw/dhparams.pem;
ssl_ecdh_curve secp384r1;
#!/bin/sh
set -e
mkdir -p /vol/log
# preprocess the nginx config
update-nginx-config
# ensure the configuration is valid before starting the server
nginx -t
# run !
exec nginx -g 'daemon off;'
......@@ -9,51 +9,24 @@ import sys
INPUT = "/vol/ro/config"
OUTPUT = "/etc/nginx/sites-available/default"
# skip entrypoint for image admin scripts
#FIXME: race condition in docker-compose -> need to match the "sh + sleep1" command too
#if len(sys.argv) > 1 and sys.argv[1].startswith("/dk/"):
if (len(sys.argv) > 1 and sys.argv[1].startswith("/dk/")) or (sys.argv[1:] == ["sh", "-c", 'sleep 1 ; /dk/container_init']):
os.execvp(sys.argv[1], sys.argv[1:])
def die(fmt, *k):
sys.stderr.write("error:%s:%d: %s\n" % (INPUT, lineno+1, (fmt % k)))
def die(lineno, fmt, *k):
sys.stderr.write("error:%s:%s: %s\n" % (INPUT, ((lineno+1) if isinstance(lineno, int) else ""), (fmt % k)))
sys.exit(1)
def security_config(fqdn):
# config taken from:
# - https://www.ssllabs.com/projects/best-practices/index.html
# - https://weakdh.org/sysadmin.html
# - https://wiki.mozilla.org/Security/Server_Side_TLS
# - https://cipherli.st/
# test with: https://www.ssllabs.com/ssltest/index.html
# TODO: append 'always' to the add_header directive (with nginx >=1.7.5)
return """
# ssl config
security_config = lambda fqdn: """
ssl on;
ssl_certificate /vol/ro/ssl/{fqdn}.crt;
ssl_certificate_key /vol/ro/ssl/{fqdn}.key;
ssl_dhparam /vol/rw/dhparams.pem;
ssl_prefer_server_ciphers on;
ssl_protocols TLSv1.2;
ssl_ecdh_curve secp384r1;
ssl_session_cache shared:SSL:1m;
ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
more_set_headers 'Strict-Transport-Security: max-age=31536000';
more_set_headers 'X-Frame-Options: SAMEORIGIN';
more_set_headers 'X-Content-Type-Options: nosniff';
# do not disclose that the server is running an outdated version of jboss
more_clear_headers X-Powered-By;
include /etc/nginx/ssl.conf;
""".format(fqdn=fqdn)
def default_server(close=True):
out_file.write("""
# default server (returns 404)
default_server = lambda **kw: """
# do not transmit nginx version number
server_tokens off;
......@@ -74,46 +47,13 @@ server {{
return 404;
}}
{close}
""".format(security=security_config("invalid"), close=("}" if close else "")))
with open(INPUT) as in_file, open(OUTPUT, "w") as out_file:
default_done = False
{block}
allgo_section = None
for lineno, line in enumerate(in_file):
if allgo_section is None:
if line.startswith("DEFAULT"):
if not re.match("DEFAULT\s*($|#)", line):
die("malformatted DEFAULT line")
if default_done:
die("duplicated DEFAULT line")
allgo_section = lineno
default_server(close=False)
default_done = True
elif line.startswith("ALLGO"):
# enter allgo section
if not default_done:
default_server()
default_done = True
allgo_section = lineno
try:
container, port, fqdn, fport = (x.strip() for x in re.match(r"ALLGO\(([^)]*)\)\s*($|#)", line).group(1).split(","))
int(port)
int(fport)
except Exception:
die("malformatted ALLGO line")
}}
""".format(security=security_config("invalid"), **kw)
sys.stderr.write("Allgo instance: backend %s:%s frontend %s:%s\n" % (container, port, fqdn, fport))
out_file.write("""
allgo_server = lambda **kw: """
server {{
# HTTP server
......@@ -155,43 +95,66 @@ server {{
proxy_set_header X-Forwarded-Proto $scheme;
}}
location /django
{{
proxy_pass http://dev-django:8080/;
proxy_redirect off;
proxy_buffering off;
{block}
}}
""".format(security=security_config(kw["fqdn"]), **kw)
# To work properly, turbolinks must know the host and scheme (https),
# thus we do not use proxy_redirect but forward the two headers instead
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Proto $scheme;
}}
""".format(container=container, port=port, fqdn=fqdn, fport=fport,
security=security_config(fqdn)))
else:
# ordinary line
out_file.write(line)
def read_block(line_reader):
first_lineno, line = next(line_reader, (None, None))
if line is None:
die(lineno, "unexpected EOF (expected: '{'")
if not line.startswith("{"):
die(lineno, "expected '{' at the beginnning of line")
else:
# inside allgo section
if lineno == allgo_section + 1:
if line.startswith("{"):
line = line[1:]
else:
out_file.write("}\n")
allgo_section = None
elif line.startswith("}"):