Commit e789f843 authored by BAIRE Anthony's avatar BAIRE Anthony
Browse files

replace pipecmd with a real ssh connection

- sshd server installed in the toolbox
- ssh keys & config stored in ssh:/vol/cache and mounted as
  /.sandbox inside the sandbox
- toolbox mounted as /.toolbox inside the sandbox
- ssh agent & X11 forwarding are now working
- the toolbox commands available by default in every sandboxes
  (vim, less, nc, scp, ...)
- sandboxes now attached to a separate network (named
  'allgo_sandboxes' by default)

fix #88
parent 04219506
......@@ -21,8 +21,10 @@ ENV PORT="4567" \
ENV="" \
REGISTRY="" \
MAX_JOBS="4" \
DATASTORE_PATH="/data/{ENV}/rails/rw/datastore/" \
SANDBOX_PATH="/data/{ENV}/ssh/cache/sandbox/" \
DATASTORE_PATH="/data/{ENV}/rails/rw/datastore" \
SANDBOX_PATH="/data/{ENV}/ssh/cache/sandbox" \
TOOLBOX_PATH="/data/{ENV}/toolbox/cache" \
SANDBOX_NETWORK="allgo_sandboxes" \
DOCKER_HOST="unix:///run/docker.sock" \
SWARM_HOST="unix:///run/docker.sock" \
MYSQL_HOST="{ENV}-mysql"
......
......@@ -463,9 +463,6 @@ class SandboxManager(Manager):
if ("/" in webapp.docker_name) or (webapp.docker_name in ("", ".", "..")):
raise Error("malformatted docker_name")
pipebin = os.path.join(ctrl.sandbox_path, "bin")
pipedir = os.path.join(ctrl.sandbox_path, "srvdir", webapp.docker_name)
uid = webapp.id + 2000
if uid < 2000:
# just for safety
......@@ -478,28 +475,135 @@ class SandboxManager(Manager):
container = webapp.sandbox_name
try:
# prepare the sandbox
# (create files for pipesrv)
# (create ssh keys)
ctrl.check_host_path("isdir", ctrl.toolbox_path)
ctrl.check_host_path("isdir", ctrl.sandbox_path)
ctrl.sandbox.create_container("busybox:latest", name=container,
command = ["/bin/sh", "-c", "set -x ;rm -rf -- /.pipedir/* && chmod 0700 /.pipedir && mkfifo -m 0600 /.pipedir/srv && chown -R %d:65534 /.pipedir" % uid],
command = ["/bin/sh", "-c", """
set -ex
export PATH="$PATH:/.toolbox/bin"
# clean sandbox dir
rm -rf {sbx}
# create dirs
for dir in {sbx} {etc} {run}
do
mkdir -p ${{dir}}
chown {uid}:65534 ${{dir}}
chmod 0700 ${{dir}}
done
# xauth file
touch {run}/XAuthority
chown {uid}:65534 {run}/XAuthority
chmod 0600 {run}/XAuthority
# generate ssh keys
(for type in ecdsa ed25519 rsa
do
key={etc}/ssh_host_${{type}}_key
[ -f $key ] || ssh-keygen -N '' -f $key -t $type >&2
echo -n '{hostname}. ' | cat - ${{key}}.pub
done) > {etc}/ssh_known_hosts
# known_host file for allgo-shell
ssh-keygen -H -f {etc}/ssh_known_hosts
chmod 0644 {etc}/ssh_known_hosts
rm -f {etc}/ssh_known_hosts.old
# authentication key for allgo-shell
rm -f {etc}/identity
ssh-keygen -N '' -f {etc}/identity
chown {uid}:65534 {etc}/identity
# forced shell for the sshd config
cat > {etc}/shell <<EOF
#!/bin/sh
export PATH="\$PATH:/.toolbox/bin"
uid=\`id -u\`
shell="\`getent passwd \$uid 2>/dev/null| cut -d : -f 7\`"
if [ -z "\$shell" ] ; then
shell=/bin/sh
fi
if [ -n "\$SSH_ORIGINAL_COMMAND" ] ; then
exec "\$shell" -c "\$SSH_ORIGINAL_COMMAND"
else
exec "\$shell"
fi
EOF
chmod 755 {etc}/shell
# sshd config
cat > {etc}/sshd_config <<EOF
Port 22
Protocol 2
# turned off because it requires creating a 'sshd' user inside the sandbox
UsePrivilegeSeparation no
StrictModes no
ForceCommand /.sandbox/etc/ssh/shell
PermitRootLogin without-password
PubkeyAuthentication yes
AuthorizedKeysFile /.sandbox/etc/ssh/identity.pub .ssh/authorized_keys .ssh/authorized_keys2
ChallengeResponseAuthentication no
PasswordAuthentication no
X11Forwarding yes
X11DisplayOffset 10
PrintMotd no
PrintLastLog no
TCPKeepAlive yes
# Allow client to pass locale environment variables
AcceptEnv LANG LC_*
Subsystem sftp internal-sftp
UsePAM no
EOF
""".format(uid=uid,
hostname = "%s-sandbox-%s" % (ctrl.env, webapp.docker_name),
sbx = "/mnt/%s" % webapp.docker_name,
etc = "/mnt/%s/etc/ssh" % webapp.docker_name,
run = "/mnt/%s/run" % webapp.docker_name,
)],
host_config = ctrl.sandbox.create_host_config(
binds = {pipedir: {"bind": "/.pipedir"}}
))
binds = {
ctrl.sandbox_path: {"bind": "/mnt"},
ctrl.toolbox_path: {"bind": "/.toolbox", "mode": "ro"},
}))
ctrl.sandbox.start(container)
if ctrl.sandbox.wait(container):
log.debug("sandbox %s output:\n%s", webapp.docker_name,
ctrl.sandbox.logs(container).decode(errors="replace"))
raise Error("sandbox preparation failed")
ctrl.sandbox.remove_container(container)
# create and start the sandbox
ctrl.check_host_path("isdir", pipedir)
ctrl.check_host_path("isdir", pipebin)
command = ["/.pipebin/pipesrv", "-d", "/.pipedir"]
etc_dir = os.path.join(ctrl.sandbox_path, webapp.docker_name, "etc")
run_dir = os.path.join(ctrl.sandbox_path, webapp.docker_name, "run")
ctrl.check_host_path("isdir", etc_dir)
ctrl.check_host_path("isdir", run_dir)
if version is None and webapp.entrypoint:
# prepend instructions to initialise a dummy entrypoint
dn, bn = os.path.split(webapp.entrypoint)
# FIXME: do nothing if entrypoint already exists
command = ["/bin/sh", "-c", """
prepare = """
{mkdir}
test -f {entrypoint} || cat > {entrypoint} <<EOF
#!/bin/sh
......@@ -510,22 +614,38 @@ echo "The workdir contains:"
ls -l
EOF
chmod 0755 -- {entrypoint}
exec {cmd}
""".format( entrypoint = shlex.quote(webapp.entrypoint),
name = webapp.docker_name,
mkdir = (("mkdir -p -- %s" % shlex.quote(dn)) if dn else ""),
cmd = " ".join(map(shlex.quote, command)))]
mkdir = (("mkdir -p -- %s" % shlex.quote(dn)) if dn else ""))
else:
prepare = ""
command = ["/bin/sh", "-c", """
set -x
export PATH="$PATH:/.toolbox/bin"
{prepare}
# xauth file (needed for X11 forwarding)
touch /root/.Xauthority
chmod 600 /root/.Xauthority
exec /.toolbox/bin/sshd -D
""".format(prepare=prepare)]
ctrl.sandbox.create_container(image, name=container, hostname=container,
command = command,
host_config = ctrl.sandbox.create_host_config(
binds = {
pipedir: {"bind": "/.pipedir", "mode": "ro"},
pipebin: {"bind": "/.pipebin", "mode": "ro"},
etc_dir: {"bind": "/.sandbox/etc", "mode": "ro"},
run_dir: {"bind": "/.sandbox/run", "mode": "rw"},
ctrl.toolbox_path: {"bind": "/.toolbox", "mode": "ro"},
},
# TODO: maybe drop other caps
cap_drop = ["NET_RAW"],
restart_policy = {"Name": "unless-stopped"},
network_mode = ctrl.sandbox_network,
))
ctrl.sandbox.start(container)
......@@ -1299,7 +1419,7 @@ class ImageManager:
class DockerController:
def __init__(self, sandbox_host, swarm_host, mysql_host,
port, registry, env, datastore_path, sandbox_path,
max_jobs):
toolbox_path, max_jobs, sandbox_network):
self.sandbox = docker.Client(sandbox_host)
self.sandbox_watcher = DockerWatcher(self.sandbox)
......@@ -1328,6 +1448,8 @@ class DockerController:
self.env = env
self.datastore_path = datastore_path
self.sandbox_path = sandbox_path
self.toolbox_path = toolbox_path
self.sandbox_network= sandbox_network
self._task = None
self._shutdown_requested = None
......@@ -1349,10 +1471,8 @@ class DockerController:
return "%s-job-%d-%s" % (self.env, job.id, job.webapp.docker_name)
def gen_job_path(self, job):
assert self.datastore_path.endswith("/")
return "%s%d/%d/%s" % (
self.datastore_path, job.user_id,
job.webapp_id, job.access_token)
return os.path.join(self.datastore_path, str(job.user_id),
str(job.webapp_id), job.access_token)
def check_host_path(self, funcname, path, *, nolink=True):
......@@ -1369,7 +1489,7 @@ class DockerController:
raise Error("host path %r is not absolute" % path)
if path != os.path.normpath(path):
raise Error("host path %r is not canonical (os.path.normpath())")
raise Error("host path %r is not canonical (os.path.normpath())" % path)
ctrpath = "/vol/host" + path
log.debug("ctrpath %r", ctrpath)
......
......@@ -58,6 +58,14 @@ def main():
logging.exception("")
die("invalid environment variable %s=%r", name, value)
def parse_path(path):
if not path:
raise ValueError("path is empty")
if not os.path.isabs(path):
raise ValueError("path is not absolute")
return "/" + path.strip("/")
with get_envvar("DEBUG") as val:
debug = bool(int(val or 0))
......@@ -86,21 +94,28 @@ def main():
log.info("registry: %s", registry)
with get_envvar("DATASTORE_PATH") as val:
datastore_path = val.format(ENV=env)
re.match(r"/.*/\Z", datastore_path).groups()
datastore_path = parse_path(val.format(ENV=env))
log.info("datastore path %s", datastore_path)
with get_envvar("SANDBOX_PATH") as val:
sandbox_path = val.format(ENV=env)
re.match(r"/.*/\Z", sandbox_path).groups()
sandbox_path = parse_path(val.format(ENV=env))
log.info("sandbox path %s", sandbox_path)
with get_envvar("TOOLBOX_PATH") as val:
toolbox_path = parse_path(val.format(ENV=env))
log.info("toolbox path %s", toolbox_path)
with get_envvar("MAX_JOBS") as val:
max_jobs = int(val) if val else 4
if max_jobs <= 0:
raise ValueError("must provide a positive value")
log.info("max concurrent jobs %d", max_jobs)
with get_envvar("SANDBOX_NETWORK") as val:
re.match(r"\A[\w]+[\w. _-]*[\w]+\Z", val)
sandbox_network = val
log.info("sandbox network %s", sandbox_network)
docker_host = os.environ.get("DOCKER_HOST")
swarm_host = os.environ.get("SWARM_HOST")
log.info("docker host %s", docker_host)
......@@ -120,7 +135,8 @@ def main():
console_handler.setLevel(logging.WARNING)
return controller.DockerController(docker_host, swarm_host, mysql_host,
port, registry, env, datastore_path, sandbox_path, max_jobs).run()
port, registry, env, datastore_path, sandbox_path, toolbox_path,
max_jobs, sandbox_network).run()
finally:
lock.release()
......
......@@ -38,6 +38,10 @@ DATASTORE_PATH = "/vol/tmp/test/datastore/"
#FIXME this will fail if the ssh container was not run previously
SANDBOX_HOST_PATH = "/data/dev/ssh/cache/sandbox"
SANDBOX_NETWORK = "allgo_sandboxes"
#FIXME this will fail if the toolbox container was not run previously
TOOLBOX_HOST_PATH = "/data/dev/toolbox/cache"
S = SandboxState
V = VersionState
......@@ -167,6 +171,8 @@ class ControllerTestCase(unittest.TestCase):
env = ENV,
datastore_path = DATASTORE_HOST_PATH,
sandbox_path = SANDBOX_HOST_PATH,
sandbox_network= SANDBOX_NETWORK,
toolbox_path = TOOLBOX_HOST_PATH,
max_jobs = 4,
)
......
......@@ -68,7 +68,7 @@ services:
- "./ssh:/opt/allgo-ssh"
environment:
DB_HOST: "dev-mysql"
ENV: "dev"
networks: [dev, sandboxes]
......
......@@ -32,11 +32,22 @@ then
sudo mount -t tmpfs tmpfs-docker "$dir"
fi
DOCKER_HOST="unix://$sock"
export DOCKER_HOST="unix://$sock"
echo "export DOCKER_HOST='$DOCKER_HOST'"
sudo dockerd \
create_network()
{
sleep 2
net=allgo_sandboxes
echo "create network $net"
docker network create "$net"
exit 0
}
create_network &
exec sudo dockerd \
--graph "$graph" \
--host "$DOCKER_HOST" \
--pidfile "$pidfile" \
......
/ssh/allgo-authorized-keys-command
/shell/pipecmd/pipesrv
/shell/pipecmd/pipecli
......@@ -3,7 +3,7 @@ FROM allgo/base-debian
COPY backports/local-backports.pref /etc/apt/preferences.d/
COPY backports/local-backports.list /etc/apt/sources.list.d/
RUN apt-getq update && apt-getq install openssh-server libmysqlclient18
RUN apt-getq update && apt-getq install ssh libmysqlclient18 xauth
COPY . /tmp/context
RUN sh /tmp/context/setup.sh
......
......@@ -15,18 +15,14 @@ TARGETS=ssh/allgo-authorized-keys-command
USER=$(shell id -u):$(shell id -g)
all: image
docker run --rm -v '$(PWD):/mnt' -w /mnt -u '$(USER)' '$(DOCKER_IMAGE)' make DEBUG=1 clean pipecmd $(TARGETS)
docker run --rm -v '$(PWD):/mnt' -w /mnt -u '$(USER)' '$(DOCKER_IMAGE)' make DEBUG=1 clean $(TARGETS)
image:
docker build -t '$(DOCKER_IMAGE)' - < Dockerfile.build
pipecmd:
cd shell/pipecmd && $(MAKE)
%: %.c
$(CC) $(CFLAGS) $(LIBS) -o '$@' '$<'
clean:
cd shell/pipecmd && $(MAKE) clean
rm -f -- $(TARGETS)
......@@ -20,7 +20,7 @@ getspuid SELECT name,'*','1','','99999999999','','','99999999999','' \
LIMIT 1
#host <value of the DB_HOST environment variable>
#host <set at runtime by run-sshd>
database allgo
username ssh
......@@ -6,22 +6,14 @@ cp /etc/passwd.override /etc/passwd
cp /etc/group.override /etc/group
cp /etc/nsswitch.conf.override /etc/nsswitch.conf
mkdir -p /vol/cache/sandbox/bin
if [ -z "$DB_HOST" ]
then
echo "error: env var DB_HOST is not set (should be the mysql server hostname)" >&2
if [ -z "$ENV" ] ; then
echo "error: the environment variable ENV is not set" >&2
exit 1
fi
echo "host $DB_HOST" | cat /etc/libnss-mysql.cfg.template - > /etc/libnss-mysql.cfg
# make the ENV variable available to allgo-shell (in the sshd sesssion)
echo "ENV=$ENV" >/etc/environment
# NOTE: it is necessary to explicitely remove the file before copying it
# because there may be existing containers using the previous files (and the
# kernel will not allow overwriting the file because of the binding, but
# removing is ok)
rm -f /vol/cache/sandbox/bin/pipesrv
install -m 0755 /bin/pipesrv /vol/cache/sandbox/bin/pipesrv
echo "host $ENV-mysql" | cat /etc/libnss-mysql.cfg.template - > /etc/libnss-mysql.cfg
exec /usr/sbin/sshd -D
......@@ -32,7 +32,7 @@ install -m 0444 nss/libnss-mysql.cfg.template /etc/
install -m 0400 nss/libnss-mysql-root.cfg /etc/
install -m 0444 nss/*.override /etc/
mkdir -m 0711 /nohome
mkdir -p -m 0711 /nohome/.ssh
#
# sshd config
......@@ -42,31 +42,28 @@ mkdir -m 0711 /nohome
# - Authentication by public key only. Authorized keys are obtained from the
# webapps database (field sshkey) through the allgo-authorized-keys command
#
# - ssh agent forwarding and X11 forwarding are enabled (and effectively
# forwarded to the sandbox)
#
# - however TCP port and unix socket forwarding are disabled. If the user
# really need it needs it, then he should use netcat or a ssh proxy command
#
(cd / && apply-patches /tmp/context/ssh/*.diff)
mkdir /var/run/sshd
rm /etc/ssh/ssh_host_*_key*
install -m 0711 ssh/allgo-authorized-keys-command /sbin/
install -m 0711 ssh/allgo-authorized-keys-command /sbin/
install -m 0755 ssh/xauth /usr/local/bin/
#
# allgo shell
#
# pipesrv/pipecli is a pair of commands to run a shell inside a container
# through a set of pipes mounted in an external volume
# - pipesrv is run inside the container. It is bult as a static executable so
# that we can use it with any distribution.
# - pipecli is run by allgo-shell to connect to the pipesrv inside the target
# container
# - the external volume of the sandbox must be mounted read-only (for security), all
# pipes/fifos are created on client side
# - the pipes are created inside /vol/cache/sandbox/srvdir
# - a copy of pipesrv is put inside /vol/cache/sandbox/bin at run time (to
# make it available in the docker host filesystem so that we can mount it as
# an external volume in the sandboxes)
# allgo-shell opens a shell in the sandbox via a second ssh connection. It uses
# various files generated by the controller and provided in /vol/cache/sandbox
# - ssh host keys (for the sandbox) + known_hosts (for allgo-shell)
# - identity key (for allgo-shell) + identity.pub (for the sandbox -> used as authorized_keys)
install -m 0644 shell/motd /etc/
install -m 0755 shell/pipecmd/pipecli /bin/
install -m 0755 shell/pipecmd/pipesrv /bin/
install -m 0755 shell/allgo-shell /bin/
mkdir /allgo
......@@ -77,8 +74,6 @@ ln -s /vol/cache/sandbox /allgo/sandbox
#
# - overrides nsswitch.conf passwd and group
#
# - install pipesrv into /allgo/sandbox to make it available to the sandbox
# containers
install -m 0755 run-sshd /sbin/
......
......@@ -3,6 +3,7 @@
import argparse
import os
import pwd
import socket
import sys
SANDBOX_DIR = "/allgo/sandbox"
......@@ -11,6 +12,10 @@ def die(msg, *k):
print("\nerror: " + msg % k, file=sys.stderr)
sys.exit(255)
env=os.environ.get("ENV")
if not env:
die("environment variable ENV is not defined")
#####################################################################
# parse the command line
parser=argparse.ArgumentParser()
......@@ -43,19 +48,30 @@ if uid < 2000:
if user in ("root", "sshd", ".", "..") or not user or "/" in user:
die("invalid user %r", user)
host = "%s-sandbox-%s." % (env, user)
#####################################################################
# run the shell
pipedir = os.path.join(SANDBOX_DIR, "srvdir", user)
# ensure the sandbox is running
try:
fd = os.open(os.path.join(pipedir, "srv"), os.O_WRONLY | os.O_NONBLOCK)
except OSError:
socket.getaddrinfo(host, 22)
except socket.gaierror:
die("your sandbox is not running")
os.close(fd)
cmd = ["/bin/pipecli", "-d", pipedir]
#####################################################################
# run the shell
sshdir = os.path.join(SANDBOX_DIR, user, "etc", "ssh")
cmd = ["/usr/bin/ssh", "-AX",
("-t" if os.isatty(0) else "-T"),
"-i", os.path.join(sshdir, "identity"),
"-o", "UserKnownHostsFile="+os.path.join(sshdir, "ssh_known_hosts"),
"-o", "StrictHostKeyChecking=yes",
"-o", "CheckHostIP=no",
"-o", "XAuthLocation=/usr/local/bin/xauth",
"--", "root@"+host]
if command is not None:
cmd.extend(("-c", command))
cmd.append(command)
try:
os.execv(cmd[0], cmd)
except OSError as e:
......
FROM debian_i386:jessie
RUN apt-get update -y -qq
RUN apt-get install -y -qq --no-install-recommends libklibc-dev libc6-dev make gcc
CC = gcc
DEBUG = 1
CFLAGS = -DPROGNAME=\"$@\" -Wall $(if $(DEBUG),-g)
LDFLAGS = $(if $(DEBUG),,-Wl,--strip-all) -static
TARGETS= pipecli pipesrv
CLISRC = pipecli.c common.c
SRVSRC = pipesrv.c common.c
DEPS = common.h
DOCKER_IMAGE = pipecmd-build
USER=$(shell id -u):$(shell id -g)
all: $(TARGETS)
pipecli: $(CLISRC) $(DEPS)
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(CLISRC)
pipesrv: $(SRVSRC) $(DEPS)
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(SRVSRC)
final: image
docker run --rm -v '$(PWD):/mnt' -w /mnt -u '$(USER)' '$(DOCKER_IMAGE)' make DEBUG= clean all
ls -lh -- $(TARGETS)
image:
docker build -t '$(DOCKER_IMAGE)' .
clean:
rm -f -- $(TARGETS)
#define _GNU_SOURCE
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <poll.h>
#include <signal.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <termios.h>
#include <unistd.h>
#ifndef __GLIBC__
#include <sys/splice.h>
#endif
#include "common.h"
//TODO remove
void lsof()
{