run job containers as an ordinary user

The "UID:GID" is configurable in the JOB_USER environment.

This config is the same for all the jobs. In production this has
to be set to the squashed uid/gid configured in the NFS exports
so that we can read/write job files.

controller/Dockerfile

@@ -22,6 +22,7 @@ ENV	ENV=""		\
 	SANDBOX_PATH="/data/{ENV}/ssh/cache/sandbox"		\
 	TOOLBOX_PATH="/data/{ENV}/toolbox/cache"		\
 	SANDBOX_NETWORK="allgo_sandboxes"			\
+	JOB_USER="65534:65534"					\
 	DOCKER_HOST="unix:///run/docker.sock"			\
 	SWARM_HOST="unix:///run/docker.sock"			\
 	MYSQL_HOST="{ENV}-mysql"				\

controller/controller.py

@@ -1070,7 +1070,6 @@ class JobManager(Manager):
                 image = tmp_img = info.client.commit(ctrl.gen_sandbox_name(webapp), repo, info.version)["Id"]
             
             # TODO use another workdir
-            # TODO use another uid
 
             ctrl.check_host_path("isdir", job_path)
             hc = ctrl.sandbox.create_host_config(
@@ -1086,6 +1085,7 @@ class JobManager(Manager):
 #                        cpu_shares = info.cpu,
                         mem_limit = info.mem,
                     )
+            log.info("host config: %r", hc)
             # NOTE: cpu_shares has a different meaining in docker swarm and docker engine
             #  - swarm:  nb of cpus
             #  - engine: 1/1024 share of the total cpu resouces of the machine
@@ -1095,6 +1095,13 @@ class JobManager(Manager):
                 hc["CpuShares"] = info.cpu
             log.debug("host_config %r", hc)
             info.ctr_id = info.client.create_container(image, name=info.ctr_name,
+                    # run as an arbitrary user (for security reason)
+                    # - this is configurable with JOB_USER
+                    # - NOTE: the process may still become root through a
+                    #   setuid file, therefore it is recommended to mount
+                    #   /var/lib/docker with the 'nosuid' flag on nodes that
+                    #   run jobs
+                    user = ctrl.job_user,
                     working_dir = "/tmp",
                     # NOTE: the command line is a little complex, but this is
                     #   to ensure that (TODO write tests for this):
@@ -1704,7 +1711,7 @@ class ImageManager:
 class DockerController:
     def __init__(self, sandbox_host, swarm_host, mysql_host,
             registry, env, datastore_path, sandbox_path,
-            toolbox_path, sandbox_network, redis_host,
+            toolbox_path, sandbox_network, redis_host, job_user,
             config_file="/vol/ro/config.yml",
             ):
 
@@ -1760,6 +1767,7 @@ class DockerController:
         self.sandbox_path   = sandbox_path
         self.toolbox_path   = toolbox_path
         self.sandbox_network= sandbox_network
+        self.job_user       = job_user
 
         self._task               = None
         self._shutdown_requested = None

controller/docker-controller

@@ -123,6 +123,11 @@ def main():
         sandbox_network = val
         log.info("sandbox network       %s", sandbox_network)
 
+    with get_envvar("JOB_USER") as val:
+        re.match(r"\d+:\d+\Z", val).groups()
+        job_user = val
+        log.info("run jobs as user      %s", job_user)
+
     docker_host = os.environ.get("DOCKER_HOST")
     swarm_host  = os.environ.get("SWARM_HOST")
     log.info("docker host           %s", docker_host)
@@ -147,7 +152,8 @@ def main():
 
         return controller.DockerController(docker_host, swarm_host, mysql_host,
                 registry, env, datastore_path, sandbox_path,
-                toolbox_path, sandbox_network, redis_host).run()
+                toolbox_path, sandbox_network, redis_host,
+                job_user).run()
     except config_reader.ConfigError:
         log.critical("bad config")
         sys.exit(1)

docker-compose.yml

@@ -84,6 +84,7 @@ services:
         ENV:        "dev"
         REGISTRY:   "localhost:5000"
         DEBUG:      "1"
+        JOB_USER:   "$DOCKERUSER"
     networks: [dev]