Commit d054b778 authored by BAIRE Anthony's avatar BAIRE Anthony
Browse files

Merge remote-tracking branches 'origin/eof-events', 'origin/image-entrypoint',...

Merge remote-tracking branches 'origin/eof-events', 'origin/image-entrypoint', 'origin/allgo-files-var', 'origin/prevent-option-injections' and 'origin/db-migration' into qualif
Pipeline #139297 failed with stages
in 2 seconds
......@@ -700,9 +700,28 @@ EOF
test -f {entrypoint} || cat > {entrypoint} <<EOF
#!/bin/sh
echo
echo "This is app '{name}' called with parameters '\$@'"
echo "This is app '{name}' run with:"
echo " - parameters:" "\$@"
echo " - input files:" "\$ALLGO_FILES"
echo
echo "The workdir contains:"
echo
echo "We can iterate on input files:"
IFS="
"
for file in \$ALLGO_FILES
do
echo "processing:" "\$file"
done
echo
echo "and create output files (hello.txt)"
echo "Hello World" > hello.txt
echo
echo "The workdir now contains:"
ls -l
EOF
chmod 0755 -- {entrypoint}
......@@ -1113,15 +1132,25 @@ class JobManager(Manager):
image = "%s:%s" % (repo, tag)
image_desc = ", image=" + tag
# list of input files
allgo_files = [f["name"] for f in json.loads(job.files)]
log.info("start job %d (%s:%s%s)", info.job_id,
webapp.docker_name, info.version, image_desc)
job_path = ctrl.gen_job_path(job)
log.debug("job.path: %r", job_path)
entrypoint = [webapp.entrypoint]
if info.ver_id is None:
assert info.version == "sandbox"
image = tmp_img = info.client.commit(ctrl.gen_sandbox_name(webapp))["Id"]
else:
# use the ENTRYPOINT defined in the image (if any)
image_entrypoint = info.client.inspect_image(image)["Config"]["Entrypoint"]
if image_entrypoint and image_entrypoint != [""]:
entrypoint = image_entrypoint
# ensure that the job dir exists before actually running the job
# (because the docker daemon creates them on-the-fly which makes
......@@ -1222,10 +1251,13 @@ class JobManager(Manager):
exit $code
""".format(job_id=job.id),
"job%d" % job.id, webapp.entrypoint] + shlex.split(job.param),
"job%d" % job.id, *entrypoint] + shlex.split(job.param),
labels = {"allgo.tmp_img": tmp_img or ""},
environment=["constraint:node==" + info.node_id],
environment=[
"constraint:node==" + info.node_id,
"ALLGO_FILES=" + '\n'.join(allgo_files),
],
host_config = hc)["Id"]
info.client.start(info.ctr_id)
......
......@@ -126,6 +126,7 @@ class Job(Base):
access_token = Column(String)
container_id = Column(String(64))
queue_id = Column(Integer, ForeignKey('dj_job_queues.id'))
files = Column(Text)
webapp = relationship("Webapp")
......
......@@ -19,6 +19,7 @@ RUN apt-getq update && apt-getq install \
python3-django \
python3-django-allauth \
python3-django-extensions \
python3-django-jsonfield \
python3-django-taggit \
python3-ipy \
python3-iso8601 \
......
......@@ -80,7 +80,7 @@ def jobs(request):
# FIXME: possible infoleak: because we have ATOMIC_REQUESTS=True the
# current id can be reused for another job in case anything fails before
# the end of the request
upload_data(request.FILES.values(), job)
job.files = upload_data(request.FILES.values(), job)
# start the job
job.state = Job.WAITING
......
......@@ -103,7 +103,7 @@ class JobForm(forms.ModelForm):
class Meta:
model = Job
fields = ('param', 'queue_id', 'files', 'webapp_parameters')
fields = ('param', 'queue_id', 'webapp_parameters')
class RunnerForm(forms.ModelForm):
......
......@@ -85,31 +85,39 @@ def upload_data(uploaded_files, job):
>>> upload_data(self.request.FILES.getlist('files'), job)
Returns:
Nothing
An ordered list suitable for the Job.files field
[{"name": "...", "size": "...}, ...]
"""
job_dir = job.data_dir
os.makedirs(job_dir)
result = []
for file_data in uploaded_files:
filename = file_data.name
# sanitise the filename to prevent directory escape
# sanitise the filename to prevent directory escape and options injection
#
# The filename is provided by the user submitting the job, it cannot be
# tructed. Dangerous characters are replaced with "_" so as to
# guarantee that we do not write anything outside the job dir.
# trusted. Dangerous characters are replaced with "_" so as to
# guarantee that the user won't:
# - read/write anything outside the job dir
# - inject options (starting with '-') in a command
#
# This is a security feature, do not remove it.
#
if filename in (".", ".."):
filename = filename.replace(".", "_")
filename = filename.replace("/", "_")
if filename.startswith("-"):
filename = "_" + filename[1:]
filepath = os.path.join(job_dir, filename)
with open(filepath, 'wb+') as destination:
with open(filepath, 'wb') as destination:
for chunk in file_data.chunks():
destination.write(chunk)
result.append({"name": filename, "size": destination.tell()})
return result
def lookup_job_file(job_id, filename):
"""Look up a job data file and return its real path
......
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2020-04-23 10:09
from __future__ import unicode_literals
from django.conf import settings
import django.core.validators
from django.db import migrations, models
import django.db.models.deletion
import jsonfield.fields
import main.validators
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('main', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='Token',
fields=[
('id', models.CharField(max_length=12, primary_key=True, serialize=False, validators=[django.core.validators.RegexValidator('\\A[a-zA-Z0-9]{12}\\Z')])),
('secret', models.CharField(max_length=128)),
('name', models.CharField(max_length=128)),
('created_at', models.DateTimeField(auto_now_add=True)),
('expires_at', models.DateField(blank=True, null=True)),
('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
('webapp', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='main.Webapp')),
],
options={
'verbose_name': 'allgo token',
'db_table': 'dj_tokens',
'verbose_name_plural': 'allgo tokens',
},
),
migrations.AddField(
model_name='job',
name='files',
field=jsonfield.fields.JSONField(default=dict),
),
migrations.AddField(
model_name='webappversion',
name='deleted_at',
field=models.DateTimeField(blank=True, null=True),
),
migrations.AlterField(
model_name='webappparameter',
name='value',
field=models.CharField(blank=True, max_length=255, validators=[main.validators.job_param_validator]),
),
migrations.AlterField(
model_name='webappversion',
name='state',
field=models.IntegerField(choices=[(0, 'SANDBOX'), (1, 'COMMITTED'), (2, 'READY'), (3, 'ERROR'), (4, 'DELETED'), (5, 'USER'), (6, 'IMPORT')]),
),
]
......@@ -10,6 +10,7 @@ from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils.crypto import get_random_string
from jsonfield import JSONField
from taggit.managers import TaggableManager
from allauth.account.models import EmailAddress
......@@ -576,6 +577,12 @@ class Job(TimeStampModel):
container_id = models.CharField(max_length=64, blank=True, null=True,
validators=[docker_container_id_validator])
# Details about the input files (metadata)
#
# schema: [{"name": "<FILENAME>", "size": <SIZE>}, ...]
# Note: the list is ordered
files = JSONField()
# Relationships
queue = models.ForeignKey(JobQueue, related_name="job_queue")
webapp = models.ForeignKey(Webapp, related_name="job_webapp")
......
......@@ -718,16 +718,20 @@ class WebappVersionUpdate(UserAccessMixin, UpdateView):
if version.state in (WebappVersion.READY, WebappVersion.DELETED):
# delete the version that is currently READY
other_versions.filter(state=WebappVersion.READY
).update(state=WebappVersion.DELETED, deleted_at=django.utils.timezone.now())
deleted_versions = other_versions.filter(state=WebappVersion.READY)
deleted_ids = [v.id for v in deleted_versions]
deleted_versions.update(state=WebappVersion.DELETED, deleted_at=django.utils.timezone.now())
if version.state == WebappVersion.DELETED:
# mark the current version as READY
version.state = WebappVersion.READY
version.deleted_at = None
messages.success(self.request, "Version %s (#%d) restored"
% (version.number, version.id))
for i in deleted_ids:
messages.success(self.request, "Version %s (#%d) deleted" % (version.number, i))
messages.success(self.request, "Version %s (#%d) restored" % (
version.number, version.id))
else:
messages.success(self.request, "Version %s (#%d) updated"
% (version.number, version.id))
......@@ -1386,7 +1390,7 @@ class JobCreate(AllAccessMixin, SuccessMessageMixin, CreateView):
obj.save()
# Upload files if there are any
upload_data(self.request.FILES.getlist('files'), obj)
obj.files = upload_data(self.request.FILES.getlist('files'), obj)
# start the job
obj.state = Job.WAITING
......
......@@ -259,9 +259,21 @@
image and push it to deploy it on Allgo.
<p class="mt-3">You will need to generate a secret token
authenticate with the registry.</p>
authenticate with the registry.
<a href="{% url "main:webapp_token_create" webapp.docker_name %}" class="btn btn-primary">Manage deploy tokens</a>
<br><a href="{% url "main:webapp_token_create" webapp.docker_name %}" class="btn btn-primary">Manage deploy tokens</a>
</p>
<p>
<b>Notes</b>
<ul>
<li>the image must be tagged within the docker repository: <b>{{ repository }}</b></li>
<li>the image should define a docker <b>entrypoint</b> (if not, Allgo will use the
default entrypoint defined in your app properties)</li>
<li>you may provide a textual description in the <b>allgo.description</b> label</li>
</ul>
</p>
<pre class="language-bash mt-5"><code class="language-*"
># Build your image and tag it as '{{repository}}:VERSION'
......
......@@ -114,8 +114,9 @@ THIRD_PARTY_APPS = [
'allauth.account',
'allauth.socialaccount',
'allauth.socialaccount.providers.gitlab',
'taggit',
'django_extensions',
'jsonfield',
'taggit',
]
LOCAL_APPS = [
'main',
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment