Une nouvelle version du portail de gestion des comptes externes sera mise en production lundi 09 août. Elle permettra d'allonger la validité d'un compte externe jusqu'à 3 ans. Pour plus de détails sur cette version consulter : https://doc-si.inria.fr/x/FCeS

Commit 821c68fd authored by BAIRE Anthony's avatar BAIRE Anthony
Browse files

add the ALLGO_FILES env var to list the input files of the job

- the submission order is preserved
- Job.files is a JSON list so that we can expand it later to store
  other details
- in ALLGO_FILES the filenames are line-delimited to allow working
  with input files containing spaces in their name (the IFS env var
  can be tuned for that purpose)

close allgo.inria.fr#11
parent 6dba797b
Pipeline #137067 failed with stages
in 1 second
......@@ -1113,6 +1113,9 @@ class JobManager(Manager):
image = "%s:%s" % (repo, tag)
image_desc = ", image=" + tag
# list of input files
allgo_files = [f["name"] for f in json.loads(job.files)]
log.info("start job %d (%s:%s%s)", info.job_id,
webapp.docker_name, info.version, image_desc)
......@@ -1225,7 +1228,10 @@ class JobManager(Manager):
"job%d" % job.id, webapp.entrypoint] + shlex.split(job.param),
labels = {"allgo.tmp_img": tmp_img or ""},
environment=["constraint:node==" + info.node_id],
environment=[
"constraint:node==" + info.node_id,
"ALLGO_FILES=" + '\n'.join(allgo_files),
],
host_config = hc)["Id"]
info.client.start(info.ctr_id)
......
......@@ -126,6 +126,7 @@ class Job(Base):
access_token = Column(String)
container_id = Column(String(64))
queue_id = Column(Integer, ForeignKey('dj_job_queues.id'))
files = Column(Text)
webapp = relationship("Webapp")
......
......@@ -19,6 +19,7 @@ RUN apt-getq update && apt-getq install \
python3-django \
python3-django-allauth \
python3-django-extensions \
python3-django-jsonfield \
python3-django-taggit \
python3-ipy \
python3-iso8601 \
......
......@@ -80,7 +80,7 @@ def jobs(request):
# FIXME: possible infoleak: because we have ATOMIC_REQUESTS=True the
# current id can be reused for another job in case anything fails before
# the end of the request
upload_data(request.FILES.values(), job)
job.files = upload_data(request.FILES.values(), job)
# start the job
job.state = Job.WAITING
......
......@@ -103,7 +103,7 @@ class JobForm(forms.ModelForm):
class Meta:
model = Job
fields = ('param', 'queue_id', 'files', 'webapp_parameters')
fields = ('param', 'queue_id', 'webapp_parameters')
class RunnerForm(forms.ModelForm):
......
......@@ -85,12 +85,14 @@ def upload_data(uploaded_files, job):
>>> upload_data(self.request.FILES.getlist('files'), job)
Returns:
Nothing
An ordered list suitable for the Job.files field
[{"name": "...", "size": "...}, ...]
"""
job_dir = job.data_dir
os.makedirs(job_dir)
result = []
for file_data in uploaded_files:
filename = file_data.name
......@@ -107,9 +109,11 @@ def upload_data(uploaded_files, job):
filename = filename.replace("/", "_")
filepath = os.path.join(job_dir, filename)
with open(filepath, 'wb+') as destination:
with open(filepath, 'wb') as destination:
for chunk in file_data.chunks():
destination.write(chunk)
result.append({"name": filename, "size": destination.tell()})
return result
def lookup_job_file(job_id, filename):
"""Look up a job data file and return its real path
......
......@@ -10,6 +10,7 @@ from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils.crypto import get_random_string
from jsonfield import JSONField
from taggit.managers import TaggableManager
from allauth.account.models import EmailAddress
......@@ -576,6 +577,12 @@ class Job(TimeStampModel):
container_id = models.CharField(max_length=64, blank=True, null=True,
validators=[docker_container_id_validator])
# Details about the input files (metadata)
#
# schema: [{"name": "<FILENAME>", "size": <SIZE>}, ...]
# Note: the list is ordered
files = JSONField()
# Relationships
queue = models.ForeignKey(JobQueue, related_name="job_queue")
webapp = models.ForeignKey(Webapp, related_name="job_webapp")
......
......@@ -1386,7 +1386,7 @@ class JobCreate(AllAccessMixin, SuccessMessageMixin, CreateView):
obj.save()
# Upload files if there are any
upload_data(self.request.FILES.getlist('files'), obj)
obj.files = upload_data(self.request.FILES.getlist('files'), obj)
# start the job
obj.state = Job.WAITING
......
......@@ -114,8 +114,9 @@ THIRD_PARTY_APPS = [
'allauth.account',
'allauth.socialaccount',
'allauth.socialaccount.providers.gitlab',
'taggit',
'django_extensions',
'jsonfield',
'taggit',
]
LOCAL_APPS = [
'main',
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment