Commit 1a6316b6 authored by CAMPION Sebastien's avatar CAMPION Sebastien

Store job output (docker output) in a redis db during 24h

parent 30954aa6
......@@ -10,7 +10,7 @@ COPY setup/backports/. /
RUN apt-getq update && apt-getq install mysql-server default-libmysqlclient-dev \
python3-django python3-django-allauth python3-misaka \
nginx-light zip gcc python3-dev python3-pip python3-wheel python3-mysqldb \
python-mysqldb python3-crypto supervisor\
python-mysqldb python3-crypto supervisor python3-redis \
&& pip3 install gunicorn
COPY requirements.txt /tmp/
......
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-04-19 15:06
from __future__ import unicode_literals
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('main', '0005_auto_20180418_0922'),
]
operations = [
migrations.RemoveField(
model_name='joblog',
name='dj_job',
),
migrations.DeleteModel(
name='JobLog',
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-04-20 12:53
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('main', '0006_auto_20180419_1506'),
]
operations = [
migrations.AlterField(
model_name='job',
name='runner',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='runner', to='main.Runner'),
),
]
......@@ -203,7 +203,7 @@ class Job(TimeStampModel):
webapp = models.ForeignKey(Webapp, related_name="webapp")
user = models.ForeignKey(User, related_name="user")
runner = models.OneToOneField(Runner, related_name="runner", blank=True, null=True)
runner = models.ForeignKey(Runner, related_name="runner", blank=True, null=True)
# Not in use because I don't have any related table yet
container_id = models.IntegerField(blank=True, null=True)
......@@ -212,15 +212,6 @@ class Job(TimeStampModel):
db_table = 'dj_jobs'
class JobLog(TimeStampModel):
msg = models.TextField(blank=True, null=True)
linenumber = models.IntegerField(default=0)
dj_job = models.ForeignKey(Job, related_name="job")
class Meta:
db_table = 'dj_job_logs'
class JobUploads(TimeStampModel):
"""
Data uploaded by the user to feed a given webapp
......
......@@ -9,7 +9,7 @@ urlpatterns = [
url(r"^runner/cmd$", views.runner_cmd, name="runner_cmd"),
url(r"^runner/dw/(\d+)/(.*)", views.runner_dw, name="runner_dw"),
url(r"^runner/up/(\d+)/[sha1|sha256|md5]*:(\b[0-9a-f]{5,40}\b)/(\d+)/(\d+)", views.runner_up, name="runner_up"),
url(r"^runner/logs/(\d+)", views.runner_log, name="runner_log"),
url(r"^runner/log/(\d+)", views.runner_log, name="runner_log"),
url(r'^registryhook', views.registryhook, name="registryhook"),
url(r'^jupyter$', views.jupyter, name="jupyter"),
url(r'^apps/$', views.WebappList.as_view(), name='webapp_list'),
......
......@@ -6,6 +6,7 @@ import os
import socket
import tarfile
import redis
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import PasswordChangeForm
......@@ -24,7 +25,7 @@ from django.views.generic import (
from .forms import UserForm, HomeSignupForm
from .models import User
from .models import Webapp, Job, AllgoUser, WebappVersion, Runner, JobLog
from .models import Webapp, Job, AllgoUser, WebappVersion, Runner
from .tokens import Token
log = logging.getLogger('allgo')
......@@ -109,19 +110,36 @@ def concatenate_chunks_in_onefiletar(odir, tarfilepath):
@csrf_exempt
def runner_log(request, jobid):
redis_host = getattr(settings, "ALLGO_DJANGO_REDIS_HOST")
r = redis.StrictRedis(host=redis_host, port=6379, db=0)
username, runner = get_token_cred(request)
if username != "$token" or not runner:
log.warning("Runner request $token user or a valid token")
return HttpResponse(status=401)
job = Job.objects.get(runner=runner, id=jobid).first()
job = Job.objects.get(runner=runner, id=jobid)
if not job:
log.warning("No job found for id %s and runner %s", jobid, runner)
return HttpResponse(status=401)
for i, l in enumerate(request.readlines()):
JobLog(linenumber=i, msg=l).save()
return
log.info("Job logger called for job %s", jobid)
log.info(request)
stream = request.META['wsgi.input']
line = None
linenumber = 0
while True:
c = stream.read(1)
if not c:
break
if c == b'\n':
k = "job:log:%s:%s" % (jobid, linenumber)
log.debug("redis %s %s", k, line)
r.setex(k, 60 * 60 * 24, line) # keep job log in cache 24 hours
linenumber += 1
line = None
else:
line = c if not line else line + c
log.info("Log finished ")
return JsonResponse({'status': 'ok'}, status=200)
def get_token_cred(request):
......
......@@ -27,6 +27,7 @@ USE_TZ = True
ALLGO_DJANGO_REGISTRY = os.environ.get("ALLGO_DJANGO_REGISTRY")
ALLGO_DJANGO_MAXUPLOADSIZE = os.environ.get("ALLGO_DJANGO_MAXUPLOADSIZE")
ALLGO_DJANGO_REDIS_HOST = os.environ.get("ALLGO_DJANGO_REDIS_HOST")
# DATABASES
# ------------------------------------------------------------------------------
......
......@@ -34,7 +34,8 @@ services:
ENV: dev
PYTHONUNBUFFERED: 1
ALLGO_DJANGO_REGISTRY: "localhost:5000"
ALLGO_DJANGO_MAXUPLOADSIZE: 1024
ALLGO_DJANGO_REDIS_HOST: "dev-redis"
ALLGO_DJANGO_MAXUPLOADSIZE: 16384
DJANGO_DEBUG: 1
DJANGO_LOG_LEVEL: "DEBUG"
ALLGO_ALLOWED_HOSTS: 0.0.0.0,dev-django,localhost
......@@ -54,6 +55,17 @@ services:
TOKEN_TYPE: "JWT"
# REDIS
######################################################################################################################
dev-redis:
container_name: dev-redis
image: redis:alpine
ports:
- "6379:6379"
networks: [dev]
# REGISTRY
######################################################################################################################
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment