Commit dc16a66d authored by CAMPION Sebastien's avatar CAMPION Sebastien
Browse files

first release compatible with runner

parent 08b546b9
......@@ -7,6 +7,9 @@ from main.models import AllgoUser
# Define an inline admin descriptor for Employee model
# which acts a bit like a singleton
from .models import Runner, Webapp
class AllgoUserInline(admin.StackedInline):
model = AllgoUser
can_delete = False
......@@ -21,3 +24,5 @@ class UserAdmin(BaseUserAdmin):
# Re-register UserAdmin
admin.site.unregister(User)
admin.site.register(User, UserAdmin)
admin.site.register(Runner)
admin.site.register(Webapp)
\ No newline at end of file
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-04-18 09:14
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('main', '0002_auto_20180418_0755'),
]
operations = [
migrations.CreateModel(
name='JobLogs',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now_add=True)),
('msg', models.TextField(blank=True, null=True)),
('dj_job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='job', to='main.Job')),
],
options={
'db_table': 'dj_job_logs',
},
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-04-18 09:19
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('main', '0003_joblogs'),
]
operations = [
migrations.AddField(
model_name='joblogs',
name='linenumber',
field=models.IntegerField(default=0),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-04-18 09:22
from __future__ import unicode_literals
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('main', '0004_joblogs_linenumber'),
]
operations = [
migrations.RenameModel(
old_name='JobLogs',
new_name='JobLog',
),
]
......@@ -35,7 +35,7 @@ class AllgoUser(models.Model):
db_table = 'dj_users'
def getApp(self):
return [a.name for a in Webapp.objects.filter(user_id=self.user.id)]
return [a.name.lower() for a in Webapp.objects.filter(user_id=self.user.id)]
class DockerOs(TimeStampModel):
......@@ -212,6 +212,15 @@ class Job(TimeStampModel):
db_table = 'dj_jobs'
class JobLog(TimeStampModel):
msg = models.TextField(blank=True, null=True)
linenumber = models.IntegerField(default=0)
dj_job = models.ForeignKey(Job, related_name="job")
class Meta:
db_table = 'dj_job_logs'
class JobUploads(TimeStampModel):
"""
Data uploaded by the user to feed a given webapp
......
......@@ -6,9 +6,10 @@ app_name = 'main'
urlpatterns = [
url(r'^$', views.index, name="home"),
url(r'^tokens$', views.tokens, name="tokens"),
url(r"^runner/cmd/[sha1|sha256|md5]*:(\b[0-9a-f]{5,40}\b)$", views.runner_cmd, name="runner_cmd"),
url(r"^runner/dw/(\d+)/(.*)", views.runner_dw, name="runner_dw"),
url(r"^runner/up/(.*)", views.runner_up, name="runner_up"),
url(r"^runner/cmd$", views.runner_cmd, name="runner_cmd"),
url(r"^runner/dw/(\d+)/(.*)", views.runner_dw, name="runner_dw"),
url(r"^runner/up/(\d+)/[sha1|sha256|md5]*:(\b[0-9a-f]{5,40}\b)/(\d+)", views.runner_up, name="runner_up"),
url(r"^runner/logs/(\d+)", views.runner_log, name="runner_log"),
url(r'^registryhook', views.registryhook, name="registryhook"),
url(r'^jupyter$', views.jupyter, name="jupyter"),
url(r'^apps/$', views.WebappList.as_view(), name='webapp_list'),
......
import base64
import hashlib
import json
import logging
import os
import socket
import time
import tarfile
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import PasswordChangeForm
from django.contrib.auth.mixins import LoginRequiredMixin
from django.core.urlresolvers import reverse
from django.http import JsonResponse, HttpResponse
from django.http import StreamingHttpResponse
from django.http import JsonResponse, HttpResponse, FileResponse
from django.shortcuts import redirect
from django.shortcuts import render, get_object_or_404
from django.urls import reverse, reverse_lazy
......@@ -24,37 +24,91 @@ from django.views.generic import (
from .forms import UserForm, HomeSignupForm
from .models import User
from .models import Webapp, Job, AllgoUser, WebappVersion, Runner, JobLog
from .tokens import Token
from .models import Webapp, Job, AllgoUser, WebappVersion, Runner
log = logging.getLogger('allgo')
BUF_SIZE = 65536
@csrf_exempt
def runner_dw(request, jobid, filename):
username, runner = get_token_cred(request)
if username != "$token" or not runner:
log.warning("Runner request $token user or a valid token")
return HttpResponse(status=401)
job = Job.objects.get(runner=runner, id=jobid).first()
job = Job.objects.get(runner=runner, id=jobid)
if not job:
log.warning("No job found for id %s and runner %s", jobid, runner)
return HttpResponse(status=401)
assert ".." not in filename, "filename unsecure"
datastore = os.environ.get("ALLGO_DATASTORE")
filepath = os.path.join(datastore, jobid, filename)
with open(filepath, 'r') as fp:
data = fp.read()
response = HttpResponse(mimetype="application/octet-stream")
response['Content-Disposition'] = 'attachment; filename=%s' % filename
response.write(data)
return
assert ".." not in filepath, "filepath unsecure"
return FileResponse(open(filepath, 'rb'))
def sha1file(filepath):
sha1 = hashlib.sha1()
with open(filepath, 'rb') as f:
while True:
data = f.read(BUF_SIZE)
if not data:
break
sha1.update(data)
return "sha1:%s" % sha1.hexdigest()
@csrf_exempt
def runner_up(request, jobid, digest, chunkid):
username, runner = get_token_cred(request)
if username != "$token" or not runner:
log.warning("Runner request $token user or a valid token")
return HttpResponse(status=401)
def runner_up(request):
return "FIXME"
job = Job.objects.get(runner=runner, id=jobid)
if not job:
log.warning("No job found for id %s and runner %s", jobid, runner)
return HttpResponse(status=401)
datastore = os.environ.get("ALLGO_DATASTORE")
odir = os.path.join(datastore, jobid, ".%s" % digest)
if not os.path.exists(odir):
os.mkdir(odir)
filepath = os.path.join(odir, chunkid)
assert ".." not in filepath, "filepath unsecure"
with open(filepath, 'wb') as fp:
fp.write(request.body)
tarfilepath = os.path.join(datastore, jobid, ".%s.tar" % digest)
assert ".." not in tarfilepath, "filepath unsecure"
with open(tarfilepath, 'wb') as fp:
for chunk in range(len(os.listdir(odir))):
with open(os.path.join(odir, str(chunk)), 'rb') as c:
fp.write(c.read())
print(sha1file(tarfilepath), digest)
if sha1file(tarfilepath).split(":")[1] == digest:
with tarfile.open(tarfilepath, "r") as tar:
tar.extractall(os.path.join(datastore, jobid))
log.info("Results successfully uploaded for jobid %s", jobid)
return HttpResponse(status=200)
@csrf_exempt
def runner_log(request, jobid):
username, runner = get_token_cred(request)
if username != "$token" or not runner:
log.warning("Runner request $token user or a valid token")
return HttpResponse(status=401)
job = Job.objects.get(runner=runner, id=jobid).first()
if not job:
log.warning("No job found for id %s and runner %s", jobid, runner)
return HttpResponse(status=401)
for i, l in enumerate(request.readlines()):
JobLog(linenumber=i, msg=l).save()
return
def get_token_cred(request):
......@@ -69,31 +123,39 @@ def get_token_cred(request):
def runner_jobs(runner):
while True:
job = Job.objects.filter(state=0, runner=runner).first()
if job:
yield job
time.sleep(5)
job = Job.objects.filter(state=0, runner=runner.id).first()
if job:
log.debug("Send job %s to runner %s", job.id, runner.token)
datastore = os.environ.get("ALLGO_DATASTORE")
jdir = os.path.join(datastore, str(job.id))
files = {f: sha1file(os.path.join(jdir, f)) for f in os.listdir(jdir)}
job.state = 1
job.save()
return json.dumps({"jobid": job.id,
"registry": getattr(settings, "ALLGO_DJANGO_REGISTRY"),
"image": job.webapp.name,
"files": files})
else:
return ""
@csrf_exempt
def runner_cmd(request):
username, runner = get_token_cred(request)
if username != "$token" or not runner:
log.warning("Runner request $token user or a valid token")
return HttpResponse(status=401)
return StreamingHttpResponse(runner_jobs(runner))
return HttpResponse(runner_jobs(runner))
def get_allowed_actions(user, scope, actions):
allgouser = AllgoUser(user=user)
if not scope:
return []
resource_type, resource_name, resource_actions = scope.split(":")
if resource_type == "repository" and resource_name.rstrip('-incoming') in allgouser.getApp():
return actions
if isinstance(user, Runner):
if resource_type == "repository" and resource_name in [w.name for w in user.webapps]:
return ['pull']
else:
return []
allgouser = AllgoUser(user=user)
if resource_type == "repository" and resource_name.rstrip('-incoming') in allgouser.getApp():
return actions
def update_webapp_metadata(repository, url):
log.info("New webapp version received %s %s", repository, url)
......@@ -103,15 +165,17 @@ def update_webapp_metadata(repository, url):
def notify_controler():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5)
sock.connect((os.environ.get('ALLGO_CONTROLLER_HOST'), int(os.environ.get('ALLGO_CONTROLLER_PORT'))))
sock.settimeout(None)
sock.makefile('rb', 0)
sock.shutdown(socket.SHUT_RDWR)
sock.close()
log.info("Controller notified")
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5)
sock.connect((os.environ.get('ALLGO_CONTROLLER_HOST'), int(os.environ.get('ALLGO_CONTROLLER_PORT'))))
sock.settimeout(None)
sock.makefile('rb', 0)
sock.shutdown(socket.SHUT_RDWR)
sock.close()
log.info("Controller notified")
except Exception as e:
log.error("Controller notification failed !!! %s", str(e ))
def index(request):
"""
......@@ -121,10 +185,10 @@ def index(request):
webapps = Webapp.objects.all().count()
jobs = Job.objects.all().count()
context = {
'signup_form': HomeSignupForm(),
'user_nb': users,
'webapp_nb': webapps,
'job_nb': jobs,
'signup_form': HomeSignupForm(),
'user_nb': users,
'webapp_nb': webapps,
'job_nb': jobs,
}
return render(request, "home.html", context)
......@@ -158,15 +222,19 @@ def tokens(request):
return HttpResponse(status=401)
token_type, credentials = auth_header.split(' ')
username, password = base64.b64decode(credentials).decode('utf-8').split(':')
try:
user = User.objects.get(email=username)
except User.DoesNotExist:
log.warning("Token request but user doest not exist")
return HttpResponse(status=401)
password_valid = user.check_password(password)
if token_type != 'Basic' or not password_valid:
log.info("Token request but user password mismatch")
return HttpResponse(status=401)
if username == "$token" and Runner.objects.get(token=password):
log.info("Token for runner called")
user = Runner.objects.get(token=password)
else:
try:
user = User.objects.get(email=username)
except User.DoesNotExist:
log.warning("Token request but user doest not exist")
return HttpResponse(status=401)
password_valid = user.check_password(password)
if token_type != 'Basic' or not password_valid:
log.info("Token request but user password mismatch")
return HttpResponse(status=401)
service = request.GET['service']
scope = request.GET['scope'] if 'scope' in request.GET.keys() else None
......@@ -182,8 +250,8 @@ def tokens(request):
name = params[1]
actions = params[2].split(',')
authorized_actions = get_allowed_actions(user, scope, actions)
authorized_actions = get_allowed_actions(user, scope, actions) if scope else []
log.info("Token authorized actions %s %s %s", authorized_actions, user, scope)
token = Token(service, typ, name, authorized_actions)
encoded_token = token.encode_token()
......@@ -234,9 +302,9 @@ class WebappDetail(DetailView):
# Check if a readme is declared in the database
if self.object.readme:
readme_file = os.path.join(
settings.MEDIA_ROOT,
self.object.docker_name,
'Readme')
settings.MEDIA_ROOT,
self.object.docker_name,
'Readme')
with open(readme_file, 'r') as md_data:
print(md_data)
context['readme'] = md_data.read()
......@@ -300,7 +368,7 @@ class UserPasswordUpdateView(LoginRequiredMixin, UpdateView):
def dispatch(self, request, *args, **kwargs):
return super(UserPasswordUpdateView, self) \
.dispatch(request, *args, **kwargs)
.dispatch(request, *args, **kwargs)
class UserUpdateView(LoginRequiredMixin, UpdateView):
......
......@@ -25,6 +25,9 @@ USE_L10N = True
USE_TZ = True
ALLGO_DJANGO_REGISTRY = os.environ.get("ALLGO_DJANGO_REGISTRY")
# DATABASES
# ------------------------------------------------------------------------------
DATABASES = {
......
......@@ -22,7 +22,7 @@ services:
user: "$DOCKERUSER"
ports:
- "8008:8000"
#command: "python3 manage.py runserver 0.0.0.0:8000"
command: "python3 manage.py runserver 0.0.0.0:8000"
volumes:
- "/data/dev/django:/vol"
- "./django:/opt/allgo"
......@@ -33,9 +33,11 @@ services:
environment:
ENV: dev
PYTHONUNBUFFERED: 1
ALLGO_ALLOWED_HOSTS: 0.0.0.0,dev-django,localhost
ALLGO_DJANGO_REGISTRY: "http://localhost:5000/"
DJANGO_DEBUG: 1
DJANGO_LOG_LEVEL: "DEBUG"
ALLGO_ALLOWED_HOSTS: 0.0.0.0,dev-django,localhost
ALLGO_DATASTORE: "/vol/rw/datastore"
ALLGO_DEBUG: "True"
ALLGO_JUPYTER_URL: "http://0.0.0.0:8000/hub/login"
ALLGO_EMAIL_BACKEND: "django.core.mail.backends.console.EmailBackend"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment