Skip to content

GitLab

Commit 32af9563 authored by BERJON Matthieu's avatar BERJON Matthieu
Browse files

Merge branch 'django' of gitlab.inria.fr:allgo/allgo into open-bar-feature 


Signed-off-by: BERJON Matthieu's avatarMatthieu Berjon <matthieu.berjon@inria.fr>
parents 137d0724 d1e68c29
Pipeline #29872 failed with stage
in 46 seconds
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml 0 → 100644
View file @ 32af9563
image: docker:latest
variables:
DOCKER_DRIVER: overlay2
stages:
- build
bootstrap:
stage: build
script:
- docker info
- apk update
- apk upgrade
- apk add python python-dev py-pip build-base bash openssl python3
- pip install docker-compose
- mkdir -p /data/dev
- /bin/bash bootstrap dev-mysql dev-controller dev-ssh dev-django dev-nginx dev-smtpsink dev-registry
README.md
View file @ 32af9563
......@@ -128,7 +128,7 @@ To set up the development environment, run:
**Note** by default `bootstrap` works on all containers. It is possible
to give an explicit list of containers instead. Example:
<pre>
./bootstrap dev-mysql dev-rails
./bootstrap dev-mysql dev-django
</pre>
4. for convenience, you may want to alias `docker-compose` as `fig` (because
......@@ -138,6 +138,11 @@ To set up the development environment, run:
alias fig=docker-compose
</pre>
5. after bootstrap, in development the db is initialised with three users (all with the password `allgo`):
- `admin@localhost` which is superuser
- `devel@localhost` which is the owner of a webapp named `sleep`
- `guest@localhost`
### Common commands
......
bootstrap
View file @ 32af9563
......@@ -41,7 +41,7 @@ install_secrets()
if [ -f "$src" ] && [ -d "`dirname "$dst"`" ] && ! cmp -s "$src" "$dst" ; then
echo "Install tokens certificate as $dst/tokens.crt"
cp "$src" "$dst"
fig restart dev-registry
docker-compose restart dev-registry
fi
}
......
django/.gitignore
View file @ 32af9563
# Do not store any migration in the repository for the moment
# (to be removed when we deploy in production)
/allgo/main/migrations/*.py
### Python template
# Byte-compiled / optimized / DLL files
__pycache__/
......
django/allgo/main/admin.py
View file @ 32af9563
......@@ -10,6 +10,7 @@ from .models import (
JobQueue,
Runner,
Webapp,
WebappParameter,
WebappVersion,
)
......@@ -29,6 +30,16 @@ class WebappAdmin(admin.ModelAdmin):
list_display = ('id', 'name', 'docker_name', 'user')
class WebappParameterAdmin(admin.ModelAdmin):
list_display = ('id', 'get_webapp', 'name')
def get_webapp(self, obj):
return obj.webapp.name
get_webapp.short_description = 'Webapp'
get_webapp.admin_order_field = 'webapp__name'
class WebappVersionAdmin(admin.ModelAdmin):
list_display = ('id', 'get_webapp', 'number')
......@@ -51,6 +62,7 @@ admin.site.unregister(User)
admin.site.register(User, UserAdmin)
admin.site.register(Runner, RunnerAdmin)
admin.site.register(Webapp, WebappAdmin)
admin.site.register(WebappParameter, WebappParameterAdmin)
admin.site.register(WebappVersion, WebappVersionAdmin)
admin.site.register(Job, JobAdmin)
admin.site.register(AllgoUser)
......
django/allgo/main/forms.py
View file @ 32af9563
......@@ -10,6 +10,7 @@ from .models import (
JobQueue,
Runner,
Webapp,
WebappParameter,
WebappVersion,
)
......@@ -90,9 +91,16 @@ class JobForm(forms.ModelForm):
label_suffix='',
)
param = forms.CharField(label='Parameters', label_suffix='', required=False)
webapp_parameters = forms.ModelChoiceField(
queryset=WebappParameter.objects.all(),
label="presets",
required=False)
def __init__(self, *args, **kwargs):
super(JobForm, self).__init__(*args, **kwargs)
webapp = kwargs.pop('webapp')
super().__init__(*args, **kwargs)
self.fields['webapp_parameters'].queryset = WebappParameter.objects.filter(webapp=webapp)
for field in self.fields.values():
field.error_messages = {'required':'The field "{fieldname}" is required'.format(
......@@ -100,7 +108,7 @@ class JobForm(forms.ModelForm):
class Meta:
model = Job
fields = ('param', 'queue_id', 'files')
fields = ('param', 'queue_id', 'files', 'webapp_parameters')
class RunnerForm(forms.ModelForm):
......
django/allgo/main/helpers.py
View file @ 32af9563
import base64
import hashlib
import os
import redis
from django.conf import settings
import config
DEFAULT_ENTROPY = 32 # number of bytes to return by default
......@@ -54,3 +57,15 @@ def upload_data(file_obj, job_id):
with open(endpoint, 'wb+') as destination:
for chunk in file_data.chunks():
destination.write(chunk)
def read_runner_logs(jobid):
redis_host = config.env.ALLGO_REDIS_HOST
r = redis.StrictRedis(host=redis_host, port=6379, db=0)
linenumber = 0
k = "job:log:%s:%s" % (jobid, linenumber)
log.debug("Runner read logs k %s - %s", k, r.exists(k))
while r.exists(k):
yield r.get(k)
linenumber += 1
k = "job:log:%s:%s" % (jobid, linenumber)
django/allgo/main/management/commands/db_seed.py 0 → 100644
View file @ 32af9563
#!/usr/bin/python3
from django.core.management.base import BaseCommand
from main.models import User, Webapp, AllgoUser, Runner, DockerOs, JobQueue
from django.contrib.auth.models import User, Group
import os
PASSWORD = "allgo"
def create_user(email, *, superuser=False, **kw):
kind = "superuser" if superuser else "user"
print("create %s %r with password %r" % (kind, email, PASSWORD))
# create user
user = getattr(User.objects, "create_"+kind)(
email, email, PASSWORD, **kw)
# create & validate email account
user.emailaddress_set.create(email=email, verified=True, primary=True)
return user
class Command(BaseCommand):
def handle(self, **options):
if User.objects.count() or Webapp.objects.count() or JobQueue.objects.count():
raise RuntimeError("db is not empty")
# docker os list
DockerOs.objects.bulk_create([
DockerOs(name=name, docker_name=name, version=version)
for name, version in (
("debian", "8"),
("debian", "9"),
("centos", "6"),
("fedora", "20"),
("ubuntu", "18.04"),
)])
docker_os = DockerOs.objects.first()
# jobqueue
queue = JobQueue.objects.create(name="default", is_default=True)
# inria group
# FIXME: it seems that the 'inria' group is required (hardcoded in the
# FIXME: should put 'devel@localhost' into the inria group
# sources)
inria = Group.objects.create(name="inria")
# development setup
if os.environ["ENV"] == "dev":
# create admin user
admin = create_user("admin@localhost", superuser=True).allgouser
admin.sshkey="""
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwUS3knoJ3/Mqrc9B9CWih+VglNfcuz7Tr0beiIuCSwvu+9+hLIA2mjcdhjGzuAvJ1jepktxpWPe0PrpJ8vaG8nw/L8jG3Rt6SIwwXZVumeWqH/I/kGEk4ieE0bkqypCXIxAh748KEqW75gsZqmBaONl9sX5NqabhWmZImM1A/5l3kbyQWKYt+6wSLmtldd9saVkqhUtV1IJGgN0bNYTs7npyUYWSNNWFMM+3lhpiGN+QgUsralnXF67nVRlC/o8mZN3zyeT4dnThLZJkttycDgQTovgFRkhd2tNo08iVhzaX4KlwNDEeZzhPl5dXI6pquxlMKoOZku1yzwrtkvMCn anthony
ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3p5NOSk2XkPCN1PiCgJ5UXel+ByfuI3K0+SLIdGQkl0A5vRjSrVP4CVc9WqN+cJ/STYU3MRxNKUcLpCHUz4/809KUa7RlcY6Kzl4meOWzAVXEJD6zFgRmL33zp/vOmArYsqarjFIndR1yV962r+Pb7g2lFWV/HC9/VaKm/JinGCgJ89BMj1p0mkFQUVtAHrCMKzEMMYQ/vCbdA80ttySm3VWIeSWnDj2nbBHEBXVW2HkI+2vgtAFKI8W0F4sTNYHNhqz7R9s3q0Q548nloajDyEGToyJ4Y9X0fD8Jl7EwjUzmvJ2NhDYkn8fnWU2SvJDtEMvqToQSKAz3Mk/4aUrJQ== seb
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDS8WGGYVgPP0gkH5FTXK6C0HlzGfQVHtFueJ4lthtARcKCvAEStFdXkVqfZ146IAlP0jRgjLRJLZBwijGQDXrEMhXyerAfnvxIylYdb2pZhbhm4Kkt4as8gzOe57WQ0pEZWKyFuGc23B/k/UVzhdjKya1EJ1UrSfriIm30MW+awsgvh5UAERcX8qG6N0CJYnzW836dqUnFqRJ6rR7NydlJ22Fym/a1cyL2qyRSeUUsZBxJPotIinDHGQmdfmvMJ93dGJX3D7jK8GNxieRxs3JpRnHh+qBaeXqhsD5BOzMh+0W/440xK7tgtjWQu3nKJwKVhMV6fhVIelh4+MixaUk2qGa0tGXlMlcf+2lmDAXGuOiD6uMFqj2GOLSLNFk1wWQy+9//k+lHoZg/MP56PGBvUNXyxITCMsuP/WEtcDAq21aMGjimaIAfXSME/C8YtVFGhr/2az5kuIBUPGsFxVjohVstgDiaTC9BtcpCxRIWaOPFYIRA4bXf3Fnw3bQBlaLto6YgjPJ0Byf69ftRvbapEvAxFYXYGnC/mIVqG3GAdiD5o5BLr+has2lTDwKuYZMp7WGDkW0LI04ooVcAyqPV1Xe5/v+KguO1oY2FWlmp002qf8YJhAG/mPfYb88V5lA+wpTDuqCpj+ycFMlL8BT68etA+ZWz6BavkXrNED9lZw== matthieu
"""
admin.save()
devel = create_user("devel@localhost") # developer account
devel.groups.add(inria)
create_user("guest@localhost") # guest account
Webapp.objects.create(
name = "Sleep",
description = "a webapp that does nothing during the given number of seconds",
contact = "devel@localhost",
docker_name = "sleep",
entrypoint = "/bin/sleep",
private = False,
sandbox_state = Webapp.IDLE,
memory_limit = 32 * 1024**2,
docker_os = docker_os,
user = devel,
job_queue = queue,
)
django/allgo/main/migrations/0001_initial.py deleted 100644 → 0
View file @ 137d0724
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-04-18 07:48
from __future__ import unicode_literals
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
initial = True
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='AllgoUser',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('sshkey', models.CharField(max_length=40)),
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='allgouser', to=settings.AUTH_USER_MODEL)),
],
options={
'db_table': 'dj_users',
},
),
migrations.CreateModel(
name='DockerOs',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now_add=True)),
('name', models.CharField(blank=True, max_length=255)),
('version', models.CharField(blank=True, max_length=255)),
('docker_name', models.CharField(blank=True, max_length=255)),
],
options={
'db_table': 'dj_docker_os',
},
),
migrations.CreateModel(
name='Job',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now_add=True)),
('param', models.CharField(blank=True, max_length=255, null=True)),
('datasize', models.IntegerField(blank=True, null=True)),
('version', models.CharField(blank=True, max_length=255, null=True)),
('exec_time', models.IntegerField(blank=True, null=True)),
('access_token', models.CharField(blank=True, max_length=255, null=True)),
('state', models.IntegerField()),
('result', models.IntegerField()),
('container_id', models.IntegerField(blank=True, null=True)),
],
options={
'db_table': 'dj_jobs',
},
),
migrations.CreateModel(
name='JobQueue',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now_add=True)),
('name', models.CharField(max_length=255)),
('timeout', models.IntegerField(blank=True, null=True)),
('is_default', models.IntegerField()),
],
options={
'db_table': 'dj_job_queues',
},
),
migrations.CreateModel(
name='JobUploads',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now_add=True)),
('job_file_file_name', models.CharField(blank=True, max_length=255)),
('job_file_content_type', models.CharField(blank=True, max_length=255)),
('job_file_file_size', models.IntegerField(blank=True, null=True)),
('job_file_updated_at', models.DateTimeField(auto_now_add=True, null=True)),
('dj_job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='jobuploadjob', to='main.Job')),
],
options={
'db_table': 'dj_job_uploads',
},
),
migrations.CreateModel(
name='Quota',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now_add=True)),
('quantity', models.BigIntegerField(blank=True, null=True)),
('dj_user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='quotauser', to=settings.AUTH_USER_MODEL)),
],
options={
'db_table': 'dj_quotas',
},
),
migrations.CreateModel(
name='Webapp',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now_add=True)),
('name', models.CharField(blank=True, max_length=255, unique=True)),
('description', models.TextField(blank=True, null=True)),
('contact', models.CharField(blank=True, max_length=255, null=True)),
('logo_file_name', models.CharField(blank=True, max_length=255, null=True)),
('logo_content_type', models.CharField(blank=True, max_length=255, null=True)),
('logo_file_size', models.IntegerField(blank=True, null=True)),
('logo_updated_at', models.DateTimeField(blank=True, null=True)),
('default_quota', models.IntegerField(blank=True, null=True)),
('docker_name', models.CharField(blank=True, max_length=255)),
('readme', models.IntegerField(blank=True, null=True)),
('entrypoint', models.CharField(blank=True, max_length=255)),
('exec_time', models.IntegerField(blank=True, null=True)),
('private', models.IntegerField(default=1)),
('access_token', models.CharField(blank=True, max_length=255, null=True)),
('sandbox_state', models.IntegerField(null=True)),
('memory_limit', models.BigIntegerField(blank=True, null=True)),
('docker_os', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='webappdockeros', to='main.DockerOs')),
('job_queue', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='webappjobqueue', to='main.JobQueue')),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='webappuser', to=settings.AUTH_USER_MODEL)),
],
options={
'db_table': 'dj_webapps',
},
),
migrations.CreateModel(
name='WebappParameter',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now_add=True)),
('value', models.CharField(blank=True, max_length=255, null=True)),
('name', models.CharField(blank=True, max_length=255, null=True)),
('detail', models.CharField(blank=True, max_length=255, null=True)),
('webapp', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='webappparameters', to='main.Webapp')),
],
options={
'db_table': 'dj_webapp_parameters',
},
),
migrations.CreateModel(
name='WebappVersion',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now_add=True)),
('number', models.CharField(blank=True, max_length=255)),
('changelog', models.CharField(blank=True, max_length=255)),
('docker_image_size', models.FloatField(blank=True, null=True)),
('state', models.IntegerField(null=True)),
('published', models.IntegerField()),
('url', models.TextField()),
('webapp', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='webappversion', to='main.Webapp')),
],
options={
'db_table': 'dj_webapp_versions',
},
),
migrations.AddField(
model_name='quota',
name='dj_webapp',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='quotawebapp', to='main.Webapp'),
),
migrations.AddField(
model_name='job',
name='queue',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='queue', to='main.JobQueue'),
),
migrations.AddField(
model_name='job',
name='user',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='user', to=settings.AUTH_USER_MODEL),
),
migrations.AddField(
model_name='job',
name='webapp',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='webapp', to='main.Webapp'),
),
]
django/allgo/main/migrations/0002_auto_20180418_0755.py deleted 100644 → 0
View file @ 137d0724
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-04-18 07:55
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('main', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='Runner',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now_add=True)),
('token', models.CharField(max_length=255, unique=True)),
('hostname', models.CharField(blank=True, max_length=255, null=True)),
('cpu_count', models.IntegerField(blank=True, null=True)),
('mem_in_GB', models.IntegerField(blank=True, null=True)),
('last_seen', models.DateTimeField(null=True)),
('webapps', models.ManyToManyField(related_name='webapps', to='main.Webapp')),
],
options={
'db_table': 'dj_runners',
},
),
migrations.AddField(
model_name='job',
name='runner',
field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='runner', to='main.Runner'),
),
]
django/allgo/main/migrations/0003_joblogs.py deleted 100644 → 0
View file @ 137d0724
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-04-18 09:14
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('main', '0002_auto_20180418_0755'),
]
operations = [
migrations.CreateModel(
name='JobLogs',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now_add=True)),
('msg', models.TextField(blank=True, null=True)),
('dj_job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='job', to='main.Job')),
],
options={
'db_table': 'dj_job_logs',
},
),
]
django/allgo/main/migrations/0004_joblogs_linenumber.py deleted 100644 → 0
View file @ 137d0724
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-04-18 09:19
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('main', '0003_joblogs'),
]
operations = [
migrations.AddField(
model_name='joblogs',
name='linenumber',
field=models.IntegerField(default=0),
),
]
django/allgo/main/migrations/0005_auto_20180418_0922.py deleted 100644 → 0
View file @ 137d0724
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-04-18 09:22
from __future__ import unicode_literals
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('main', '0004_joblogs_linenumber'),
]
operations = [
migrations.RenameModel(
old_name='JobLogs',
new_name='JobLog',
),
]
django/allgo/main/migrations/0006_auto_20180419_1506.py deleted 100644 → 0
View file @ 137d0724
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-04-19 15:06
from __future__ import unicode_literals
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('main', '0005_auto_20180418_0922'),
]
operations = [
migrations.RemoveField(
model_name='joblog',
name='dj_job',
),
migrations.DeleteModel(
name='JobLog',
),
]
django/allgo/main/migrations/0007_auto_20180420_1253.py deleted 100644 → 0
View file @ 137d0724
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2018-04-20 12:53
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('main', '0006_auto_20180419_1506'),
]