...
  View open merge request
Commits (17)
#!/bin/bash
CONTAINERS="dev-redis dev-mysql dev-controller dev-ssh dev-django dev-smtpsink dev-registry dev-nginx dev-toolbox"
CONTAINERS="dev-redis dev-mysql dev-controller dev-ssh dev-django dev-smtpsink dev-registry dev-nginx dev-toolbox dev-jupyterhub"
die()
......
......@@ -8,7 +8,7 @@ from django.shortcuts import redirect
from django.urls import reverse
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import View
from main.helpers import upload_data, get_base_url, lookup_job_file, get_request_user
from main.helpers import upload_data, get_base_url, lookup_job_file, get_request_user, LoggerAdapter
from main.mixins import JobAuthMixin
from main.models import Job, Webapp, JobQueue
......@@ -44,10 +44,14 @@ class APIJobView(JobAuthMixin, View):
@csrf_exempt
def jobs(request):
o_log = LoggerAdapter(log, {'prefix': "API-jobs"})
o_log.info("request received")
user = get_request_user(request)
if not user:
log.info("API request without http authorisation %s %s %s", request.META['HTTP_USER_AGENT'],
request.META['REMOTE_ADDR'], request.META['QUERY_STRING'])
o_log.info("%s request without http authorisation %s %s %s",
request.META['HTTP_USER_AGENT'],
request.META['REMOTE_ADDR'],
request.META['QUERY_STRING'])
return JsonResponse({'error': 'API request without http authorisation'}, status=401)
if request.POST['job[webapp_id]'].isdigit():
......@@ -69,8 +73,13 @@ def jobs(request):
except JobQueue.DoesNotExist:
return JsonResponse({'error': 'Unknown queue'}, status=400)
log.info("Job submit by user %s", user)
o_log.info("Job submit by user %s", user)
job = Job.objects.create(param=request.POST.get('job[param]', ''), queue=queue, webapp=app, user=user)
if app.get_webapp_version() is None:
o_log.debug('No usable versions')
return JsonResponse({'error': "This app is not yet published"}, status=404)
job.version = app.get_webapp_version().number # TODO: add version selection in the api
upload_data(request.FILES.values(), job)
......@@ -83,6 +92,7 @@ def jobs(request):
# start the job
job.state = Job.WAITING
job.save()
o_log.info("request successfully submitted.")
no_domain_url = reverse('api:job', kwargs={'pk':job.id})
response = {
......
......@@ -128,8 +128,9 @@ class RunnerForm(forms.ModelForm):
class WebappForm(forms.ModelForm):
# Basic
name = forms.CharField(label="Application name", label_suffix="")
contact = forms.EmailField(label="Email contact", label_suffix="", required=False, help_text="By default this will be your personnal e-mail address. You may fill this field if you wish to use a different contact address.")
name = forms.CharField(label="Application name", label_suffix="")
contact = forms.EmailField(label="Email contact", label_suffix="",
required=False, help_text="By default this will be your personnal e-mail address. You may fill this field if you wish to use a different contact address.")
description = forms.CharField(widget=forms.Textarea, label="Description", label_suffix="")
private = forms.TypedChoiceField(
coerce=lambda x: x == 'True',
......@@ -143,6 +144,7 @@ class WebappForm(forms.ModelForm):
# Advanced
ADVANCED_FIELDS = ("docker_os", "memory_limit_mb", "job_queue",
"entrypoint", "owner", "tags")
docker_os = forms.ModelChoiceField(
queryset=DockerOs.objects.all().distinct(),
label='Operating sytem',
......@@ -158,8 +160,13 @@ class WebappForm(forms.ModelForm):
entrypoint = forms.CharField(label="Entrypoint", label_suffix="",
help_text=mark_safe('This is the <a href="https://allgo.gitlabpages.inria.fr/doc/deploy.html#entrypoint">command executed when allgo runs a job</a> for this app.'),
initial="/home/allgo/entrypoint")
owner = forms.CharField(required=False, label="Owner", label_suffix='', help_text="Username of the new owner of the application. You will immediately loose access to the application.")
tags = TagField(required=False, label_suffix='', help_text="Tags are separated by a comma.")
notebook_gitrepo = forms.CharField(required=False, label="Notebook repository", label_suffix="",
help_text=mark_safe('git repository URL (https only) publicly available.<br /> This repo should contain a notebook (ipynb file).'),
initial="https://")
owner = forms.CharField(required=False, label="Owner", label_suffix='',
help_text="Username of the new owner of the application. You will immediately loose access to the application.")
tags = TagField(required=False, label_suffix='',
help_text="Tags are separated by a comma.")
def __init__(self, *args, **kwargs):
super(WebappForm, self).__init__(*args, **kwargs)
......@@ -186,7 +193,9 @@ class WebappForm(forms.ModelForm):
class Meta:
model = Webapp
fields = ('name', 'description', 'contact', 'entrypoint', 'job_queue', 'private', 'docker_os', 'entrypoint', 'owner', 'tags')
fields = ('name', 'description', 'contact', 'private',
'job_queue', 'notebook_gitrepo',
'docker_os', 'entrypoint', 'owner', 'tags')
class WebappSandboxForm(forms.ModelForm):
......
......@@ -36,6 +36,15 @@ REDIS_MESSAGE_WEBAPP_UPDATED = "webapp:%d"
##################################################
class LoggerAdapter(logging.LoggerAdapter):
"""This class makes all log message prefixed.
Usage: log = LoggerAdapter("your prefix", logging.getLogger('allgo'))
cf: https://docs.python.org/3/howto/logging-cookbook.html
https://stackoverflow.com/questions/30002070/create-logger-that-prefixes-log-messages"""
def process(self, msg, kwargs):
return '[%s] %s' % (self.extra['prefix'], msg), kwargs
# global redis connection pool
_redis_connection_pool = None
......@@ -222,5 +231,3 @@ def query_webapps_for_user(user):
# select webapps that are either public or owned by the user
return Webapp.objects.filter(Q(private=False) | Q(user_id=user.id))
......@@ -5,7 +5,7 @@ from django.conf import settings
from django.contrib import auth
from django.contrib.auth.models import User, AnonymousUser
from django.core.validators import MinLengthValidator, MinValueValidator, \
RegexValidator
RegexValidator, URLValidator
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
......@@ -226,11 +226,14 @@ class Webapp(TimeStampModel):
private = models.BooleanField(default=1)
#FIXME: unused field
access_token = models.CharField(max_length=255, blank=True, null=True,
access_token = models.CharField(max_length=255, blank=True, null=True,
validators=[token_validator])
sandbox_state = models.IntegerField(null=True, choices=SANDBOX_STATE_CHOICES, default=IDLE)
sandbox_version = models.ForeignKey('WebappVersion', null=True, blank=True, related_name='webappversions')
notebook_gitrepo = models.CharField(max_length=255, blank=True, null=True)
sandbox_version = models.ForeignKey('WebappVersion', null=True, blank=True,
related_name='webappversions')
nb_url_validator = URLValidator(schemes=['https'], message="Only https url are allowed as notebook url.")
notebook_gitrepo = models.URLField(max_length=255, blank=True, null=True,
validators=[nb_url_validator])
memory_limit = models.BigIntegerField(null=True,
validators=[MinValueValidator(0)])
......
......@@ -282,10 +282,10 @@ class WebappCreate(ProviderAccessMixin, SuccessMessageMixin, CreateView):
group_required: groups that user must belong to.
"""
model = Webapp
model = Webapp
form_class = WebappForm
success_message = 'Webapp created successfully.'
template_name = 'webapp_add_update.html'
template_name = 'webapp_add_update.html'
# group_required = ['inria', ]
def get_success_url(self):
......@@ -316,6 +316,10 @@ class WebappCreate(ProviderAccessMixin, SuccessMessageMixin, CreateView):
obj.docker_name = slugify(form.cleaned_data['name'])
obj.memory_limit = form.get_memory_limit(self.request)
notebook_field = form.declared_fields['notebook_gitrepo']
if( notebook_field.initial == form.cleaned_data['notebook_gitrepo'] ):
obj.notebook_gitrepo = None
# validate the Webapp record before saving
# (this is a safety measure, do not remove)
# FIXME: currently this raises an exception if the slugify-generated
......@@ -1104,7 +1108,7 @@ class JobDetail(JobAuthMixin, DetailView):
class JobCreate(AllAccessMixin, SuccessMessageMixin, CreateView):
""" Display the data related a specific web and create a job instance
""" Display the data related a specific app and create a job instance
into the database
Attributes:
......
......@@ -304,3 +304,15 @@ a.fa-layers.disabled:not(.btn) {
height: 100%;
}
}
#jupyterhub
/* Values have been fixed based on the notebook used as test,
* to make disappear the elevators.
*/
{
width : 114%;
height: 350px; /* nothing works with % */
margin-left: -6%;
border: none;
}
......@@ -11,10 +11,10 @@
<meta name="description" content="allgo allow you to use console applications made by research team, online." />
<meta name="viewport" content="width=device-width, initial-scale=1.0, shrink-to-fit=no">
<link rel="stylesheet" type="text/css" href="{% static 'css/bootstrap.min.css' %}" media="all" charset="utf-8">
<link rel="stylesheet" type="text/css" href="{% static 'css/fontawesome-all.min.css' %}" media="all" charset="utf-8">
<link rel="stylesheet" type="text/css" href="{% static 'css/allgo.css' %}" media="all" charset="utf-8">
<link rel="stylesheet" type="text/css" href="{% static 'css/prism.css' %}" media="all" charset="utf-8">
<link rel="stylesheet" type="text/css" href="{% static 'css/bootstrap.min.css' %}" media="all">
<link rel="stylesheet" type="text/css" href="{% static 'css/fontawesome-all.min.css' %}" media="all">
<link rel="stylesheet" type="text/css" href="{% static 'css/allgo.css' %}" media="all">
<link rel="stylesheet" type="text/css" href="{% static 'css/prism.css' %}" media="all">
<link rel="icon" type="image/png" href="{% static 'images/favicon.ico' %}" sizes="16x16">
{% endblock %}
......
......@@ -41,8 +41,8 @@
</div>
</div>
<div class="logos">
<a href="https://inria.fr"><img class="float-left" src="{% static 'images/inria.png' %}"></a>
<a href="https://www.irisa.fr"><img class="float-right" src="{% static 'images/irisa.png' %}"></a>
<a href="https://inria.fr"><img class="float-left" src="{% static 'images/inria.png' %}" alt="inria logo"></a>
<a href="https://www.irisa.fr"><img class="float-right" src="{% static 'images/irisa.png' %}" alt="irisa logo"></a>
</div>
</div>
</footer>
......@@ -4,7 +4,7 @@
<div class="container">
<nav class="navbar navbar-expand-xl navbar-dark bg-dark">
<a class="navbar-brand text-hide" href="{% url 'main:home' %}">
<img src="{% static "images/logo-allgo.png" %}" width="50px" height="50px" alt="allgo logo">A||go</a>
<img src="{% static "images/logo-allgo.png" %}" width=50 height=50 alt="allgo logo">A||go</a>
<button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarNav" aria-controls="navbarNav" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
......
......@@ -16,30 +16,42 @@
<div class="card-body">
<nav class="navbar">
<ul class="nav nav-bar mx-auto">
<li class="nav-item">
<li class="nav-item"><!-- Run a job -->
<a
class="nav-link"
href="{% url 'main:webapp_detail' webapp.docker_name %}"
title="Start a job"
data-toggle="tooltip"
data-placement="bottom" ><i class="fas fa-play" data-fa-transform="shrink-10 up-.5" data-fa-mask="fas fa-circle"></i><span class="text-hide">Run a job</a></li>
data-placement="bottom" >
<i class="fas fa-play" data-fa-transform="shrink-10 up-.5" data-fa-mask="fas fa-circle"></i>
<span class="text-hide">Run a job</span>
</a>
</li>
{% if webapp.notebook_gitrepo %}
<li class="nav-item">
<li class="nav-item"><!-- Run a Notebook -->
<a
class="nav-link"
href="/jupyter?repo={{ webapp.notebook_gitrepo }}"
title="Start a Jupyter notebook"
data-toggle="tooltip"
data-placement="bottom"><i class="fas fa-sticky-note" data-fa-transform="shrink-10 up-.5" data-fa-mask="fas fa-circle"></i><span class="text-hide">Run notebook</a></li>
{% endif %}
data-placement="bottom">
<i class="fas fa-sticky-note" data-fa-transform="shrink-10 up-.5" data-fa-mask="fas fa-circle"></i>
<span class="text-hide">Run notebook</span>
</a>
</li>
{% endif %}
{% if webapp.user == user and webapp.sandbox_state != webapp.IDLE %}
<li class="nav-item">
<li class="nav-item"><!-- sandbox -->
<a
class="nav-link app-unfinished"
href="{% url 'main:webapp_sandbox_panel' webapp.docker_name %}"
title="There is an active sandbox"
data-toggle="tooltip"
data-placement="bottom"><i class="fas fa-exclamation-circle"></i><span class="text-hide">There is an active sandbox</span></a></li>
data-placement="bottom">
<i class="fas fa-exclamation-circle"></i>
<span class="text-hide">There is an active sandbox</span>
</a>
</li>
{% endif %}
</ul>
</nav>
......
......@@ -27,17 +27,14 @@
{% endif %}
</div>
</div>
<div class="col">
<div class="form-group">
{{ form.contact.label_tag }}
{{ form.contact | attr:"placeholder:john.smith@example.net" | add_class:"form-control" }}
<small class="form-text text-muted">{{ form.contact.help_text }}</small>
</div>
</div>
</div>
</div><!--form-row-->
<div class="form-group">
{{ form.description.label_tag }}
......@@ -48,20 +45,20 @@
<div class="form-group">
{{ form.private.label_tag }}
<div class="btn-group btn-group-toggle" data-toggle="buttons">
{% for private_field in form.private %}
<label class="btn btn-secondary {% if private_field.data.selected %}active{% endif %}">
<input
type="{{ private_field.data.type }}"
id="{{ private_field.id_for_label }}"
value="{{ private_field.data.value }}"
name="{{ private_field.data.name }}"
{% if private_field.data.selected %}checked{% endif %}>
{{ private_field.choice_label }}
</label>
{% endfor %}
{% for private_field in form.private %}
<label class="btn btn-secondary {% if private_field.data.selected %}active{% endif %}">
<input
type="{{ private_field.data.type }}"
id="{{ private_field.id_for_label }}"
value="{{ private_field.data.value }}"
name="{{ private_field.data.name }}"
{% if private_field.data.selected %}checked{% endif %}>
{{ private_field.choice_label }}
</label>
{% endfor %}
</div>
<small class="form-text text-muted">{{ form.private.help_text }}</small>
</div>
</div><!--form-group-->
<div class="card">
<div class="card-header">
......@@ -98,7 +95,6 @@
<div class="form-row">
<div class="form-group col-md-6">
{{ form.job_queue.label_tag }}
<select name="{{ form.job_queue.name }}" id="{{ form.job_queue.id_for_label }}" class="form-control">
{% for choice in form.job_queue.field.queryset %}
<option value="{{ choice.pk }}"
......@@ -113,7 +109,7 @@
{{ form.tags | attr:"placeholder:comma, separated, tags" | add_class:"form-control" }}
<small class="form-text text-muted">{{ form.tags.help_text }}</small>
</div>
</div>
</div><!-- form-group col-md-6 -->
<div class="form-group col-md-6">
{{ form.entrypoint.label_tag }}
......@@ -122,15 +118,20 @@
</div>
<small class="form-text text-muted">{{ form.entrypoint.help_text }}</small>
{% if action != "Create" %}
<div class="form-group">
{{ form.owner.label_tag }}
{{ form.owner | attr:"placeholder:Enter the new owner username" | add_class:"form-control" }}
<small class="form-text text-muted">{{ form.owner.help_text }}</small>
</div>
{% endif %}
</div>
{% if action != "Create" %}
<div class="form-group">
{{ form.owner.label_tag }}
{{ form.owner | attr:"placeholder:Enter the new owner username" | add_class:"form-control" }}
<small class="form-text text-muted">{{ form.owner.help_text }}</small>
</div>
{% endif %}
{{ form.notebook_gitrepo.label_tag }}
<div class="input-group">
{{ form.notebook_gitrepo | add_class:"form-control" }}
</div>
<small class="form-text text-muted">{{ form.notebook_gitrepo.help_text }}</small>
</div><!-- form-group col-md-6 -->
</div>
</div>
</div>
......
......@@ -203,11 +203,12 @@
<p class="mt-3">A Jupyter notebook is available to try the application online. You can
launch it <a href="/jupyter?repo={{ webapp.notebook_gitrepo }}" title="Jupyter notebook">using this link</a>.</p>
<p class="small text-muted">If you don't know what Jupyter is, you can <a href="https://jupyter.org/" title="Project Jupyter">visit their website</a>.</p>
<iframe id="jupyterhub"
src="/jupyter?repo={{ webapp.notebook_gitrepo }}"
></iframe>
</div>
{% endif %}
</div>
</div>
</div>
</div>
{% endblock %}
......
# code from https://github.com/opendns/registry-oauth-server/blob/master/tokens.py
import time
import hashlib
import base64
......@@ -10,12 +11,12 @@ import config
SIGNING_KEY_PATH = config.env.ALLGO_TOKEN_SIGNING_KEY_PATH
SIGNING_KEY_TYPE = config.env.ALLGO_TOKEN_SIGNING_KEY_TYPE
SIGNING_KEY_ALG = config.env.ALLGO_TOKEN_SIGNING_KEY_ALG
SIGNING_KEY_ALG = config.env.ALLGO_TOKEN_SIGNING_KEY_ALG
SIGNING_KEY = open(SIGNING_KEY_PATH).read()
ISSUER = config.env.ALLGO_TOKEN_ISSUER
ISSUER = config.env.ALLGO_TOKEN_ISSUER
TOKEN_EXPIRATION = config.env.ALLGO_TOKEN_EXPIRATION
TOKEN_TYPE = config.env.ALLGO_TOKEN_TYPE
TOKEN_TYPE = config.env.ALLGO_TOKEN_TYPE
def run_command(command):
......@@ -28,10 +29,11 @@ def key_id_encode(the_bytes):
result = []
for i in range(0, len(source), 4):
start = i
end = start+4
end = start + 4
result.append(str(source[start:end], 'utf-8'))
return ":".join(result)
def kid_from_crypto_key(private_key_path, key_type):
"""
python implementation of
......@@ -101,4 +103,3 @@ class Token(object):
headers=self.header)
return token
......@@ -4,13 +4,6 @@ from . import views
app_name = 'jwt'
urlpatterns = [
url(r'^jwt/auth$', views.jwt_auth, name="jwt_auth"), # REGISTRY_AUTH_TOKEN_REALM for docker registry
# hooks for registry pull/push for image manifests
url(r'^jwt/pre-(push|pull)$', views.pre_pushpull, name="pre_pushpull"),
url(r'^jwt/post-push$', views.post_push, name="post_push"),
# default catch-all route for docker registry urls (normally unused because
# the reverse-proxy is expected to route them directly to the registry)
url(r'^v2/', views.registry_notfound),
url(r'^jwt/auth', views.jwt_auth, name="jwt_auth"), # REGISTRY_AUTH_TOKEN_REALM for docker registry
url(r'^jupyter$', views.jupyter, name="jupyter"), # 302 redirect to jupyterhub + bearer + gitrepo
]
This diff is collapsed.
......@@ -42,10 +42,32 @@ services:
ALLGO_DEBUG: "True"
ALLGO_ALLOW_LOCAL_ACCOUNTS: "True"
ALLGO_JUPYTER_URL: "http://0.0.0.0:8000/hub/login"
ALLGO_TOKEN_SIGNING_KEY_PATH: "/vol/ro/certs/tokens.key"
ALLGO_HTTP_SERVER: "django"
ALLGO_EMAIL_BACKEND: "django.core.mail.backends.console.EmailBackend"
# JUPYTER HUB
######################################################################################################################
dev-jupyterhub:
container_name: dev-jupyterhub
build: jupyterhub
ports:
- "8000:8000"
volumes:
- "/data/dev/django/ro/certs:/certs"
- "./jupyterhub/jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py"
- "./django/allgo-logo.svg:/srv/jupyterhub/allgo_logo.svg"
- "./jupyterhub/jupyter_notebook_config.py:/etc/jupyter/jupyter_notebook_config.py"
networks: [dev]
environment:
JUPYTERHUB_LOG_LEVEL: "debug"
JUPYTERHUB_JWT_KEY: "/certs/tokens.key"
ALLGO_URL: "https://dev-django"
# IFRAME_CSP: "http://localhost:8008" # we do not succeed in unsing it.
# REDIS
######################################################################################################################
......
jupyterhub_cookie_secret
jupyterhub.sqlite
FROM debian:jessie
ENV DEBIAN_FRONTEND noninteractive
RUN REPO=http://cdn-fastly.deb.debian.org && \
echo "deb $REPO/debian jessie main\ndeb $REPO/debian-security jessie/updates main" > /etc/apt/sources.list && \
apt-get -y update && \
apt-get -y upgrade && \
apt-get -y install wget locales git bzip2 &&\
/usr/sbin/update-locale LANG=C.UTF-8 && \
locale-gen C.UTF-8 && \
apt-get remove -y locales && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
ENV LANG C.UTF-8
# install Python + NodeJS with conda
RUN wget -q https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh -O /tmp/miniconda.sh && \
echo 'd0c7c71cc5659e54ab51f2005a8d96f3 */tmp/miniconda.sh' | md5sum -c - && \
bash /tmp/miniconda.sh -f -b -p /opt/conda && \
/opt/conda/bin/conda install --yes -c conda-forge \
python=3.5 sqlalchemy tornado jinja2 traitlets requests pip pycurl \
nodejs configurable-http-proxy && \
/opt/conda/bin/pip install --upgrade pip && \
rm /tmp/miniconda.sh
ENV PATH=/opt/conda/bin:$PATH
ADD . /src/jupyterhub
WORKDIR /src/jupyterhub
RUN rm -rf $PWD ~/.cache ~/.npm
RUN pip install notebook==5.4.1
RUN pip install matplotlib
#RUN conda install matplotlib
#RUN conda install pandas
#RUN conda install -c anaconda numpy
#RUN conda install -c anaconda scipy
#RUN conda install scikit-learn
RUN pip install https://github.com/data-8/nbgitpuller/archive/28fe9b1af2ba64b346d59bd13c99581346bf349f.zip --upgrade
RUN jupyter serverextension enable --py nbgitpuller --sys-prefix
RUN pip install ipywidgets
RUN jupyter nbextension enable --py widgetsnbextension
RUN pip install https://github.com/scampion/jwtauthenticator/archive/master.zip --upgrade
RUN pip install allgo
RUN mkdir -p /srv/jupyterhub/
WORKDIR /srv/jupyterhub/
EXPOSE 8000
ENV PYTHONUNBUFFERED 1
CMD jupyterhub -f /srv/jupyterhub/jupyterhub_config.py
\ No newline at end of file
import os
c.NotebookApp.tornado_settings = {
'headers': {
'Content-Security-Policy': "frame-ancestors 'self' http://localhost:8008 "
# 'Content-Security-Policy': "frame-ancestors 'self' %s " % os.environ.get("IFRAME_CSP") #FIXME
}
}
This diff is collapsed.