Commit e8aa7dd8 authored by BAIRE Anthony's avatar BAIRE Anthony

Merge remote-tracking branch 'refs/remotes/origin/django' into django

# Conflicts:
#	django/Dockerfile
#	docker-compose.yml
parents be96a87a d5c1cd43
......@@ -4,6 +4,12 @@ set -e
mkdir -p certs
if [ ! -f cert/server.key ] & [ ! -f certs/server.crt ]; then
openssl req -subj '/CN=localhost/O=Registry Demo/C=US' -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout certs/server.key -out certs/server.crt
# build base image (if not present)
(set -x ; make base-debian)
......@@ -8,9 +8,9 @@ FROM allgo/base-debian
COPY setup/backports/. /
RUN apt-getq update && apt-getq install mysql-server default-libmysqlclient-dev \
python3-django python3-django-allauth \
python3-django python3-django-allauth python3-misaka \
nginx-light zip gcc python3-dev python3-pip python3-wheel python3-mysqldb \
python-mysqldb supervisor\
python-mysqldb python3-crypto supervisor\
&& pip3 install gunicorn
COPY requirements.txt /tmp/
......@@ -31,4 +31,5 @@ RUN apply-patches /opt/allgo/setup/patches/*.diff &&\
WORKDIR /opt/allgo
LABEL dk.migrate_always=1
CMD run-allgo
from allauth.account.forms import SignupForm
from django import forms
from .models import User
class UserForm(forms.ModelForm):
first_name = forms.CharField(
label='First name',
last_name = forms.CharField(label='Last name', label_suffix='')
class Meta:
model = User
fields = ('first_name', 'last_name')
class HomeSignupForm(SignupForm):
def __init__(self, *args, **kwargs):
super(HomeSignupForm, self).__init__(*args, **kwargs)
......@@ -30,12 +30,14 @@ class AllgoUser(models.Model):
# Relationships
user = models.OneToOneField(
User, on_delete=models.CASCADE, related_name="allgouser")
User, on_delete=models.CASCADE, related_name="allgouser")
class Meta:
db_table = 'dj_users'
def getApp(self):
return [ for a in Webapp.objects.filter(]
class DockerOs(TimeStampModel):
......@@ -108,9 +110,9 @@ class Webapp(TimeStampModel):
# A webapp has one docker os type
docker_os = models.OneToOneField(DockerOs, related_name="webappdockeros")
user = models.ForeignKey(
User, on_delete=models.CASCADE, related_name="webappuser")
User, on_delete=models.CASCADE, related_name="webappuser")
job_queue = models.OneToOneField(
JobQueue, on_delete=models.CASCADE, related_name="webappjobqueue")
JobQueue, on_delete=models.CASCADE, related_name="webappjobqueue")
class Meta:
db_table = 'dj_webapps'
......@@ -145,9 +147,10 @@ class WebappVersion(TimeStampModel):
docker_image_size = models.FloatField(blank=True, null=True)
state = models.IntegerField(null=True)
published = models.IntegerField()
url = models.TextField()
# Relationships
dj_webapp = models.ForeignKey(Webapp, related_name="webappversion")
webapp = models.ForeignKey(Webapp, related_name="webappversion")
class Meta:
db_table = 'dj_webapp_versions'
......@@ -166,9 +169,9 @@ class Quota(TimeStampModel):
# Relationships
dj_user = models.ForeignKey(
User, on_delete=models.CASCADE, related_name="quotauser")
User, on_delete=models.CASCADE, related_name="quotauser")
dj_webapp = models.ForeignKey(
Webapp, on_delete=models.CASCADE, related_name="quotawebapp")
Webapp, on_delete=models.CASCADE, related_name="quotawebapp")
class Meta:
db_table = 'dj_quotas'
......@@ -210,7 +213,7 @@ class JobUploads(TimeStampModel):
job_file_content_type = models.CharField(max_length=255, blank=True)
job_file_file_size = models.IntegerField(blank=True, null=True)
job_file_updated_at = models.DateTimeField(
blank=True, auto_now_add=True, null=True)
blank=True, auto_now_add=True, null=True)
# Relationships
dj_job = models.ForeignKey(Job, related_name="jobuploadjob")
from django import template
from misaka import Markdown, HtmlRenderer
register = template.Library()
def markdown(value):
""" Convert markdown content into HTML. """
renderer = HtmlRenderer()
markdown = Markdown(
return markdown(value)
from copy import copy
from django import template
import types
register = template.Library()
def silence_without_field(fn):
def wrapped(field, attr):
if not field:
return ""
return fn(field, attr)
return wrapped
def _process_field_attributes(field, attr, process):
# split attribute name and value from 'attr:value' string
params = attr.split(':', 1)
attribute = params[0]
value = params[1] if len(params) == 2 else True
field = copy(field)
# decorate field.as_widget method with updated attributes
old_as_widget = field.as_widget
def as_widget(self, widget=None, attrs=None, only_initial=False):
attrs = attrs or {}
process(widget or self.field.widget, attrs, attribute, value)
html = old_as_widget(widget, attrs, only_initial)
self.as_widget = old_as_widget
return html
field.as_widget = types.MethodType(as_widget, field)
return field
def set_attr(field, attr):
def process(widget, attrs, attribute, value):
attrs[attribute] = value
return _process_field_attributes(field, attr, process)
def append_attr(field, attr):
def process(widget, attrs, attribute, value):
if attrs.get(attribute):
attrs[attribute] += ' ' + value
elif widget.attrs.get(attribute):
attrs[attribute] = widget.attrs[attribute] + ' ' + value
attrs[attribute] = value
return _process_field_attributes(field, attr, process)
def add_class(field, css_class):
return append_attr(field, 'class:' + css_class)
import os
import time
import hashlib
import base64
import subprocess
from Crypto.PublicKey import RSA
from jose import jwt
ISSUER = os.environ.get('ISSUER')
TOKEN_TYPE = os.environ.get('TOKEN_TYPE')
def run_command(command):
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
return process.communicate()
def key_id_encode(the_bytes):
source = base64.b32encode(the_bytes)
result = []
for i in range(0, len(source), 4):
start = i
end = start+4
result.append(str(source[start:end], 'utf-8'))
return ":".join(result)
def kid_from_crypto_key(private_key_path, key_type):
python implementation of
returns a distinct identifier which is unique to
the public key derived from this private key.
The format generated by this library is a base32 encoding of a 240 bit
hash of the public key data divided into 12 groups like so:
assert key_type == 'RSA', "Only RSA key type supported"
algorithm = hashlib.sha256()
key = RSA.importKey(open(private_key_path).read())
der = key.publickey().exportKey("DER")
return key_id_encode(algorithm.digest()[:30])
class Token(object):
def __init__(self, service, access_type="", access_name="",
access_actions=None, subject=''):
if access_actions is None:
access_actions = []
self.issuer = ISSUER
self.signing_key = SIGNING_KEY
self.signing_key_path = SIGNING_KEY_PATH
self.signing_key_type = SIGNING_KEY_TYPE
self.signing_key_alg = SIGNING_KEY_ALG
self.token_expiration = TOKEN_EXPIRATION
self.token_type = TOKEN_TYPE
self.header = {
'typ': self.token_type,
'alg': self.signing_key_alg,
'kid': kid_from_crypto_key(self.signing_key_path, self.signing_key_type)
self.claim = {
'iss': self.issuer,
'sub': subject,
'aud': service,
'exp': int(time.time()) + int(self.token_expiration),
'nbf': int(time.time()) - 30,
'iat': int(time.time()),
'access': [
'type': access_type,
'name': access_name,
'actions': access_actions
def set_header(self, header):
self.header = header
def get_header(self):
return self.header
def set_claim(self, claim):
self.claim = claim
def get_claim(self):
return self.claim
def encode_token(self):
token = jwt.encode(self.claim, self.signing_key,
return token
......@@ -5,15 +5,11 @@ app_name = 'main'
urlpatterns = [
url(r'^$', views.index, name="home"),
url(r'^tokens$', views.tokens, name="tokens"),
url(r'^registryhook', views.registryhook, name="registryhook"),
url(r'^apps/$', views.WebappList.as_view(), name='webapp_list'),
url(r'^app/(?P<docker_name>[\w-]+)/$', views.WebappDetail.as_view(), name='webapp_detail'),
url(r'^jobs/$', views.JobList.as_view(), name='job_list'),
# Settings
url(r'^settings/$', views.UserUpdateView.as_view(), name='user_settings'),
url(r'^settings/password$', views.UserPasswordUpdateView.as_view(), name='user_password_update'),
import base64
import json
import socket
import logging
import os
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.decorators.csrf import csrf_exempt
from django.contrib.auth.forms import PasswordChangeForm
from django.contrib.auth.models import User
from django.shortcuts import render, get_object_or_404
from django.urls import reverse
from django.conf import settings
from django.http import JsonResponse, HttpResponse
from django.urls import reverse, reverse_lazy
from django.views.generic import (
from .tokens import Token
from .models import Webapp, Job, User, AllgoUser, WebappVersion
from .forms import UserForm, HomeSignupForm
log = logging.getLogger('allgo')
def get_allowed_actions(user, scope, actions):
allgouser = AllgoUser(user=user)
if not scope:
return []
resource_type, resource_name, resource_actions = scope.split(":")
if resource_type == "repository" and resource_name.rstrip('-incoming') in allgouser.getApp():
return actions
return []
def update_webapp_metadata(repository, url):"New webapp version received %s %s", repository, url)
webapp = Webapp.objects.filter(name=repository)[0]
WebappVersion(url=url, state=0, published=0, webapp=webapp).save()
from .models import Webapp, Job, AllgoUser
def notify_controler():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((os.environ.get('ALLGO_CONTROLLER_HOST'), int(os.environ.get('ALLGO_CONTROLLER_PORT'))))
sock.makefile('rb', 0)
sock.close()"Controller notified")
def index(request):
return render(request, "home.html")
Do nothing specific just return a specific template
users = User.objects.all().count()
webapps = Webapp.objects.all().count()
jobs = Job.objects.all().count()
context = {
'signup_form': HomeSignupForm(),
'user_nb': users,
'webapp_nb': webapps,
'job_nb': jobs,
return render(request, "home.html", context)
def registryhook(request):
body_unicode = request.body.decode('utf-8')
body = json.loads(body_unicode)
for event in body['events']:
if event['action'] == "push":
update_webapp_metadata(event['target']['repository'], event['target']['url'])
return HttpResponse(status=200)
def tokens(request):
auth_header = request.META.get('HTTP_AUTHORIZATION', '')
if not auth_header:"Token request without http authorisation %s %s %s", request.META['HTTP_USER_AGENT'],
request.META['REMOTE_ADDR'], request.META['QUERY_STRING'])
return HttpResponse(status=401)
token_type, credentials = auth_header.split(' ')
username, password = base64.b64decode(credentials).decode('utf-8').split(':')
user = User.objects.get(email=username)
except User.DoesNotExist:
log.warning("Token request but user doest not exist")
return HttpResponse(status=401)
password_valid = user.check_password(password)
if token_type != 'Basic' or not password_valid:"Token request but user password mismatch")
return HttpResponse(status=401)
service = request.GET['service']
scope = request.GET['scope'] if 'scope' in request.GET.keys() else None
if not scope:
typ = ''
name = ''
actions = []
params = scope.split(':')
if len(params) != 3:
return JsonResponse({'error': 'Invalid scope parameter'}, status=400)
typ = params[0]
name = params[1]
actions = params[2].split(',')
authorized_actions = get_allowed_actions(user, scope, actions)
token = Token(service, typ, name, authorized_actions)
encoded_token = token.encode_token()
return JsonResponse({'token': encoded_token})
class WebappList(ListView):
......@@ -31,7 +139,7 @@ class WebappList(ListView):
pattern_name = 'webapp_list'
paginate_by = 10
template_name = 'webapp_list.html'
queryset = Webapp.objects.filter(private=0).order_by('created_at')
queryset = Webapp.objects.filter(private=0).order_by('-created_at')
class WebappDetail(DetailView):
......@@ -44,12 +152,36 @@ class WebappDetail(DetailView):
context_object_name = 'webapp'
def get_object(self):
"""recover data according the docker_name rather than the ID."""
data = self.kwargs.get('docker_name', None)
queryset = get_object_or_404(Webapp, docker_name=data)
return queryset
def get_context_data(self, **kwargs):
"""Recover the README file if exist and convert it from markdown to
context = super(WebappDetail, self).get_context_data(**kwargs)
# Check if a readme is declared in the database
if self.object.readme:
readme_file = os.path.join(
with open(readme_file, 'r') as md_data:
context['readme'] =
readme_file = None
return context
class JobList(LoginRequiredMixin, ListView):
Display the list of jobs for a given identified user
model = Job
pattern_name = 'job_list'
template_name = 'job_list.html'
......@@ -58,10 +190,51 @@ class JobList(LoginRequiredMixin, ListView):
redirect_field_name = 'redirect_to'
def get_queryset(self):
"""Filter jobs for a given user"""
queryset = Job.objects.filter(
return queryset
class UserSettingsView(LoginRequiredMixin, UpdateView):
""" Update of the user basic settings
first name, last name, ...
model = User
form_class = UserForm
template_name = 'user_update_form.html'
success_url = reverse_lazy('main:user_settings')
def get_object(self):
""" Get the data from the user according to its ID as he is identified
queryset = get_object_or_404(User,
return queryset
class UserPasswordUpdateView(LoginRequiredMixin, UpdateView):
""" Update the password.
We reuse the Django password form system in order to keep something robust
even if it dedicates a specific view for it.
success_url = reverse_lazy('main:user_settings')
form_class = PasswordChangeForm
template_name = "user_password_update_form.html"
def get_object(self, queryset=None):
return self.request.user
def get_form_kwargs(self):
kwargs = super(UserPasswordUpdateView, self).get_form_kwargs()
kwargs['user'] = kwargs.pop('instance')
return kwargs
def dispatch(self, request, *args, **kwargs):
return super(UserPasswordUpdateView, self) \
.dispatch(request, *args, **kwargs)
class UserUpdateView(LoginRequiredMixin, UpdateView):
fields = ['name', ]
......@@ -69,8 +242,8 @@ class UserUpdateView(LoginRequiredMixin, UpdateView):
def get_success_url(self):
return reverse(
kwargs={'username': self.request.user.username})
kwargs={'username': self.request.user.username})
def get_object(self):
# Only get the User record for the user making the request
Django settings for allgo project.
For more information on this file, see
For the full list of settings and their values, see
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
# Quick-start development settings - unsuitable for production
# See
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '1ww(nv**$*h1b3tw)51h5stjf1b=2*02s&2^!bll6sdm()-cam'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
# Application definition
ROOT_URLCONF = 'allgo.urls'
WSGI_APPLICATION = 'allgo.wsgi.application'
# Database
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
# Internationalization
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
STATIC_URL = '/static/'