Commit e8aa7dd8 authored by BAIRE Anthony's avatar BAIRE Anthony

Merge remote-tracking branch 'refs/remotes/origin/django' into django

# Conflicts:
#	django/Dockerfile
#	docker-compose.yml
parents be96a87a d5c1cd43
......@@ -4,6 +4,12 @@ set -e
. prepare.sh
mkdir -p certs
if [ ! -f cert/server.key ] & [ ! -f certs/server.crt ]; then
openssl req -subj '/CN=localhost/O=Registry Demo/C=US' -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout certs/server.key -out certs/server.crt
fi
# build base image (if not present)
(set -x ; make base-debian)
......
......@@ -8,9 +8,9 @@ FROM allgo/base-debian
COPY setup/backports/. /
RUN apt-getq update && apt-getq install mysql-server default-libmysqlclient-dev \
python3-django python3-django-allauth \
python3-django python3-django-allauth python3-misaka \
nginx-light zip gcc python3-dev python3-pip python3-wheel python3-mysqldb \
python-mysqldb supervisor\
python-mysqldb python3-crypto supervisor\
&& pip3 install gunicorn
COPY requirements.txt /tmp/
......@@ -31,4 +31,5 @@ RUN apply-patches /opt/allgo/setup/patches/*.diff &&\
WORKDIR /opt/allgo
LABEL dk.migrate_always=1
ENV PYTHONUNBUFFERED 1
CMD run-allgo
from allauth.account.forms import SignupForm
from django import forms
from .models import User
class UserForm(forms.ModelForm):
first_name = forms.CharField(
label='First name',
label_suffix='',
initial="hello",
required=False,
)
last_name = forms.CharField(label='Last name', label_suffix='')
class Meta:
model = User
fields = ('first_name', 'last_name')
class HomeSignupForm(SignupForm):
def __init__(self, *args, **kwargs):
super(HomeSignupForm, self).__init__(*args, **kwargs)
......@@ -30,12 +30,14 @@ class AllgoUser(models.Model):
# Relationships
user = models.OneToOneField(
User, on_delete=models.CASCADE, related_name="allgouser")
User, on_delete=models.CASCADE, related_name="allgouser")
class Meta:
db_table = 'dj_users'
def getApp(self):
return [a.name for a in Webapp.objects.filter(user_id=self.user.id)]
class DockerOs(TimeStampModel):
"""
......@@ -108,9 +110,9 @@ class Webapp(TimeStampModel):
# A webapp has one docker os type
docker_os = models.OneToOneField(DockerOs, related_name="webappdockeros")
user = models.ForeignKey(
User, on_delete=models.CASCADE, related_name="webappuser")
User, on_delete=models.CASCADE, related_name="webappuser")
job_queue = models.OneToOneField(
JobQueue, on_delete=models.CASCADE, related_name="webappjobqueue")
JobQueue, on_delete=models.CASCADE, related_name="webappjobqueue")
class Meta:
db_table = 'dj_webapps'
......@@ -145,9 +147,10 @@ class WebappVersion(TimeStampModel):
docker_image_size = models.FloatField(blank=True, null=True)
state = models.IntegerField(null=True)
published = models.IntegerField()
url = models.TextField()
# Relationships
dj_webapp = models.ForeignKey(Webapp, related_name="webappversion")
webapp = models.ForeignKey(Webapp, related_name="webappversion")
class Meta:
db_table = 'dj_webapp_versions'
......@@ -166,9 +169,9 @@ class Quota(TimeStampModel):
# Relationships
dj_user = models.ForeignKey(
User, on_delete=models.CASCADE, related_name="quotauser")
User, on_delete=models.CASCADE, related_name="quotauser")
dj_webapp = models.ForeignKey(
Webapp, on_delete=models.CASCADE, related_name="quotawebapp")
Webapp, on_delete=models.CASCADE, related_name="quotawebapp")
class Meta:
db_table = 'dj_quotas'
......@@ -210,7 +213,7 @@ class JobUploads(TimeStampModel):
job_file_content_type = models.CharField(max_length=255, blank=True)
job_file_file_size = models.IntegerField(blank=True, null=True)
job_file_updated_at = models.DateTimeField(
blank=True, auto_now_add=True, null=True)
blank=True, auto_now_add=True, null=True)
# Relationships
dj_job = models.ForeignKey(Job, related_name="jobuploadjob")
......
from django import template
from misaka import Markdown, HtmlRenderer
register = template.Library()
@register.filter(name='markdown')
def markdown(value):
""" Convert markdown content into HTML. """
renderer = HtmlRenderer()
markdown = Markdown(
renderer,
extensions=(
'fenced-code',
'footnotes',
'superscript',
'quote',
))
return markdown(value)
from copy import copy
from django import template
import types
register = template.Library()
def silence_without_field(fn):
def wrapped(field, attr):
if not field:
return ""
return fn(field, attr)
return wrapped
def _process_field_attributes(field, attr, process):
# split attribute name and value from 'attr:value' string
params = attr.split(':', 1)
attribute = params[0]
value = params[1] if len(params) == 2 else True
field = copy(field)
# decorate field.as_widget method with updated attributes
old_as_widget = field.as_widget
def as_widget(self, widget=None, attrs=None, only_initial=False):
attrs = attrs or {}
process(widget or self.field.widget, attrs, attribute, value)
html = old_as_widget(widget, attrs, only_initial)
self.as_widget = old_as_widget
return html
field.as_widget = types.MethodType(as_widget, field)
return field
@register.filter("attr")
@silence_without_field
def set_attr(field, attr):
def process(widget, attrs, attribute, value):
attrs[attribute] = value
return _process_field_attributes(field, attr, process)
@register.filter("append_attr")
@silence_without_field
def append_attr(field, attr):
def process(widget, attrs, attribute, value):
if attrs.get(attribute):
attrs[attribute] += ' ' + value
elif widget.attrs.get(attribute):
attrs[attribute] = widget.attrs[attribute] + ' ' + value
else:
attrs[attribute] = value
return _process_field_attributes(field, attr, process)
@register.filter("add_class")
@silence_without_field
def add_class(field, css_class):
return append_attr(field, 'class:' + css_class)
import os
import time
import hashlib
import base64
import subprocess
from Crypto.PublicKey import RSA
from jose import jwt
SIGNING_KEY_PATH = os.environ.get('SIGNING_KEY_PATH')
SIGNING_KEY_TYPE = os.environ.get('SIGNING_KEY_TYPE')
SIGNING_KEY_ALG = os.environ.get('SIGNING_KEY_ALG')
SIGNING_KEY = open(SIGNING_KEY_PATH).read()
ISSUER = os.environ.get('ISSUER')
TOKEN_EXPIRATION = os.environ.get('TOKEN_EXPIRATION')
TOKEN_TYPE = os.environ.get('TOKEN_TYPE')
def run_command(command):
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
return process.communicate()
def key_id_encode(the_bytes):
source = base64.b32encode(the_bytes)
result = []
for i in range(0, len(source), 4):
start = i
end = start+4
result.append(str(source[start:end], 'utf-8'))
return ":".join(result)
def kid_from_crypto_key(private_key_path, key_type):
"""
python implementation of
https://github.com/jlhawn/libtrust/blob/master/util.go#L192
returns a distinct identifier which is unique to
the public key derived from this private key.
The format generated by this library is a base32 encoding of a 240 bit
hash of the public key data divided into 12 groups like so:
ABCD:EFGH:IJKL:MNOP:QRST:UVWX:YZ23:4567:ABCD:EFGH:IJKL:MNOP
"""
assert key_type == 'RSA', "Only RSA key type supported"
algorithm = hashlib.sha256()
key = RSA.importKey(open(private_key_path).read())
der = key.publickey().exportKey("DER")
algorithm.update(der)
return key_id_encode(algorithm.digest()[:30])
class Token(object):
def __init__(self, service, access_type="", access_name="",
access_actions=None, subject=''):
if access_actions is None:
access_actions = []
self.issuer = ISSUER
self.signing_key = SIGNING_KEY
self.signing_key_path = SIGNING_KEY_PATH
self.signing_key_type = SIGNING_KEY_TYPE
self.signing_key_alg = SIGNING_KEY_ALG
self.token_expiration = TOKEN_EXPIRATION
self.token_type = TOKEN_TYPE
self.header = {
'typ': self.token_type,
'alg': self.signing_key_alg,
'kid': kid_from_crypto_key(self.signing_key_path, self.signing_key_type)
}
self.claim = {
'iss': self.issuer,
'sub': subject,
'aud': service,
'exp': int(time.time()) + int(self.token_expiration),
'nbf': int(time.time()) - 30,
'iat': int(time.time()),
'access': [
{
'type': access_type,
'name': access_name,
'actions': access_actions
}
]
}
def set_header(self, header):
self.header = header
def get_header(self):
return self.header
def set_claim(self, claim):
self.claim = claim
def get_claim(self):
return self.claim
def encode_token(self):
token = jwt.encode(self.claim, self.signing_key,
algorithm=self.signing_key_alg,
headers=self.header)
return token
......@@ -5,15 +5,11 @@ app_name = 'main'
urlpatterns = [
url(r'^$', views.index, name="home"),
url(r'^tokens$', views.tokens, name="tokens"),
url(r'^registryhook', views.registryhook, name="registryhook"),
url(r'^apps/$', views.WebappList.as_view(), name='webapp_list'),
url(r'^app/(?P<docker_name>[\w-]+)/$',
views.WebappDetail.as_view(),
name='webapp_detail'),
url(r'^app/(?P<docker_name>[\w-]+)/$', views.WebappDetail.as_view(), name='webapp_detail'),
url(r'^jobs/$', views.JobList.as_view(), name='job_list'),
# Settings
url(
r'^settings/$',
views.UserUpdateView.as_view(),
name='user_settings'),
url(r'^settings/$', views.UserUpdateView.as_view(), name='user_settings'),
url(r'^settings/password$', views.UserPasswordUpdateView.as_view(), name='user_password_update'),
]
import base64
import json
import socket
import logging
import os
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.decorators.csrf import csrf_exempt
from django.contrib.auth.forms import PasswordChangeForm
from django.contrib.auth.models import User
from django.shortcuts import render, get_object_or_404
from django.urls import reverse
from django.conf import settings
from django.http import JsonResponse, HttpResponse
from django.urls import reverse, reverse_lazy
from django.views.generic import (
ListView,
DetailView,
UpdateView,
)
ListView,
DetailView,
UpdateView,
)
from .tokens import Token
from .models import Webapp, Job, User, AllgoUser, WebappVersion
from .forms import UserForm, HomeSignupForm
log = logging.getLogger('allgo')
def get_allowed_actions(user, scope, actions):
allgouser = AllgoUser(user=user)
if not scope:
return []
resource_type, resource_name, resource_actions = scope.split(":")
if resource_type == "repository" and resource_name.rstrip('-incoming') in allgouser.getApp():
return actions
else:
return []
def update_webapp_metadata(repository, url):
log.info("New webapp version received %s %s", repository, url)
webapp = Webapp.objects.filter(name=repository)[0]
WebappVersion(url=url, state=0, published=0, webapp=webapp).save()
notify_controler()
from .models import Webapp, Job, AllgoUser
def notify_controler():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5)
sock.connect((os.environ.get('ALLGO_CONTROLLER_HOST'), int(os.environ.get('ALLGO_CONTROLLER_PORT'))))
sock.settimeout(None)
sock.makefile('rb', 0)
sock.shutdown(socket.SHUT_RDWR)
sock.close()
log.info("Controller notified")
def index(request):
return render(request, "home.html")
"""
Do nothing specific just return a specific template
"""
users = User.objects.all().count()
webapps = Webapp.objects.all().count()
jobs = Job.objects.all().count()
context = {
'signup_form': HomeSignupForm(),
'user_nb': users,
'webapp_nb': webapps,
'job_nb': jobs,
}
return render(request, "home.html", context)
@csrf_exempt
def registryhook(request):
body_unicode = request.body.decode('utf-8')
body = json.loads(body_unicode)
for event in body['events']:
if event['action'] == "push":
update_webapp_metadata(event['target']['repository'], event['target']['url'])
return HttpResponse(status=200)
def tokens(request):
auth_header = request.META.get('HTTP_AUTHORIZATION', '')
if not auth_header:
log.info("Token request without http authorisation %s %s %s", request.META['HTTP_USER_AGENT'],
request.META['REMOTE_ADDR'], request.META['QUERY_STRING'])
return HttpResponse(status=401)
token_type, credentials = auth_header.split(' ')
username, password = base64.b64decode(credentials).decode('utf-8').split(':')
try:
user = User.objects.get(email=username)
except User.DoesNotExist:
log.warning("Token request but user doest not exist")
return HttpResponse(status=401)
password_valid = user.check_password(password)
if token_type != 'Basic' or not password_valid:
log.info("Token request but user password mismatch")
return HttpResponse(status=401)
service = request.GET['service']
scope = request.GET['scope'] if 'scope' in request.GET.keys() else None
if not scope:
typ = ''
name = ''
actions = []
else:
params = scope.split(':')
if len(params) != 3:
return JsonResponse({'error': 'Invalid scope parameter'}, status=400)
typ = params[0]
name = params[1]
actions = params[2].split(',')
authorized_actions = get_allowed_actions(user, scope, actions)
token = Token(service, typ, name, authorized_actions)
encoded_token = token.encode_token()
return JsonResponse({'token': encoded_token})
class WebappList(ListView):
......@@ -31,7 +139,7 @@ class WebappList(ListView):
pattern_name = 'webapp_list'
paginate_by = 10
template_name = 'webapp_list.html'
queryset = Webapp.objects.filter(private=0).order_by('created_at')
queryset = Webapp.objects.filter(private=0).order_by('-created_at')
class WebappDetail(DetailView):
......@@ -44,12 +152,36 @@ class WebappDetail(DetailView):
context_object_name = 'webapp'
def get_object(self):
"""recover data according the docker_name rather than the ID."""
data = self.kwargs.get('docker_name', None)
queryset = get_object_or_404(Webapp, docker_name=data)
return queryset
def get_context_data(self, **kwargs):
"""Recover the README file if exist and convert it from markdown to
HTML
"""
context = super(WebappDetail, self).get_context_data(**kwargs)
# Check if a readme is declared in the database
if self.object.readme:
readme_file = os.path.join(
settings.MEDIA_ROOT,
self.object.docker_name,
'Readme')
with open(readme_file, 'r') as md_data:
print(md_data)
context['readme'] = md_data.read()
else:
readme_file = None
return context
class JobList(LoginRequiredMixin, ListView):
"""
Display the list of jobs for a given identified user
"""
model = Job
pattern_name = 'job_list'
template_name = 'job_list.html'
......@@ -58,10 +190,51 @@ class JobList(LoginRequiredMixin, ListView):
redirect_field_name = 'redirect_to'
def get_queryset(self):
"""Filter jobs for a given user"""
queryset = Job.objects.filter(user_id=self.request.user.id)
return queryset
class UserSettingsView(LoginRequiredMixin, UpdateView):
""" Update of the user basic settings
first name, last name, ...
"""
model = User
form_class = UserForm
template_name = 'user_update_form.html'
success_url = reverse_lazy('main:user_settings')
def get_object(self):
""" Get the data from the user according to its ID as he is identified
"""
queryset = get_object_or_404(User, pk=self.request.user.id)
return queryset
class UserPasswordUpdateView(LoginRequiredMixin, UpdateView):
""" Update the password.
We reuse the Django password form system in order to keep something robust
even if it dedicates a specific view for it.
"""
success_url = reverse_lazy('main:user_settings')
form_class = PasswordChangeForm
template_name = "user_password_update_form.html"
def get_object(self, queryset=None):
return self.request.user
def get_form_kwargs(self):
kwargs = super(UserPasswordUpdateView, self).get_form_kwargs()
kwargs['user'] = kwargs.pop('instance')
return kwargs
def dispatch(self, request, *args, **kwargs):
return super(UserPasswordUpdateView, self) \
.dispatch(request, *args, **kwargs)
class UserUpdateView(LoginRequiredMixin, UpdateView):
fields = ['name', ]
......@@ -69,8 +242,8 @@ class UserUpdateView(LoginRequiredMixin, UpdateView):
def get_success_url(self):
return reverse(
'main:user_settings',
kwargs={'username': self.request.user.username})
'main:user_settings',
kwargs={'username': self.request.user.username})
def get_object(self):
# Only get the User record for the user making the request
......
"""
Django settings for allgo project.
For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '1ww(nv**$*h1b3tw)51h5stjf1b=2*02s&2^!bll6sdm()-cam'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
TEMPLATE_DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = (
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'main',
)
MIDDLEWARE_CLASSES = (
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
)
ROOT_URLCONF = 'allgo.urls'
WSGI_APPLICATION = 'allgo.wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
}
}
# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/
STATIC_URL = '/static/'