Commit fe32dd23 authored by BERJON Matthieu's avatar BERJON Matthieu

Merge branch 'django-fix' into mberjon

Signed-off-by: BERJON Matthieu's avatarMatthieu Berjon <matthieu.berjon@inria.fr>
parents a7d7ca85 6d2cef61
*.sql
.DS_Store
*.pyc
.idea
.*.sw[op]
.stamp.*
.deps.*
......
......@@ -4,6 +4,12 @@ set -e
. prepare.sh
mkdir -p certs
if [ ! -f cert/server.key ] & [ ! -f certs/server.crt ]; then
openssl req -subj '/CN=localhost/O=Registry Demo/C=US' -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout certs/server.key -out certs/server.crt
fi
# build base image (if not present)
(set -x ; make base-debian)
......
FROM debian:stretch
# install system packages
# Installation of gunicorn through pip because the debian package
# python3-gunicorn doesn't install any binary
# Install Stretch backport
RUN echo "deb http://ftp.debian.org/debian stretch-backports main" \
>> /etc/apt/sources.list.d/stretch.list
......@@ -8,14 +12,15 @@ RUN apt-get update && apt-get -t stretch-backports install -y \
python3-django python3-django-allauth python3-misaka
RUN apt-get install -y mysql-server default-libmysqlclient-dev \
nginx-light zip python3-dev python3-pip python3-mysqldb \
nginx-light zip python3-dev python3-pip python3-mysqldb python-mysqldb python3-crypto \
supervisor && pip3 install gunicorn
COPY . /opt/allgo
COPY .env /opt/allgo
COPY init /dk/container_init
RUN pip3 install -r /opt/allgo/requirements.txt
RUN patch /etc/nginx/nginx.conf < /opt/allgo/setup/dk/nginx.patch
RUN rm /etc/nginx/sites-enabled/default && \
rm /etc/supervisor/supervisord.conf && \
......@@ -26,4 +31,5 @@ RUN rm /etc/nginx/sites-enabled/default && \
WORKDIR /opt/allgo
LABEL dk.migrate_always=1
ENV PYTHONUNBUFFERED 1
CMD run-allgo
......@@ -30,11 +30,14 @@ class AllgoUser(models.Model):
# Relationships
user = models.OneToOneField(
User, on_delete=models.CASCADE, related_name="allgouser")
User, on_delete=models.CASCADE, related_name="allgouser")
class Meta:
db_table = 'dj_users'
def getApp(self):
return [a.name for a in Webapp.objects.filter(user_id=self.user.id)]
class DockerOs(TimeStampModel):
"""
......@@ -107,9 +110,9 @@ class Webapp(TimeStampModel):
# A webapp has one docker os type
docker_os = models.OneToOneField(DockerOs, related_name="webappdockeros")
user = models.ForeignKey(
User, on_delete=models.CASCADE, related_name="webappuser")
User, on_delete=models.CASCADE, related_name="webappuser")
job_queue = models.OneToOneField(
JobQueue, on_delete=models.CASCADE, related_name="webappjobqueue")
JobQueue, on_delete=models.CASCADE, related_name="webappjobqueue")
class Meta:
db_table = 'dj_webapps'
......@@ -144,6 +147,7 @@ class WebappVersion(TimeStampModel):
docker_image_size = models.FloatField(blank=True, null=True)
state = models.IntegerField(null=True)
published = models.IntegerField()
url = models.TextField()
# Relationships
webapp = models.ForeignKey(Webapp, related_name="webappversion")
......@@ -165,9 +169,9 @@ class Quota(TimeStampModel):
# Relationships
dj_user = models.ForeignKey(
User, on_delete=models.CASCADE, related_name="quotauser")
User, on_delete=models.CASCADE, related_name="quotauser")
dj_webapp = models.ForeignKey(
Webapp, on_delete=models.CASCADE, related_name="quotawebapp")
Webapp, on_delete=models.CASCADE, related_name="quotawebapp")
class Meta:
db_table = 'dj_quotas'
......@@ -209,7 +213,7 @@ class JobUploads(TimeStampModel):
job_file_content_type = models.CharField(max_length=255, blank=True)
job_file_file_size = models.IntegerField(blank=True, null=True)
job_file_updated_at = models.DateTimeField(
blank=True, auto_now_add=True, null=True)
blank=True, auto_now_add=True, null=True)
# Relationships
dj_job = models.ForeignKey(Job, related_name="jobuploadjob")
......
import os
import time
import hashlib
import base64
import subprocess
from Crypto.PublicKey import RSA
from jose import jwt
SIGNING_KEY_PATH = os.environ.get('SIGNING_KEY_PATH')
SIGNING_KEY_TYPE = os.environ.get('SIGNING_KEY_TYPE')
SIGNING_KEY_ALG = os.environ.get('SIGNING_KEY_ALG')
SIGNING_KEY = open(SIGNING_KEY_PATH).read()
ISSUER = os.environ.get('ISSUER')
TOKEN_EXPIRATION = os.environ.get('TOKEN_EXPIRATION')
TOKEN_TYPE = os.environ.get('TOKEN_TYPE')
def run_command(command):
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
return process.communicate()
def key_id_encode(the_bytes):
source = base64.b32encode(the_bytes)
result = []
for i in range(0, len(source), 4):
start = i
end = start+4
result.append(str(source[start:end], 'utf-8'))
return ":".join(result)
def kid_from_crypto_key(private_key_path, key_type):
"""
python implementation of
https://github.com/jlhawn/libtrust/blob/master/util.go#L192
returns a distinct identifier which is unique to
the public key derived from this private key.
The format generated by this library is a base32 encoding of a 240 bit
hash of the public key data divided into 12 groups like so:
ABCD:EFGH:IJKL:MNOP:QRST:UVWX:YZ23:4567:ABCD:EFGH:IJKL:MNOP
"""
assert key_type == 'RSA', "Only RSA key type supported"
algorithm = hashlib.sha256()
key = RSA.importKey(open(private_key_path).read())
der = key.publickey().exportKey("DER")
algorithm.update(der)
return key_id_encode(algorithm.digest()[:30])
class Token(object):
def __init__(self, service, access_type="", access_name="",
access_actions=None, subject=''):
if access_actions is None:
access_actions = []
self.issuer = ISSUER
self.signing_key = SIGNING_KEY
self.signing_key_path = SIGNING_KEY_PATH
self.signing_key_type = SIGNING_KEY_TYPE
self.signing_key_alg = SIGNING_KEY_ALG
self.token_expiration = TOKEN_EXPIRATION
self.token_type = TOKEN_TYPE
self.header = {
'typ': self.token_type,
'alg': self.signing_key_alg,
'kid': kid_from_crypto_key(self.signing_key_path, self.signing_key_type)
}
self.claim = {
'iss': self.issuer,
'sub': subject,
'aud': service,
'exp': int(time.time()) + int(self.token_expiration),
'nbf': int(time.time()) - 30,
'iat': int(time.time()),
'access': [
{
'type': access_type,
'name': access_name,
'actions': access_actions
}
]
}
def set_header(self, header):
self.header = header
def get_header(self):
return self.header
def set_claim(self, claim):
self.claim = claim
def get_claim(self):
return self.claim
def encode_token(self):
token = jwt.encode(self.claim, self.signing_key,
algorithm=self.signing_key_alg,
headers=self.header)
return token
......@@ -5,20 +5,11 @@ app_name = 'main'
urlpatterns = [
url(r'^$', views.index, name="home"),
url(r'^tokens$', views.tokens, name="tokens"),
url(r'^registryhook', views.registryhook, name="registryhook"),
url(r'^apps/$', views.WebappList.as_view(), name='webapp_list'),
url(r'^app/(?P<docker_name>[\w-]+)/$',
views.WebappDetail.as_view(),
name='webapp_detail'),
url(r'^app/(?P<docker_name>[\w-]+)/$', views.WebappDetail.as_view(), name='webapp_detail'),
url(r'^jobs/$', views.JobList.as_view(), name='job_list'),
# Settings
url(
r'^settings/$',
views.UserSettingsView.as_view(),
name='user_settings'),
url(
# r'^settings/(?P<pk>[0-9]+)/$',
r'^settings/password$',
views.UserPasswordUpdateView.as_view(),
name='user_password_update'),
url(r'^settings/$', views.UserUpdateView.as_view(), name='user_settings'),
url(r'^settings/password$', views.UserPasswordUpdateView.as_view(), name='user_password_update'),
]
from django.conf import settings
from django.contrib.auth.forms import PasswordChangeForm
import base64
import json
import socket
import logging
import os
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.decorators.csrf import csrf_exempt
from django.contrib.auth.forms import PasswordChangeForm
from django.contrib.auth.models import User
from django.shortcuts import render, get_object_or_404
from django.urls import reverse_lazy
from django.conf import settings
from django.http import JsonResponse, HttpResponse
from django.urls import reverse, reverse_lazy
from django.views.generic import (
ListView,
DetailView,
UpdateView,
)
import os
ListView,
DetailView,
UpdateView,
)
from .models import Webapp, Job, User
from .tokens import Token
from .models import Webapp, Job, User, AllgoUser, WebappVersion
from .forms import UserForm, HomeSignupForm
log = logging.getLogger('allgo')
def get_allowed_actions(user, scope, actions):
allgouser = AllgoUser(user=user)
resource_type, resource_name, resource_actions = scope.split(":")
if resource_type == "repository" and resource_name.rstrip('-incoming') in allgouser.getApp():
return actions
else:
return []
def update_webapp_metadata(repository, url):
log.info("New webapp version received %s %s", repository, url)
webapp = Webapp.objects.filter(name=repository)[0]
WebappVersion(url=url, state=0, published=0, webapp=webapp).save()
notify_controler()
def notify_controler():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5)
sock.connect((os.environ.get('ALLGO_CONTROLLER_HOST'), int(os.environ.get('ALLGO_CONTROLLER_PORT'))))
sock.settimeout(None)
sock.makefile('rb', 0)
sock.shutdown(socket.SHUT_RDWR)
sock.close()
log.info("Controller notified")
def index(request):
"""
Do nothing specific just return a specific template
......@@ -31,6 +70,56 @@ def index(request):
return render(request, "home.html", context)
@csrf_exempt
def registryhook(request):
body_unicode = request.body.decode('utf-8')
body = json.loads(body_unicode)
for event in body['events']:
if event['action'] == "push":
update_webapp_metadata(event['target']['repository'], event['target']['url'])
return HttpResponse(status=200)
def tokens(request):
auth_header = request.META.get('HTTP_AUTHORIZATION', '')
if not auth_header:
log.info("Token request without http authorisation %s %s %s", request.META['HTTP_USER_AGENT'],
request.META['REMOTE_ADDR'], request.META['QUERY_STRING'])
return HttpResponse(status=401)
token_type, credentials = auth_header.split(' ')
username, password = base64.b64decode(credentials).decode('utf-8').split(':')
try:
user = User.objects.get(email=username)
except User.DoesNotExist:
log.warning("Token request but user doest not exist")
return HttpResponse(status=401)
password_valid = user.check_password(password)
if token_type != 'Basic' or not password_valid:
log.info("Token request but user password mismatch")
return HttpResponse(status=401)
service = request.GET['service']
scope = request.GET['scope'] if 'scope' in request.GET.keys() else None
if not scope:
typ = ''
name = ''
actions = []
else:
params = scope.split(':')
if len(params) != 3:
return JsonResponse({'error': 'Invalid scope parameter'}, status=400)
typ = params[0]
name = params[1]
actions = params[2].split(',')
authorized_actions = get_allowed_actions(user, scope, actions)
token = Token(service, typ, name, authorized_actions)
encoded_token = token.encode_token()
return JsonResponse({'token': encoded_token})
class WebappList(ListView):
"""
Display the paginated list of available webapps.
......@@ -142,3 +231,18 @@ class UserPasswordUpdateView(LoginRequiredMixin, UpdateView):
def dispatch(self, request, *args, **kwargs):
return super(UserPasswordUpdateView, self) \
.dispatch(request, *args, **kwargs)
class UserUpdateView(LoginRequiredMixin, UpdateView):
fields = ['name', ]
model = AllgoUser
def get_success_url(self):
return reverse(
'main:user_settings',
kwargs={'username': self.request.user.username})
def get_object(self):
# Only get the User record for the user making the request
return AllgoUser.objects.get(user__username=self.request.user.username)
import os
import sys
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
APPS_DIR = os.path.join(ROOT_DIR, 'allgo')
......@@ -17,7 +17,7 @@ SECRET_KEY = os.environ['ALLGO_SECRET_KEY']
DEBUG = os.environ.get('ALLGO_DEBUG', default=False)
ALLOWED_HOSTS = os.environ.get(
'ALLGO_ALLOWED_HOSTS',
default=['localhost', '127.0.0.1'],
default=['localhost', '127.0.0.1', '0.0.0.0'],
)
TIME_ZONE = 'UTC'
......@@ -230,3 +230,32 @@ SOCIALACCOUNT_PROVIDERS = {
'SCOPE': ['read_user'],
},
}
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}',
'style': '{',
},
'simple': {
'format': '{levelname} {message}',
'style': '{',
},
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'formatter': 'simple'
},
},
'loggers': {
'allgo': {
'handlers': ['console'],
'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'),
},
},
}
......@@ -2,3 +2,5 @@ Django==1.11
mysqlclient==1.3.12
django-allauth==0.35.0
misaka==2.1.0
python-jose==2.0.2
pyopenssl
......@@ -135,7 +135,6 @@ INSERT INTO `django_site`
VALUES (NULL, 'http://localhost:8000', 'A||go dev'),
(NULL, 'https://allgo.inria.fr', 'A||go');
--
-- Table structure for table `auth_group`
--
......@@ -284,7 +283,6 @@ Alter TABLE `dj_job_uploads`
ADD PRIMARY KEY (`id`),
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
--
-- Table structure for table `socialaccount_socialapp`
--
......@@ -404,4 +402,7 @@ ALTER TABLE `account_emailconfirmation`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,
ADD CONSTRAINT `account_emailconfirm_email_address_id_5b7f8c58_fk_account_e` FOREIGN KEY (`email_address_id`) REFERENCES `account_emailaddress` (`id`);
ALTER TABLE `dj_webapp_versions`
ADD COLUMN `url` text;
COMMIT;
......@@ -2,149 +2,172 @@
version: '2'
networks:
dev:
driver: bridge
sandboxes:
driver: bridge
services:
dev-registry:
container_name: dev-registry
build: registry
#image: allgo/registry
# DJANGO
######################################################################################################################
dev-django:
container_name: dev-django
build: django
user: "$DOCKERUSER"
ports:
- "127.0.0.1:8000-8002:8000-8002"
- "8008:8000"
command: "python3 manage.py runserver 0.0.0.0:8000"
volumes:
- "/data/dev/django:/vol"
- "./django:/opt/allgo"
- "./certs:/certs"
networks: [dev]
tty: true
stdin_open: true
environment:
PYTHONUNBUFFERED: 1
ALLGO_ALLOWED_HOSTS: 0.0.0.0,dev-django
DJANGO_DEBUG: 1
DJANGO_LOG_LEVEL: "DEBUG"
ALLGO_DEBUG: "True"
ALLGO_EMAIL_BACKEND: "django.core.mail.backends.console.EmailBackend"
ALLGO_SECRET_KEY: "nFgLEiedSJfYKyJA6WjkiGs8c23vokcVoM4DDLi9GsCX36TdsR"
ALLGO_DATABASE_PASSWORD: "allgo"
ALLGO_CONTROLLER_HOST: "dev-controller"
ALLGO_CONTROLLER_PORT: "4567"
SIGNING_KEY_PATH: "/certs/server.key"
SIGNING_KEY_TYPE: "RSA"
SIGNING_KEY_ALG: "RS256"
ISSUER: "allgo_oauth"
TOKEN_EXPIRATION: "3600"
TOKEN_TYPE: "JWT"
# REGISTRY
######################################################################################################################
dev-registry:
container_name: dev-registry
image: registry:2
ports:
- "5000:5000"
volumes:
- "/data/dev/registry:/vol"
- "./certs:/certs"
environment:
REGISTRY_LOG_LEVEL: "debug"
REGISTRY_HTTP_TLS_CERTIFICATE: "/certs/server.crt"
REGISTRY_HTTP_TLS_KEY: "/certs/server.key"
REGISTRY_AUTH: "token"
REGISTRY_AUTH_TOKEN_REALM: "http://0.0.0.0:8008/tokens"
REGISTRY_AUTH_TOKEN_SERVICE: "allgo_registry"
REGISTRY_AUTH_TOKEN_ISSUER: "allgo_oauth"
REGISTRY_AUTH_TOKEN_ROOTCERTBUNDLE: "/certs/server.crt"
REGISTRY_NOTIFICATIONS_ENDPOINTS: "- name: notifications-test\n url: http://dev-django:8000/registryhook\n timeout: 5s\n threshold: 8\n backoff: 10s"
networks: [dev]
# CONTROLLER
######################################################################################################################
dev-controller:
container_name: dev-controller
build: controller
#image: allgo/controller
volumes:
- "/data/dev/controller:/vol"
- "./controller:/opt/allgo-docker"
- "/:/vol/host:ro"
environment:
ENV: "dev"
REGISTRY: "localhost:8002/allgo/dev"
REGISTRY: "localhost:5000/allgo/dev"
DEBUG: "1"
networks: [dev]
# override default command (to allow running the controller manually with ./shell)
#command: ["/bin/bash"]
#tty: true
#stdin_open: true
networks: [dev]
# MYSQL
######################################################################################################################
dev-mysql:
container_name: dev-mysql
build: mysql
#image: allgo/mysql
user: "$DOCKERUSER"
ports:
- "3306:3306"
volumes:
- "/data/dev/mysql:/vol"
networks: [dev]
# SSH
######################################################################################################################
dev-ssh:
container_name: dev-ssh
build: ssh
#image: allgo/ssh
ports:
- "127.0.0.1:2222:22"
volumes:
- "/data/dev/ssh:/vol"
- "./ssh:/opt/allgo-ssh"
environment:
ENV: "dev"
networks: [dev, sandboxes]
# RAILS
######################################################################################################################
dev-rails:
container_name: dev-rails
build: rails
#image: allgo/rails
user: "$DOCKERUSER"
ports:
- "127.0.0.1:3000:8080"
volumes:
- "/data/dev/rails:/vol"
- "./rails:/opt/allgo"
environment:
RAILS_ENV: development
networks: [dev]
tty: true
stdin_open: true
dev-django:
container_name: dev-django
build: django
#image: allgo/rails
user: "$DOCKERUSER"
ports:
- "127.0.0.1:4000:8080"
volumes:
- "/data/dev/django:/vol"
- "./django:/opt/allgo"
networks: [dev]
tty: true
stdin_open: true