views.py 13.8 KB
Newer Older
CAMPION Sebastien's avatar
CAMPION Sebastien committed
1
import base64
2
import hashlib
3 4 5
import json
import logging
import os
6
import socket
7
import tarfile
8

9
import redis
10
from django.conf import settings
CAMPION Sebastien's avatar
CAMPION Sebastien committed
11
from django.contrib.auth.decorators import login_required
12 13
from django.contrib.auth.forms import PasswordChangeForm
from django.contrib.auth.mixins import LoginRequiredMixin
CAMPION Sebastien's avatar
CAMPION Sebastien committed
14
from django.core.urlresolvers import reverse
15
from django.http import JsonResponse, HttpResponse, FileResponse
16 17
from django.shortcuts import redirect
from django.shortcuts import render, get_object_or_404
18
from django.urls import reverse, reverse_lazy
19
from django.views.decorators.csrf import csrf_exempt
CAMPION Sebastien's avatar
CAMPION Sebastien committed
20
from django.views.generic import (
CAMPION Sebastien's avatar
CAMPION Sebastien committed
21 22 23 24
    ListView,
    DetailView,
    UpdateView,
)
CAMPION Sebastien's avatar
CAMPION Sebastien committed
25

26
from .forms import UserForm, HomeSignupForm
27
from .models import User
28
from .models import Webapp, Job, AllgoUser, WebappVersion, Runner
CAMPION Sebastien's avatar
bugfix  
CAMPION Sebastien committed
29
from .tokens import Token
CAMPION Sebastien's avatar
CAMPION Sebastien committed
30

31
log = logging.getLogger('allgo')
32

33
BUF_SIZE = 65536
34

35 36 37 38 39 40 41 42 43 44 45

def sha1file(filepath):
    sha1 = hashlib.sha1()
    with open(filepath, 'rb') as f:
        while True:
            data = f.read(BUF_SIZE)
            if not data:
                break
            sha1.update(data)
    return "sha1:%s" % sha1.hexdigest()

46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64

def check_token_and_jobid(func):  # Check token decorator
    def wrapper(*args, **kwargs):
        request = args[0]
        jobid = args[1]
        username, runner = get_token_cred(request)
        if username != "$token" or not runner:
            log.error("Runner request $token user or a valid token")
            return HttpResponse(status=401)
        job = Job.objects.get(runner=runner, id=jobid)
        if not job:
            log.error("No job found for id %s and runner %s", jobid, runner)
            return HttpResponse(status=401)
        func(*args, **kwargs)

    return wrapper


@check_token_and_jobid
65
@csrf_exempt
66 67 68 69 70
def runner_dw(request, jobid, filename):
    datastore = os.environ.get("ALLGO_DATASTORE")
    filepath = os.path.join(datastore, jobid, filename)
    assert ".." not in filepath, "filepath unsecure"
    return FileResponse(open(filepath, 'rb'))
71

72

73 74 75
@check_token_and_jobid
@csrf_exempt
def runner_up(request, jobid, digest, nbofchunk, chunkid):
76
    datastore = os.environ.get("ALLGO_DATASTORE")
77 78 79 80
    outputdir = os.path.join(datastore, jobid, ".%s" % digest)
    if not os.path.exists(outputdir):
        os.mkdir(outputdir)
    filepath = os.path.join(outputdir, chunkid)
81 82 83 84
    assert ".." not in filepath, "filepath unsecure"
    with open(filepath, 'wb') as fp:
        fp.write(request.body)

85 86 87 88 89 90 91 92 93 94
    if upload_is_finish(request, jobid, digest, nbofchunk, datastore, outputdir):
        return JsonResponse({'status': 'success'}, status=200)
    else:
        return HttpResponse(status=200)


def upload_is_finish(request, jobid, digest, nbofchunk, datastore, outputdir):
    if int(nbofchunk) == len(os.listdir(outputdir)):  # we receive all chunk
        username, runner = get_token_cred(request)
        job = Job.objects.get(runner=runner, id=jobid)
95 96 97 98
        tarfilepath = os.path.join(datastore, jobid, ".%s.tar" % digest)
        concatenate_chunks_in_onefiletar(outputdir, tarfilepath)
        if check_tar_digest_and_extract(datastore, digest, jobid, tarfilepath):
            log.info("Results successfully uploaded for jobid  %s", jobid)
CAMPION Sebastien's avatar
CAMPION Sebastien committed
99
            job.state = 3
100
            job.save()
101
            return True
102 103 104 105 106 107 108 109 110 111


def check_tar_digest_and_extract(datastore, digest, jobid, tarfilepath):
    if sha1file(tarfilepath).split(":")[1] == digest:
        with tarfile.open(tarfilepath, "r") as tar:
            tar.extractall(os.path.join(datastore, jobid))
        return True


def concatenate_chunks_in_onefiletar(odir, tarfilepath):
112 113 114 115
    with open(tarfilepath, 'wb') as fp:
        for chunk in range(len(os.listdir(odir))):
            with open(os.path.join(odir, str(chunk)), 'rb') as c:
                fp.write(c.read())
116

117

118
@check_token_and_jobid
119 120
@csrf_exempt
def runner_log(request, jobid):
121 122
    redis_host = getattr(settings, "ALLGO_DJANGO_REDIS_HOST")
    r = redis.StrictRedis(host=redis_host, port=6379, db=0)
123
    username, runner = get_token_cred(request)
124
    job = Job.objects.get(runner=runner, id=jobid)
CAMPION Sebastien's avatar
CAMPION Sebastien committed
125 126 127
    job.state = 2
    job.save()

128 129 130 131 132 133 134 135 136 137 138 139
    log.info("Job logger called for job %s", jobid)
    log.info(request)
    stream = request.META['wsgi.input']
    line = None
    linenumber = 0
    while True:
        c = stream.read(1)
        if not c:
            break
        if c == b'\n':
            k = "job:log:%s:%s" % (jobid, linenumber)
            log.debug("redis %s %s", k, line)
140
            r.setex(k, 60 * 60 * 24, line)  # keep job log in cache 24 hours
141 142 143 144 145 146
            linenumber += 1
            line = None
        else:
            line = c if not line else line + c
    log.info("Log finished ")
    return JsonResponse({'status': 'ok'}, status=200)
147 148 149 150 151 152 153 154 155 156 157 158 159 160


def get_token_cred(request):
    auth_header = request.META.get('HTTP_AUTHORIZATION', '')
    if not auth_header:
        log.info("Runner request without http authorisation %s %s %s", request.META['HTTP_USER_AGENT'],
                 request.META['REMOTE_ADDR'], request.META['QUERY_STRING'])
        return HttpResponse(status=401)
    token_type, credentials = auth_header.split(' ')
    username, password = base64.b64decode(credentials).decode('utf-8').split(':')
    return username, Runner.objects.get(token=password)


def runner_jobs(runner):
161 162 163 164 165
    job = Job.objects.filter(state=0, runner=runner.id).first()
    if job:
        log.debug("Send job %s to runner %s", job.id, runner.token)
        datastore = os.environ.get("ALLGO_DATASTORE")
        jdir = os.path.join(datastore, str(job.id))
166
        files = {f: sha1file(os.path.join(jdir, f)) for f in os.listdir(jdir) if os.path.isfile(os.path.join(jdir, f))}
167 168 169 170
        job.state = 1
        job.save()
        return json.dumps({"jobid": job.id,
                           "registry": getattr(settings, "ALLGO_DJANGO_REGISTRY"),
171 172
                           "image": job.webapp.name.lower(),
                           "chunksize": getattr(settings, "ALLGO_DJANGO_MAXUPLOADSIZE"),
173 174 175
                           "files": files})
    else:
        return ""
176

177

178
@csrf_exempt
179 180 181 182 183
def runner_cmd(request):
    username, runner = get_token_cred(request)
    if username != "$token" or not runner:
        log.warning("Runner request $token user or a valid token")
        return HttpResponse(status=401)
184
    return HttpResponse(runner_jobs(runner))
185

186 187 188

def get_allowed_actions(user, scope, actions):
    resource_type, resource_name, resource_actions = scope.split(":")
189 190 191
    if isinstance(user, Runner):
        if resource_type == "repository" and resource_name in [w.name for w in user.webapps]:
            return ['pull']
192
    else:
193 194 195
        allgouser = AllgoUser(user=user)
        if resource_type == "repository" and resource_name.rstrip('-incoming') in allgouser.getApp():
            return actions
196

197

198 199 200 201 202 203 204 205
def update_webapp_metadata(repository, url):
    log.info("New webapp version received %s %s", repository, url)
    webapp = Webapp.objects.filter(name=repository)[0]
    WebappVersion(url=url, state=0, published=0, webapp=webapp).save()
    notify_controler()


def notify_controler():
206 207 208 209 210 211 212 213 214 215
    try:
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.settimeout(5)
        sock.connect((os.environ.get('ALLGO_CONTROLLER_HOST'), int(os.environ.get('ALLGO_CONTROLLER_PORT'))))
        sock.settimeout(None)
        sock.makefile('rb', 0)
        sock.shutdown(socket.SHUT_RDWR)
        sock.close()
        log.info("Controller notified")
    except Exception as e:
216 217
        log.error("Controller notification failed !!! %s", str(e))

CAMPION Sebastien's avatar
CAMPION Sebastien committed
218

CAMPION Sebastien's avatar
CAMPION Sebastien committed
219
def index(request):
220 221 222
    """
    Do nothing specific just return a specific template
    """
BERJON Matthieu's avatar
BERJON Matthieu committed
223 224 225
    users = User.objects.all().count()
    webapps = Webapp.objects.all().count()
    jobs = Job.objects.all().count()
226
    context = {
227 228 229 230
        'signup_form': HomeSignupForm(),
        'user_nb': users,
        'webapp_nb': webapps,
        'job_nb': jobs,
231 232 233
    }

    return render(request, "home.html", context)
CAMPION Sebastien's avatar
CAMPION Sebastien committed
234

CAMPION Sebastien's avatar
CAMPION Sebastien committed
235

236 237 238 239 240 241 242 243 244
@csrf_exempt
def registryhook(request):
    body_unicode = request.body.decode('utf-8')
    body = json.loads(body_unicode)
    for event in body['events']:
        if event['action'] == "push":
            update_webapp_metadata(event['target']['repository'], event['target']['url'])
    return HttpResponse(status=200)

CAMPION Sebastien's avatar
CAMPION Sebastien committed
245

CAMPION Sebastien's avatar
CAMPION Sebastien committed
246
@login_required
CAMPION Sebastien's avatar
CAMPION Sebastien committed
247
def jupyter(request):
CAMPION Sebastien's avatar
CAMPION Sebastien committed
248 249 250 251 252
    token = Token("jupyter")
    user = request.user.get_username()
    token.claim['upn'] = user
    encoded_token = token.encode_token()
    next = "/user/%s/git-pull?repo=%s" % (user, request.GET.get("repo"))
253
    return redirect(os.environ.get('ALLGO_JUPYTER_URL') + "?bearer=" + encoded_token + "&next=" + next)
CAMPION Sebastien's avatar
CAMPION Sebastien committed
254 255


256
def tokens(request):
CAMPION Sebastien's avatar
CAMPION Sebastien committed
257
    auth_header = request.META.get('HTTP_AUTHORIZATION', '')
258 259 260 261 262
    if not auth_header:
        log.info("Token request without http authorisation %s %s %s", request.META['HTTP_USER_AGENT'],
                 request.META['REMOTE_ADDR'], request.META['QUERY_STRING'])
        return HttpResponse(status=401)
    token_type, credentials = auth_header.split(' ')
CAMPION Sebastien's avatar
CAMPION Sebastien committed
263
    username, password = base64.b64decode(credentials).decode('utf-8').split(':')
264 265 266 267 268 269 270 271 272 273 274 275 276
    if username == "$token" and Runner.objects.get(token=password):
        log.info("Token for runner called")
        user = Runner.objects.get(token=password)
    else:
        try:
            user = User.objects.get(email=username)
        except User.DoesNotExist:
            log.warning("Token request but user doest not exist")
            return HttpResponse(status=401)
        password_valid = user.check_password(password)
        if token_type != 'Basic' or not password_valid:
            log.info("Token request but user password mismatch")
            return HttpResponse(status=401)
CAMPION Sebastien's avatar
CAMPION Sebastien committed
277 278

    service = request.GET['service']
CAMPION Sebastien's avatar
bugfix  
CAMPION Sebastien committed
279
    scope = request.GET['scope'] if 'scope' in request.GET.keys() else None
280 281 282 283 284 285 286 287 288 289 290 291
    if not scope:
        typ = ''
        name = ''
        actions = []
    else:
        params = scope.split(':')
        if len(params) != 3:
            return JsonResponse({'error': 'Invalid scope parameter'}, status=400)
        typ = params[0]
        name = params[1]
        actions = params[2].split(',')

292 293
    authorized_actions = get_allowed_actions(user, scope, actions) if scope else []
    log.info("Token authorized actions %s %s %s", authorized_actions, user, scope)
294 295 296
    token = Token(service, typ, name, authorized_actions)
    encoded_token = token.encode_token()

CAMPION Sebastien's avatar
CAMPION Sebastien committed
297
    return JsonResponse({'token': encoded_token})
298

CAMPION Sebastien's avatar
CAMPION Sebastien committed
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316

class WebappList(ListView):
    """
    Display the paginated list of available webapps.

    **Context**

    ``Webapp``
        An instance of :model:`main.Webapp`

    **Template:**
    :template:`templates/webapp_list.html`

    """
    model = Webapp
    pattern_name = 'webapp_list'
    paginate_by = 10
    template_name = 'webapp_list.html'
317
    queryset = Webapp.objects.filter(private=0).order_by('-created_at')
CAMPION Sebastien's avatar
CAMPION Sebastien committed
318 319 320 321 322 323 324 325 326 327 328 329


class WebappDetail(DetailView):
    """
    Display the details of an app based on its docker name

    """
    model = Webapp
    template_name = 'webapp_detail.html'
    context_object_name = 'webapp'

    def get_object(self):
330
        """recover data according the docker_name rather than the ID."""
CAMPION Sebastien's avatar
CAMPION Sebastien committed
331 332 333 334
        data = self.kwargs.get('docker_name', None)
        queryset = get_object_or_404(Webapp, docker_name=data)
        return queryset

335 336 337 338 339 340 341 342 343
    def get_context_data(self, **kwargs):
        """Recover the README file if exist and convert it from markdown to
        HTML
        """
        context = super(WebappDetail, self).get_context_data(**kwargs)

        # Check if a readme is declared in the database
        if self.object.readme:
            readme_file = os.path.join(
344 345 346
                settings.MEDIA_ROOT,
                self.object.docker_name,
                'Readme')
347 348 349 350 351 352 353 354
            with open(readme_file, 'r') as md_data:
                print(md_data)
                context['readme'] = md_data.read()
        else:
            readme_file = None

        return context

CAMPION Sebastien's avatar
CAMPION Sebastien committed
355 356

class JobList(LoginRequiredMixin, ListView):
357 358 359
    """
    Display the list of jobs for a given identified user
    """
CAMPION Sebastien's avatar
CAMPION Sebastien committed
360 361 362 363 364 365 366 367
    model = Job
    pattern_name = 'job_list'
    template_name = 'job_list.html'
    paginate_by = 10
    login_url = '/auth/login'
    redirect_field_name = 'redirect_to'

    def get_queryset(self):
368
        """Filter jobs for a given user"""
CAMPION Sebastien's avatar
CAMPION Sebastien committed
369 370 371 372
        queryset = Job.objects.filter(user_id=self.request.user.id)
        return queryset


373 374
class UserSettingsView(LoginRequiredMixin, UpdateView):
    """ Update of the user basic settings
BERJON Matthieu's avatar
BERJON Matthieu committed
375

376 377 378 379 380 381
    first name, last name, ...
    """
    model = User
    form_class = UserForm
    template_name = 'user_update_form.html'
    success_url = reverse_lazy('main:user_settings')
BERJON Matthieu's avatar
BERJON Matthieu committed
382 383

    def get_object(self):
384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
        """ Get the data from the user according to its ID as he is identified
        """
        queryset = get_object_or_404(User, pk=self.request.user.id)
        return queryset


class UserPasswordUpdateView(LoginRequiredMixin, UpdateView):
    """ Update the password.

    We reuse the Django password form system in order to keep something robust
    even if it dedicates a specific view for it.
    """
    success_url = reverse_lazy('main:user_settings')
    form_class = PasswordChangeForm
    template_name = "user_password_update_form.html"

    def get_object(self, queryset=None):
        return self.request.user

    def get_form_kwargs(self):
        kwargs = super(UserPasswordUpdateView, self).get_form_kwargs()
        kwargs['user'] = kwargs.pop('instance')
        return kwargs

    def dispatch(self, request, *args, **kwargs):
        return super(UserPasswordUpdateView, self) \
410
            .dispatch(request, *args, **kwargs)
411 412


CAMPION Sebastien's avatar
CAMPION Sebastien committed
413 414 415 416 417 418 419
class UserUpdateView(LoginRequiredMixin, UpdateView):
    fields = ['name', ]

    model = AllgoUser

    def get_success_url(self):
        return reverse(
CAMPION Sebastien's avatar
CAMPION Sebastien committed
420 421
            'main:user_settings',
            kwargs={'username': self.request.user.username})
CAMPION Sebastien's avatar
CAMPION Sebastien committed
422 423 424 425

    def get_object(self):
        # Only get the User record for the user making the request
        return AllgoUser.objects.get(user__username=self.request.user.username)