# -*- coding: utf-8 -*- """Main view module This module handles most of the front-end for the Allgo system. You'll find all the logic (controller) in an MVC pattern. Attributes: log: module level variable to save information as a log data. """ # Python standard libraries import glob import io import json import logging import os import re import shutil import tempfile import zipfile # Third party imports import iso8601 import natsort import requests from django.conf import settings from django.contrib import messages from django.contrib.auth.forms import PasswordChangeForm from django.contrib.auth.mixins import LoginRequiredMixin from django.contrib.auth.models import User from django.contrib.messages.views import SuccessMessageMixin from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse from django.db import transaction from django.db.models import Count from django.http import HttpResponse, JsonResponse, HttpResponseRedirect, FileResponse, Http404 from django.shortcuts import render, get_object_or_404, redirect from django.urls import reverse, reverse_lazy from django.utils.crypto import get_random_string from django.utils.text import slugify from django.views.decorators.csrf import csrf_exempt from django.views.generic import ( CreateView, DeleteView, DetailView, FormView, ListView, RedirectView, TemplateView, UpdateView, View, ) from django.views.generic.detail import SingleObjectMixin from taggit.models import Tag from allauth.account.models import EmailAddress from .forms import ( UserForm, HomeSignupForm, UserWebappForm, JobForm, SSHForm, RunnerForm, WebappForm, WebappSandboxForm, WebappImportForm, ) # Local imports import config from .helpers import get_base_url, get_ssh_data, upload_data, notify_controller, lookup_job_file from .mixins import IsProviderMixin from .models import ( AllgoUser, DockerOs, Job, JobQueue, Quota, Runner, Webapp, WebappParameter, WebappVersion, ) from .signals import job_post_save from .templatetags.converters import status_icon # Start logger log = logging.getLogger('allgo') class IndexDetail(TemplateView): """Home view Generate the home as a standard `TemplateView` by calling a specific template. Most of the data are handled in the template itself, only few contexte data are provided for specific use. Attributes: template_name: filename of the template used. """ template_name = 'home.html' def get_context_data(self, **kwargs): """ Generate specific data to pass on in the context of the template. Returns: user_nb (int): number of users recorded in the database. webapp_nb (int): number of webapps recorded in the database. job_nb (int): number of jobs recorded in the database. signup_form: form specific for signin-up directly on the home page. """ context = super(IndexDetail, self).get_context_data(**kwargs) users = User.objects.all().count() webapps = Webapp.objects.all().count() jobs = Job.objects.all().count() context['user_nb'] = users context['webapp_nb'] = webapps context['job_nb'] = jobs context['signup_form'] = HomeSignupForm() return context # Legacy views class LegacyWebappDetail(SingleObjectMixin, RedirectView): model = Webapp permanent = True slug_field = "docker_name" def get_redirect_url(self, **kwargs): return reverse("main:webapp_detail", args=(self.get_object().docker_name,)) # WEBAPPS # ----------------------------------------------------------------------------- class WebappList(ListView): """ Display a paginated list of available webapps. The webapps are filtered from the most recent to the oldest and no private apps are displayed. Attributes: model: Webapp model is used. context_object_name: the name used in the template to display each variable. paginate_by: the number of occurences per page template_name: name of the template loaded with this view. queryset: a specific queryset designed to filter the data. Todo: - the number of occurences per page could be loaded from the config file. """ model = Webapp context_object_name = 'webapps' paginate_by = 10 template_name = 'webapp_list.html' queryset = Webapp.objects.filter(private=0).order_by('-created_at') class UserWebappList(ListView): """List of user's webapp Returns all the webapps owned by a specific user. Only the user can its apps. Attributes: model: database model context_object_name: variable name used in the template to display the data. paginate_by: number of occurences by page. template_name: template filename. """ model = Webapp context_object_name = 'webapps' paginate_by = 10 template_name = 'webapp_list.html' def get_queryset(self): """Filter apps for a given user""" user = User.objects.get(username=self.kwargs['username']) queryset = Webapp.objects.filter(user=user) return queryset class WebappUpdate(SuccessMessageMixin, LoginRequiredMixin, UpdateView): """Form to update the webapp data Attributes: form_class: form object. template_name: template filename. success_message: message when the form is properly submitted. """ form_class = UserWebappForm template_name = 'webapp_update.html' success_message = 'Your app has been successfully updated.' error_message = 'The email doesn\'t belong to any registered user. Please enter a valid owner email address.' def get_success_url(self): """If successful redirect to the same page""" return reverse('main:webapp_update', args=(self.object.docker_name,)) def get_object(self): """Returns the object according to its docker name or a 404 error""" data = self.kwargs.get('docker_name', None) queryset = get_object_or_404(Webapp, docker_name=data, user_id=self.request.user.id) return queryset def get_form(self): form = super().get_form() if not self.request.user.is_superuser: form.fields['memory_limit_mb'].widget.attrs['readonly'] = True return form def form_valid(self, form): """Save data coming from the form in the database """ obj = form.save(commit=False) try: user = User.objects.get(username=form.cleaned_data['owner']) obj.user_id = user.id obj.memory_limit = form.get_memory_limit(self.request) form.save() # Add the tag to the database (specific because it's a many to # many relationship) form.save_m2m() if user != self.request.user: messages.success(self.request, self.success_message) return redirect('main:user_webapp_list', self.request.user.username) else: return super(WebappUpdate, self).form_valid(form) except User.DoesNotExist: messages.error(self.request, self.error_message) return super(WebappUpdate, self).form_invalid(form) class WebappCreate(SuccessMessageMixin, LoginRequiredMixin, IsProviderMixin, CreateView): """Create a new webapp Attributes: model: model to use in this class. form_class: form object passed to the template. success_message: successfull message sent to the template template_name: template filename. group_required: groups that user must belong to. """ model = Webapp form_class = WebappForm success_message = 'Webapp created successfully.' template_name = 'webapp_add.html' # group_required = ['inria', ] def get_success_url(self): """If successful redirect to the webapp list page""" return reverse('main:webapp_sandbox_panel', args=(self.webapp.docker_name,)) def get_form(self): form = super().get_form() if not self.request.user.is_superuser: form.fields['memory_limit_mb'].widget.attrs['readonly'] = True return form def form_valid(self, form): """Save data coming from the form in the database """ obj = form.save(commit=False) obj.user_id = self.request.user.id if not form.cleaned_data['contact']: obj.contact = self.request.user.email obj.sandbox_state = Webapp.IDLE # Ensure that all specials characters are removed, spaces are replaced # by hyphens and everything is lower-cased obj.docker_name = slugify(form.cleaned_data['name']) obj.memory_limit = form.get_memory_limit(self.request) # validate the Webapp record before saving # (this is a safety measure, do not remove) # FIXME: currently this raises an exception if the slugify-generated # docker_name does not comply with the model constraints # (for example: 'root' and 'sshd' are reserved names) # To solve this, i think we should let the user choose the # docker_name obj.full_clean() obj.save() # set up the docker container for the app Quota.objects.create(user=self.request.user, webapp=obj) # pass on the webapp data to get_successful_url to redirect with the # correct arguments (for instance the docker_name) self.webapp = obj return super().form_valid(form) def get_rails_webapp_metadata(*, webapp_id=None, docker_name=None): """Download the metadata of a webapp from the legacy rails server Must provide either `webapp_id` or `docker_name`, but not both """ assert bool(webapp_id) != bool(docker_name) if docker_name: url = "%s/app/%s/export" % (config.env.ALLGO_IMPORT_URL, docker_name) else: url = "%s/webapp/%d/export" % (config.env.ALLGO_IMPORT_URL, webapp_id) try: # use a 1s timeout to avoid blocking the django thread if the rails # server is not responding rep = requests.get(url, timeout=1) if rep.status_code == 404: raise Http404() rep.raise_for_status() js = rep.json() if (webapp_id not in (None, js["id"]) or docker_name not in ("", js["docker_name"])): raise Exception("rails returned a webapp with inconsistent id or docker_name") return js except Exception as e: log.error("webapp import error: failed to get %s (%s)", url, e) raise class WebappImport(SuccessMessageMixin, LoginRequiredMixin, IsProviderMixin, FormView): """Import a new webapp This only creates the Webapp entry (along with the tags and webapp parameters), versions are imported separately. Once the webapp is imported, the Webapp entry is created with imported=True which enables the WebappVersionImport view (for import the actual versions, including the docker images). A webapp can be imported only if the e-mail of the current user strictly matches the owner e-mail of the imported app. If not, then the app has to be imported by a superuser, who can then transfer its ownership to the requesting user. An imported webapp will keep the same id and docker_name (to preserve the published urls). The import fails the id or docker_name is already used by another webapp. """ model = Webapp form_class = WebappImportForm success_message = 'Webapp imported successfully.' template_name = 'webapp_import.html' def get_success_url(self): return reverse('main:webapp_version_import', args=(self.object.docker_name,)) def get_context_data(self, **kwargs): ctx=super().get_context_data() ctx["import_url"] = config.env.ALLGO_IMPORT_URL+"/apps" return ctx def form_valid(self, form): def error(msg = "500 Internal Server Error"): messages.error(self.request, "Import failed : " + msg) return self.form_invalid(form) # parse the form parameters an prepare the import url webapp_id = form.cleaned_data["webapp_id"] docker_name = form.cleaned_data["docker_name"] if bool(webapp_id) == bool(docker_name): return error("You must provide either a name or an id") # get the metadata from the rails server an store them in var 'js' try: js = get_rails_webapp_metadata(webapp_id=webapp_id, docker_name=docker_name) except Http404: return error("application not found") except Exception as e: return error() webapp_id = js["id"] docker_name = js["docker_name"] # ensure this app does not already exist locally if Webapp.objects.filter(docker_name=docker_name).exists(): return error("webapp named %r already exists" % docker_name) if Webapp.objects.filter(id=webapp_id).exists(): return error("webapp id %r already exists" % webapp_id) current_user = self.request.user if not current_user.is_superuser: # ensure this app has the same owner if current_user.email != js["user"]: return error("""this webapp belongs to another user (if you think it really belongs to you, then you should contact the administrators)""") # ensure the user email is verified #TODO support gitlab accounts if not EmailAddress.objects.filter(user=current_user, email=current_user.email, verified=True).exists(): return error("your e-mail address is not yet verified") # We can import the webapp ! webapp = Webapp(user=current_user, imported=True, docker_os=DockerOs.objects.first()) # TODO: import logo+readme (but they are not yet implemented in django) for field_name in ("id", "docker_name", "name", "description", "contact", "default_quota", "entrypoint", "private", "memory_limit"): setattr(webapp, field_name, js[field_name]) # try to use job queue with same name or fallback to the default queue webapp.job_queue = JobQueue.objects.filter(name=js["default_job_queue"] ).first() or JobQueue.objects.filter(is_default=True).first() webapp.save() # import the tags and parameters webapp.tags.add(*js["tags"]) for param in js["parameters"]: if (param["value"], param["name"], param["detail"]) != ( None, None, None): WebappParameter.objects.create(webapp=webapp, name=param["name"], value=param["value"], detail=param["detail"]) self.object = webapp return super().form_valid(form) class WebappVersionImport(LoginRequiredMixin, DetailView): """Import version This view is enabled only for webapps created with imported=True The GET view lists the current status of remote version along with the local version (if any). If the remote version can be imported, it displays a checkbox to allow requesting its import. The POST view creates the WebappVersion entries with state=IMPORT and notifies the controller (which performs the actual import). The import is considered done as soon as the entry reaches the COMMITTED state. """ template_name = 'webapp_version_import.html' def get_object(self): """Returns the object according to its docker name or a 404 error""" data = self.kwargs.get('docker_name', None) queryset = get_object_or_404(Webapp, docker_name=data, user_id=self.request.user.id) return queryset def get_context_data(self, **kwargs): ctx = super().get_context_data(**kwargs) webapp = self.object if not webapp.imported: raise Http404() # url of this webapp on the legacy server ctx["import_url"] = "%s/app/%s" % ( config.env.ALLGO_IMPORT_URL, webapp.docker_name) # get the webapp metadata from the legacy server remote_versions = get_rails_webapp_metadata( docker_name=webapp.docker_name)["versions"] # dict of local webapp versions (indexed by the number) # (if multiple entries exist with the same number (this happens if a # commit/push/import is in progress), we keep the one with the highest # id) local_versions = {} for v in WebappVersion.objects.filter(webapp=webapp).exclude( state__in=(WebappVersion.ERROR, WebappVersion.REPLACED)).order_by("id"): local_versions[v.number] = v # list of versions to be displayed on the page versions = {} for remote in remote_versions: number = remote["number"] assert number not in versions, "rails must not export duplicated versions" local_version = local_versions.get(number) in_progress = getattr(local_version, "state", None) == WebappVersion.IMPORT versions[number] = { "number": number, "remote_ts": iso8601.parse_date(remote["updated_at"]), "local_ts": "-" if in_progress else getattr(local_version, "updated_at", ""), "local_imported": None if in_progress else getattr(local_version, "imported", None), "in_progress": in_progress, } ctx["versions"] = natsort.versorted(versions.values(), key=lambda v: v["number"], reverse=True) return ctx def post(self, request, *, docker_name): webapp = self.get_object() if not webapp.imported: raise Http404() remote_versions = get_rails_webapp_metadata( docker_name=webapp.docker_name)["versions"] for remote in remote_versions: number = remote["number"] if request.POST.get("version_"+number): log.info("import version %s", number) version = WebappVersion.objects.update_or_create({ "imported": True, "published": remote["published"], "description": remote["changelog"], "docker_image_size": remote["docker_image_size"], }, webapp=webapp, number=number, state=WebappVersion.IMPORT)[0] version.created_at = remote["created_at"] version.updated_at = remote["updated_at"] version.save() transaction.on_commit(lambda: notify_controller(webapp)) return HttpResponseRedirect(request.path_info) class WebappJson(LoginRequiredMixin, DetailView): """json variant of the application details (used by the /aio/apps//events endpoint) """ def get_object(self): """Returns the object according to its docker name or a 404 error""" data = self.kwargs.get('docker_name', None) queryset = get_object_or_404(Webapp, docker_name=data) return queryset def render_to_response(self, context, **kwargs): webapp = context["webapp"] return JsonResponse({ "id": webapp.id, "sandbox_state": webapp.get_sandbox_state_display(), }) class WebappSandboxPanel(LoginRequiredMixin, TemplateView): """Create a new sandbox for a given application Attributes: form_class: form object to pass on the template. model: model to use in this class template_name: template filename """ template_name = 'webapp_sandbox_panel.html' def get_object(self): """Returns the object according to its docker name or a 404 error""" data = self.kwargs.get('docker_name', None) queryset = get_object_or_404(Webapp, docker_name=data, user_id=self.request.user.id) return queryset def get_context_data(self, **kwargs): """Recover data to pass on to the template context In order to give the user a feedback regarding the way to push its image to the registry, we need to pass both the webapp `docker_name` and the `registry` URL. """ context = super().get_context_data(**kwargs) context['webapp'] = self.get_object() context["ssh_command"] = "ssh%s %s@%s" % ( (" -p %s" % config.env.ALLGO_SSH_PORT if config.env.ALLGO_SSH_PORT != "22" else ""), (kwargs["docker_name"]), (config.env.ALLGO_SSH_HOST)) # candidate docker os (start from scratch) context['docker_os_list'] = DockerOs.objects.all() # candidate versions (start from an existing version) versions = {} for state in (WebappVersion.READY, WebappVersion.COMMITTED): versions.update((v.number, v) for v in WebappVersion.objects.filter( webapp=context["webapp"], state=state)) context['versions'] = natsort.versorted(versions.values(), key=lambda v: v.number) context['versions'].reverse() return context def post(self, request, *, docker_name): log.info("POST %r", request.POST) webapp = self.get_object() action = request.POST["action"] def stop_sandbox(): webapp.sandbox_state = Webapp.STOPPING webapp.sandbox_version_id = None webapp.save() log.info("action %r", request.POST["action"]) if action == "start": if webapp.sandbox_state != Webapp.IDLE: messages.error(request, "unable to start sandbox %r because it not idle" % webapp.name) else: if "webapp_version_id" in request.POST: # start from an existing version webapp.sandbox_version_id = int(request.POST["webapp_version_id"]) else: # start from scratch webapp.docker_os_id = request.POST["docker_os_id"] webapp.sandbox_version = None webapp.sandbox_state = Webapp.STARTING webapp.save() messages.success(request, "starting sandbox %r" % webapp.name) elif action == "commit": if webapp.sandbox_state != Webapp.RUNNING: messages.error(request, "unable to commit sandbox %r because it is not running" % webapp.name) else: # query previous active versions of this webapp previous = WebappVersion.objects.filter(webapp=webapp, state__in = (WebappVersion.READY, WebappVersion.COMMITTED)) extra = {} if request.POST["version-action"] == "replace-version": number = request.POST["version-select"] # keep the previous 'created_at' timestamp when replacing an image extra["created_at"] = getattr(previous.filter(number=number).first(), "created_at") else: number = request.POST["version-new"] # ensure that this version number does not already exist if previous.filter(number=number).exists(): messages.error(request, "unable to commit because version %r already exists" " (if you want to overwrite this version, then use" " 'replace version' instead)" % number) return HttpResponseRedirect(request.path_info) WebappVersion.objects.create( webapp=webapp, number=number, state=WebappVersion.SANDBOX, published=True, description=request.POST["description"], **extra) stop_sandbox() messages.success(request, "committing sandbox %r version %r" % (webapp.name, number)) elif action == "rollback": if webapp.sandbox_state == Webapp.RUNNING: stop_sandbox() messages.success(request, "rolling back sandbox %r" % webapp.name) else: messages.error(request, "unable to roll back, sandbox %r is not running" % webapp.name) elif action == "abort": if webapp.sandbox_state == Webapp.START_ERROR: stop_sandbox() messages.success(request, "reset sandbox %r" % webapp.name) elif action == "retry": if webapp.sandbox_state == Webapp.START_ERROR: webapp.sandbox_state = Webapp.STARTING webapp.save() messages.success(request, "starting sandbox %r" % webapp.name) elif webapp.sandbox_state == Webapp.STOP_ERROR: stop_sandbox() messages.success(request, "stopping sandbox %r" % webapp.name) log.debug("new sandbox state: %r -> %r", webapp.docker_name, webapp.sandbox_state) # NOTE: we return a 302 redirect to the same page (instead of rendering # it directly) to force the browser to make a separate GET request. # This prevent reexecuting the POST request if the user refreshes the # page. return HttpResponseRedirect(request.path_info) # TAGS # ----------------------------------------------------------------------------- class TagList(ListView): """List all available tag along with their number of occurences Attributes: model: database model context_object_name: variable name used in the template to display the data. template_name: template filename. """ model = Tag context_object_name = 'tags' template_name = 'tag_list.html' def get_queryset(self): """Return all available tags Each tag return as well the number of webapps attached to it """ tags = Tag.objects.annotate(num_tag=Count('taggit_taggeditem_items')) return tags class TagWebappList(ListView): """List all available webapps for a given tag Attributes: model: database model context_object_name: variable name used in the template to display the data. paginated_by: number of occurences per page. template_name: template filename. """ model = Webapp context_object_name = 'webapps' paginated_by = 10 template_name = 'tag_webapp_list.html' def get_queryset(self): return Webapp.objects.filter(tags__slug=self.kwargs['slug']) def get_context_data(self, **kwargs): return super().get_context_data(tag=self.kwargs["slug"], **kwargs) # PROFILE # ----------------------------------------------------------------------------- class UserUpdate(SuccessMessageMixin, LoginRequiredMixin, UpdateView): """Update the user profile Attributes: form_class: form object to pass on the template. template_name: template filename success_message: successfull message sent to the template """ form_class = UserForm template_name = "user_update.html" success_message = 'Profile updated successfully.' def get_success_url(self): """If successful redirect to the user page""" return reverse('main:user_detail') def get_object(self): """Only get the User record for the user making the request""" return User.objects.get(username=self.request.user.username) def get_context_data(self, **kwargs): """Recover data to pass on to the template In order to display specific data, I process the SSH key to get its fingerprint and comment. Both the SSH key, fingerprint, comment and token to context template. """ queryset = AllgoUser.objects.get(user_id=self.request.user.id) key = queryset.sshkey token = queryset.token if key: fingerprint, comment = get_ssh_data(key) kwargs['sshkey'] = True kwargs['ssh_comment'] = comment kwargs['ssh_fingerprint'] = fingerprint if token: kwargs['token'] = token return super(UserUpdate, self).get_context_data(**kwargs) class UserToken(LoginRequiredMixin, RedirectView): """Regenerate the user token""" success_message = 'Token generated successfully.' def dispatch(self, request, *args, **kwargs): """Generate the token and save it into the database""" queryset = AllgoUser.objects.get(user_id=self.request.user.id) queryset.token = get_random_string(length=32) queryset.save() return super(UserToken, self).dispatch(request, *args, **kwargs) def get_redirect_url(self, *args, **kwargs): """Redirect the user to the user page and display a successful message""" messages.success(self.request, self.success_message) return reverse('main:user_detail') class UserSSHAdd(SuccessMessageMixin, LoginRequiredMixin, UpdateView): """Save a SSH key to the database. Attributes: form_class: form object to pass on the template. template_name: template filename success_message: successfull message sent to the template """ form_class = SSHForm template_name = 'user_ssh_add.html' success_message = 'SSH key added successfully.' def get_success_url(self): """If successful redirect to the user page""" return reverse('main:user_detail') def get_object(self): """Only get the User record for the user making the request""" return AllgoUser.objects.get(user_id=self.request.user.id) class UserSSHDelete(LoginRequiredMixin, RedirectView): """Delete the user SSH key""" success_message = 'The SSH key has been successfully deleted.' def dispatch(self, request, *args, **kwargs): """Generate an empty SSH key and save it into the database""" queryset = AllgoUser.objects.get(user_id=request.user.id) queryset.sshkey = '' queryset.save() return super(UserSSHDelete, self).dispatch(request, *args, **kwargs) def get_redirect_url(self, *args, **kwargs): """If successful redirect to the user page""" messages.success(self.request, self.success_message) return reverse('main:user_detail') class UserPasswordUpdate(SuccessMessageMixin, LoginRequiredMixin, UpdateView): """Update the user's password. We reuse the Django password form system in order to keep something robust even if it dedicates a specific view for it. Attributes: success_url: URL or handle where the user will be redirected. form_class: form object to pass on the template. template_name: template filename success_message: successfull message sent to the template """ success_url = reverse_lazy('main:user_detail') form_class = PasswordChangeForm template_name = "user_password_update.html" success_message = 'Password updated successfully.' def get_object(self, queryset=None): """Return the user data Todo: - Not sure the relevance of getting this information to the template """ return self.request.user def get_form_kwargs(self): """Return the arguments related to the user""" kwargs = super(UserPasswordUpdate, self).get_form_kwargs() kwargs['user'] = kwargs.pop('instance') return kwargs def dispatch(self, request, *args, **kwargs): """ Todo: - I'm not sure why I wrote that and why it is useful in the present case. It needs to be investigated. """ return super(UserPasswordUpdate, self) \ .dispatch(request, *args, **kwargs) # JOBS # ----------------------------------------------------------------------------- class JobList(LoginRequiredMixin, ListView): """Display the list of jobs for a given identified user Attributes: model: model used in the view. context_object_name: the name used in the template to display each variable. paginate_by: the number of occurences per page template_name: name of the template loaded with this view. redirect_field_name: None Todo: - Check the relevance of `redirect_field_name` and delete it if necessary """ model = Job context_object_name = 'job_list' template_name = 'job_list.html' paginate_by = 10 redirect_field_name = 'redirect_to' def get_queryset(self): """Filter jobs for a given user""" queryset = Job.objects.filter(user_id=self.request.user.id ).exclude(state__in=(Job.DELETED, Job.ARCHIVED)).order_by('-id') return queryset class JobDetail(LoginRequiredMixin, DetailView): """Get a job detail for a specific user Attributes: model: model used in the view. context_object_name: the name used in the template to display each variable. template_name: name of the template loaded with this view. """ model = Job template_name = 'job_detail.html' context_object_name = 'job' def get_context_data(self, **kwargs): """Recover the logs and files related to this job""" job = Job.objects.get(pk=self.object.pk) if job.state == Job.DONE: # job is done # -> read the `allgo.log` file log_file = os.path.join(job.data_dir, 'allgo.log') try: with open(log_file, 'r', errors="replace") as log_data: logs = log_data.read() except OSError as e: logs = '(logs not available)' log.error("Log file not available for job #%d (%s)", job.id, e) else: # job is pending # -> logs will be streamed (ajax request) logs = "" kwargs['logs'] = logs # Hide the logs panel if the job is not yet started kwargs["logs_hidden"] = "hidden" if job.state in (Job.NEW, Job.WAITING) else "" # Get the files and some metadata such as the webapp version webapp = Webapp.objects.get(docker_name=self.object.webapp.docker_name) # List all job files # NOTE: calling lookup_job_file is a security feature kwargs['files'] = [x for x in os.listdir(job.data_dir) if lookup_job_file(job.id, x)] return super().get_context_data(**kwargs) def render_to_response(self, context, **kwargs): if self.request.META.get("HTTP_ACCEPT") == "application/json": # json variant of the job details # (used by the /aio/jobs//events endpoint) job = context["job"] return JsonResponse({ "id": job.id, "state": job.get_state_display(), "result": job.get_result_display(), "rendered_status": status_icon(job), "exec_time": job.exec_time, }) else: return super().render_to_response(context, **kwargs) class JobCreate(SuccessMessageMixin, CreateView): """ Display the data related a specific web and create a job instance into the database Attributes: model: model used in the view. form_class: form object to pass on the template. success_url: URL or handle where the user will be redirected. success_message: successfull message sent to the template template_name: name of the template loaded with this view. """ model = Job form_class = JobForm success_message = 'Job created successfully.' template_name = 'webapp_detail.html' def get_success_url(self): return reverse('main:job_detail', args=(self.job_id,)) def form_valid(self, form): """Save data coming from the form in the database """ webapp = Webapp.objects.get(docker_name=self.kwargs['docker_name']) # If the user isn't identified, we send back an error message and # and redirect the user. if self.request.user.is_anonymous(): messages.add_message(self.request, messages.ERROR, 'You must be identified to create a job.') log.warning("Someone tried to run a job without being identified.") return redirect('main:webapp_detail', webapp.docker_name) else: obj = form.save(commit=False) obj.queue_id = form.cleaned_data.get('queue_id').id obj.state = Job.NEW obj.result = 0 obj.user_id = self.request.user.id obj.webapp_id = webapp.id obj.version = form.cleaned_data.get('version') obj.save() # Upload files if there are any upload_data(self.request.FILES.getlist('files'), obj) # start the job obj.state = Job.WAITING obj.save() self.job_id = obj.id return super().form_valid(form) def get_context_data(self, **kwargs): """Pass on the docker name to the template""" webapp = Webapp.objects.get(docker_name=self.kwargs['docker_name']) kwargs['webapp'] = webapp # Check if a readme is declared in the database if webapp.readme: readme_file = os.path.join( settings.MEDIA_ROOT, self.object.docker_name, 'Readme') if os.path.exists(readme_file): with open(readme_file, 'r') as md_data: kwargs['readme'] = md_data.read() else: log.warning("No README available for app %s", self.model.name) else: readme_file = None # select the list of versions to be displayed versions = natsort.versorted(set(v for v, in WebappVersion.objects .filter(webapp=webapp, state__in=( WebappVersion.SANDBOX, WebappVersion.COMMITTED, WebappVersion.READY)) .values_list("number"))) # also list 'sandbox' if the sandbox is running and if the current user # is allowed to use the sandbox if webapp.sandbox_state == Webapp.RUNNING and ( webapp.is_pushable_by(self.request.user)): versions.append("sandbox") versions.reverse() kwargs['versions'] = versions # build the sample command lines for using the REST API base_url = get_base_url(self.request) user = self.request.user auth = "Authorization: Token token=" + ( user.allgouser.token if user.is_authenticated else "") kwargs["job_create_cmd"] = ["curl", "-H", auth, "-X", "POST", base_url + reverse("api:jobs"), None, "-F", "job[webapp_id]=" + str(webapp.id), None, "-F", "job[param]=", None, "-F", "job[queue]=" + webapp.job_queue.name, None, "-F", "files[0]=@test.txt", None, "-F", "files[1]=@test2.csv", None, "-F", "job[file_url]=", None, "-F", "job[dataset]=", ] kwargs["job_result_cmd"] = ["curl", "-H", auth, base_url + reverse("api:job", args=(42,)).replace("42", "")] return super().get_context_data(**kwargs) def get_form_kwargs(self): """Return webapp data""" kwargs = super().get_form_kwargs() queryset = Webapp.objects.get(docker_name=self.kwargs['docker_name']) kwargs['webapp'] = queryset return kwargs class JobAbort(LoginRequiredMixin, View): def post(self, request, *, pk): job_id = int(pk) # switch state to ABORTING if the job is running (this is done # atomically to avoid messing up with the controller) if Job.objects.filter(id=job_id, state=Job.RUNNING ).update(state=Job.ABORTING): job_post_save(Job.objects.get(id=job_id)) messages.success(request, "aborting job %s" % job_id) else: messages.error(request, "unable to abort job %s because is not running" % job_id) return redirect('main:job_detail', job_id) class JobDelete(LoginRequiredMixin, DeleteView): """Delete a job from the database Attributes: model: model used in the view. success_url: URL or handle where the user will be redirected. success_message: successfull message sent to the template template_name: name of the template loaded with this view. Note: The `success_message` can't be used alone with the `SuccessMessageMixin` because it's hooked to `form_valid` method and can't work with a `DeleteView`. See also: https://code.djangoproject.com/ticket/21926 """ model = Job success_message = 'Job successfully deleted.' success_url = reverse_lazy('main:job_list') template_name = 'job_delete.html' @classmethod def as_view(cls, **kw): # manage db transactions manually return transaction.non_atomic_requests(super().as_view(**kw)) def delete(self, request, *args, pk, **kwargs): # NOTE: if job is in WAITING state, then any state update must be done # atomically so as not to mess up with che controller if not (Job.objects.filter(id=pk, state=Job.DONE ).update(state=Job.ARCHIVED) or Job.objects.filter(id=pk, state__in=(Job.NEW, Job.WAITING) ).update(state=Job.DELETED) or Job.objects.filter(id=pk, state__in=(Job.DELETED, Job.ARCHIVED)).exists() ): messages.error(self.request, "cannot delete a running job") return redirect('main:job_detail', pk) transaction.commit() self.object = job = self.get_object() notify_controller(job) # so that the DELETED/ARCHIVED state is propagated into the redis db # delete the data dir if present # FIXME: if this fail then we have dangling files staying in the way job_dir = job.data_dir if os.path.exists(job_dir): shutil.rmtree(job_dir) if job.state == Job.DELETED: job.delete() messages.success(self.request, self.success_message) return redirect(self.get_success_url()) class JobFileDownload(LoginRequiredMixin, View): """Download a given file""" def get(self, request, *args, **kwargs): """Return a file for a given job and filename """ # get file job_id = self.kwargs['pk'] filename = self.kwargs['filename'] return redirect("/datastore/%s/%s" % (job_id, filename)) class JobFileDownloadAll(LoginRequiredMixin, SingleObjectMixin, View): """Archive and download all files of a given job """ model = Job def get(self, request, *args, **kwargs): """get all the file for a given job The method gets the job ID, recover each file related to this job, archive into a ZIP file and return it. The ZIP file is stored as an anonymous file in /tmp/ then streamed with FileResponse. This is better that keepingthe whole file in memory because it may be large (and linux has a quite efficient page cache). """ job = self.get_object() tmp_file = tempfile.TemporaryFile() zip_subdir = str(job.id) zip_filename = 'job_%s.zip' % zip_subdir zip_file = zipfile.ZipFile(tmp_file, 'w') for filename in os.listdir(job.data_dir): # NOTE: calling lookup_job_file is a security feature real_path = lookup_job_file(job.id, filename) if real_path: zip_path = os.path.join(zip_subdir, filename) zip_file.write(real_path, zip_path) zip_file.close() tmp_file.seek(0) response = FileResponse(tmp_file, content_type='application/x-zip-compressed') response["Content-Disposition"] = "attachment; filename={0}".format(zip_filename) return response # RUNNERS # ----------------------------------------------------------------------------- class RunnerList(LoginRequiredMixin, ListView): """List all runners of a given user Attributes: model: model used in the view. context_object_name: the name used in the template to display each variable. paginate_by: the number of occurences per page template_name: name of the template loaded with this view. """ model = Runner context_object_name = 'runner_list' paginate_by = 10 template_name = 'runner_list.html' def get_queryset(self): """Returns all runners of a given user Returns all runners for a given user from the most recent to the oldest one. """ queryset = Runner.objects.filter(user=self.request.user).order_by('-created_at') return queryset def get_context_data(self, **kwargs): """Return the number of webapps of a given user This method returns the number of webapps of a given user and pass it onto the template in order to display or not the link to add a runner or not. There are no reasons of adding a runner if the user doesn't manage any applications. """ webapp_count = Webapp.objects.filter(user=self.request.user).count() kwargs['webapp_count'] = webapp_count return super().get_context_data(**kwargs) class RunnerCreate(SuccessMessageMixin, LoginRequiredMixin, IsProviderMixin, CreateView): """Create a runner and save it into the database Attributes: model: model used in the view. form_class: form object to pass on the template. success_message: successfull message sent to the template template_name: name of the template loaded with this view. """ model = Runner form_class = RunnerForm success_message = 'Runner saved successfully.' error_message = 'You don\'t have sufficient privileges to create an open bar runner.' success_url = reverse_lazy('main:runner_list') template_name = 'runner_add_update.html' def form_valid(self, form): """ Validate some fields before saving them.""" obj = form.save(commit=False) # If the open-bar argument is true and the user is a superuser. # We setup the field as True if form.cleaned_data.get('open_bar') and self.request.user.is_superuser: obj.open_bar = True # If the open-bar argument is true but the user isn't # We send an error message and force the field to False if form.cleaned_data.get('open_bar') and not self.request.user.is_superuser: obj.open_bar = False messages.error(self.request, self.error_message) obj.user = self.request.user obj.save() return super().form_valid(form) def get_form_kwargs(self): """Pass on the request data onto the template""" kwargs = super().get_form_kwargs() kwargs['request'] = self.request return kwargs def get_success_url(self): """If successful redirect to the runner update page""" return reverse_lazy('main:runner_update', args=(self.object.pk,)) class RunnerUpdate(SuccessMessageMixin, LoginRequiredMixin, UpdateView): """Update a runner and save it into the database Attributes: model: model used in the view. form_class: form object to pass on the template. success_message: successfull message sent to the template success_url: URL or handle where the user will be redirected. template_name: name of the template loaded with this view. """ model = Runner form_class = RunnerForm success_message = 'Runner updated successfully.' success_url = reverse_lazy('main:runner_list') template_name = 'runner_add_update.html' def get_form_kwargs(self): """Pass on the request data onto the template""" kwargs = super(RunnerUpdate, self).get_form_kwargs() kwargs['request'] = self.request return kwargs def get_context_data(self, **kwargs): kwargs["runner_launch_cmd"] = ["docker", "run", "-v", "/var/run/docker.sock:/var/run/docker.sock", "--net=host", "allgo/runner", "-", self.object.token, get_base_url(self.request)] return super().get_context_data(**kwargs) class RunnerDelete(LoginRequiredMixin, DeleteView): """Delete a runner Attributes: model: model used in the view. success_message: successfull message sent to the template success_url: URL or handle where the user will be redirected. template_name: name of the template loaded with this view. """ model = Runner success_message = 'Runner successfully deleted.' success_url = reverse_lazy('main:runner_list') template_name = 'runner_delete.html' def delete(self, request, *args, **kwargs): messages.success(self.request, self.success_message) return super().delete(request, *args, **kwargs) @csrf_exempt def auth(request): """ nginx route /datastore/jobid/filename ask an authorization here with auth_request module we must play with two kind of auth, with django and by token :param request: :return: """ log.debug("Auth request for %r", request.META.get('HTTP_X_ORIGINAL_URI')) # authenticate the user user = None if request.user and request.user.is_authenticated(): # django authentification user = request.user elif request.META.get('HTTP_AUTHORIZATION', ''): # token authentification _, credentials = request.META.get('HTTP_AUTHORIZATION', '').split(' ') _, token = credentials.split('=') try: user = AllgoUser.objects.get(token=token) except AllgoUser.DoesNotExist: pass if user is None: return HttpResponse(status=401) # find the relevant job params = request.META['HTTP_X_ORIGINAL_URI'].split('/') try: job = Job.objects.get(id=int(params[2])) except ObjectDoesNotExist: # technically this should be a 404, but nginx auth_request only # understands 401 & 403 return HttpResponse(status=403) if user.id == job.user.id: return HttpResponse(status=200) else: return HttpResponse(status=403)