Commit 51f51d9c authored by BAIRE Anthony's avatar BAIRE Anthony
Browse files

Allow importing a webapp from a legacy allgo instance

This adds two views:

- WebappImport for importing the webapp (but without the versions).
  The import is allowed if the requesting user has the same email
  as the owner of the imported app. The webapp is created with
  imported=True, which enables the WebappVersionImport view

- WebappVersionImport for requisting the import of webapp version.
  This only creates the WebappVersion entry with state=IMPORT
  (the actual import is performed by the controller)

A version may be imported multiple times. In that case, the newly
imported version overwrite the local version with the same number.

This features requires:
- that the rails server implements !138
- that the docker daemon hosting the sandboxes is configured with
  credentials for pulling from the legacy registry
parent c9fd7c9e
Pipeline #41192 failed with stage
in 3 minutes and 31 seconds
......@@ -26,6 +26,7 @@ ENV ENV="" \
DOCKER_HOST="unix:///run/docker.sock" \
SWARM_HOST="unix:///run/docker.sock" \
MYSQL_HOST="{ENV}-mysql" \
LABEL dk.migrate_always=1
......@@ -1016,6 +1016,51 @@ exec /.toolbox/bin/sshd -D
log.debug("done sandbox %d", webapp_id)
class ImportManager(Manager):
"""Manager for importing webapp versions"""
def __init__(self, ctrl, nb_threads=1):
self.ctrl = ctrl
async def _process(self, webapp_id, reset, rescheduled):
ctrl = self.ctrl
ses = ctrl.session
# lock the manager so that we only have a single import running
# concurrently (to avoid overloading the push manager and affecting
# users working on a sandbox)
with await iter(self):
log.debug("process import %d", webapp_id)
with ses.begin():
docker_name, = ses.query(Webapp.docker_name).filter_by(id=webapp_id).one()
versions = [(, v.number) + ctrl.gen_image_tag(v)
for v in ses.query(WebappVersion).filter_by(
webapp_id=webapp_id, state=int(VersionState.IMPORT))]
for version_id, number, dst_repo, dst_tag in versions:"importing %s:%s (id%s)", docker_name, number, version_id)
# pull the image from the old registry
src_repo = "%s/%s" % (ctrl.import_registry, docker_name)
src_image = "%s:%s" % (src_repo, number)
await ctrl.image_manager.sandbox_pull_manager.process(
(src_repo, number))
# commit the image"tag %s as %s:%s", src_image, dst_repo, dst_tag)
await self.run_in_executor(ctrl.sandbox.tag, src_image,
dst_repo, dst_tag, lock=False)
# atomically mark the image as committed
with ses.begin():
ses.execute("UPDATE dj_webapp_versions SET state=%d WHERE id=%d AND state=%d"
% (VersionState.COMMITTED, version_id, VersionState.IMPORT))
# push the image to the new registry
await ctrl.image_manager.push(version_id)
class JobManager(Manager):
class JobInfo:
__slots__ = "job_id", "ver_id", "ctr_id", "version", "ctr_name", "client", "cpu", "mem", "node_id", "timeout"
......@@ -1587,7 +1632,7 @@ class PushManager(Manager):
if version.state in (VersionState.READY, VersionState.REPLACED):
# already pushed
if version.state == VersionState.SANDBOX:
if version.state in (VersionState.SANDBOX, VersionState.IMPORT):
raise Error("unable to push (image not yet committed)")
raise Error("unable to push (invalid state: %s)" % version.state)
......@@ -1719,6 +1764,7 @@ class DockerController:
def __init__(self, sandbox_host, swarm_host, mysql_host,
registry, env, datastore_path, sandbox_path,
toolbox_path, sandbox_network, redis_host, job_user,
......@@ -1766,6 +1812,7 @@ class DockerController:
self.image_manager = ImageManager(self, auth_dict=auth_dict)
self.sandbox_manager = SandboxManager(self)
self.job_manager = JobManager(self)
self.import_manager = ImportManager(self)
self.registry = registry
......@@ -1775,6 +1822,7 @@ class DockerController:
self.toolbox_path = toolbox_path
self.sandbox_network= sandbox_network
self.job_user = job_user
self.import_registry= import_registry
self._task = None
self._shutdown_requested = None
......@@ -1926,6 +1974,12 @@ class DockerController:
for webapp_id, in ses.execute("""SELECT webapp_id
FROM dj_webapp_versions WHERE state=%d
GROUP BY webapp_id""" % VersionState.IMPORT):
for job_id, in ses.query(
(JobState.WAITING.value, JobState.RUNNING.value, JobState.ABORTING.value))
......@@ -1986,6 +2040,7 @@ class DockerController:
elif item_type == b"webapp":
log.warning("ignored notification for unknown item %r", msg)
......@@ -27,6 +27,7 @@ class VersionState(enum.IntEnum):
USER = 5
class JobState(enum.IntEnum):
......@@ -141,6 +141,10 @@ def main():
redis_host = val.format(ENV=env)"redis host %s", redis_host)
with get_envvar("ALLGO_IMPORT_REGISTRY") as val:
import_registry = val.format(ENV=env)"import registry %s", import_registry)
os.makedirs("/vol/cache", exist_ok=True)
lockfile = "/vol/cache/controller.lock"
lock = fasteners.InterProcessLock(lockfile)
......@@ -153,7 +157,7 @@ def main():
return controller.DockerController(docker_host, swarm_host, mysql_host,
registry, env, datastore_path, sandbox_path,
toolbox_path, sandbox_network, redis_host,
job_user, import_registry).run()
except config_reader.ConfigError:
log.critical("bad config")
......@@ -9,7 +9,7 @@ RUN apt-getq update && apt-getq install \
nginx-light zip gcc python3-dev python3-pip python3-wheel python3-mysqldb \
python-mysqldb python3-crypto gunicorn3 python3-redis python-mysqldb \
python3-crypto python3-natsort python3-aiohttp python3-aioredis supervisor \
python3-ipy python3-django-taggit
python3-ipy python3-django-taggit python3-iso8601
COPY requirements.txt /tmp/
RUN cd /tmp && pip3 install -r requirements.txt && rm requirements.txt
......@@ -15,6 +15,7 @@ from .models import (
from .validators import docker_name_validator
class UserForm(forms.ModelForm):
......@@ -217,3 +218,10 @@ class WebappSandboxForm(forms.ModelForm):
class Meta:
model = WebappVersion
fields = ('number', 'description')
class WebappImportForm(forms.Form):
webapp_id = forms.IntegerField(label="Webapp ID", required=False)
docker_name = forms.CharField(label="Short name", required=False,
......@@ -232,6 +232,10 @@ class Webapp(TimeStampModel):
memory_limit = models.BigIntegerField(null=True,
# flag indicating if this webbapp was imported from rails
# (if True, then we can import webapp versions)
imported = models.BooleanField(default=False)
# Relationships
# A webapp has one docker os type
......@@ -348,6 +352,8 @@ class WebappVersion(TimeStampModel):
USER = 5 # this version is being pushed directly by the user
IMPORT = 6 # this version is being imported from rails
......@@ -355,18 +361,19 @@ class WebappVersion(TimeStampModel):
# Notes about WebappVersion states
# - Allowed state changes:
# - by django:
# (none) -> SANDBOX
# (none) -> USER
# - by the controller:
......@@ -401,6 +408,9 @@ class WebappVersion(TimeStampModel):
state = models.IntegerField(choices=SANDBOX_STATE_CHOICES)
published = models.BooleanField()
# flag indicating if this version was imported from rails
imported = models.BooleanField(default=False)
webapp = models.ForeignKey('Webapp', on_delete=models.CASCADE, related_name="webapp")
class Meta:
......@@ -49,6 +49,8 @@ urlpatterns = [
url(r'^apps/$', views.WebappList.as_view(), name='webapp_list'),
url(r'^apps/_authors/(?P<username>[\w.@+-]+)/$', views.UserWebappList.as_view(), name='user_webapp_list'),
url(r'^apps/_create/$', views.WebappCreate.as_view(), name='webapp_creation'),
url(r'^apps/_import/$', views.WebappImport.as_view(), name='webapp_import'),
url(r'^apps/(?P<docker_name>[\w-]+)/import$', views.WebappVersionImport.as_view(), name="webapp_version_import"),
url(r'^apps/(?P<docker_name>[\w-]+)/update$', views.WebappUpdate.as_view(), name="webapp_update"),
url(r'^apps/(?P<docker_name>[\w-]+)/sandbox$', views.WebappSandboxPanel.as_view(), name="webapp_sandbox_panel"),
url(r'^apps/(?P<docker_name>[\w-]+)/json$', views.WebappJson.as_view(), name='webapp_json'),
......@@ -14,12 +14,15 @@ import io
import json
import logging
import os
import re
import shutil
import tempfile
import zipfile
import natsort
# Third party imports
import iso8601
import natsort
import requests
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.forms import PasswordChangeForm
......@@ -30,7 +33,7 @@ from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse
from django.db import transaction
from django.db.models import Count
from django.http import HttpResponse, JsonResponse, HttpResponseRedirect, FileResponse
from django.http import HttpResponse, JsonResponse, HttpResponseRedirect, FileResponse, Http404
from django.shortcuts import render, get_object_or_404, redirect
from django.urls import reverse, reverse_lazy
from django.utils.crypto import get_random_string
......@@ -40,6 +43,7 @@ from django.views.generic import (
......@@ -48,6 +52,7 @@ from django.views.generic import (
from django.views.generic.detail import SingleObjectMixin
from taggit.models import Tag
from allauth.account.models import EmailAddress
from .forms import (
......@@ -58,6 +63,7 @@ from .forms import (
# Local imports
import config
......@@ -67,9 +73,11 @@ from .models import (
from .signals import job_post_save
......@@ -285,6 +293,231 @@ class WebappCreate(SuccessMessageMixin, LoginRequiredMixin, IsProviderMixin, Cre
self.webapp = obj
return super().form_valid(form)
def get_rails_webapp_metadata(*, webapp_id=None, docker_name=None):
"""Download the metadata of a webapp from the legacy rails server
Must provide either `webapp_id` or `docker_name`, but not both
assert bool(webapp_id) != bool(docker_name)
if docker_name:
url = "%s/app/%s/export" % (config.env.ALLGO_IMPORT_URL, docker_name)
url = "%s/webapp/%d/export" % (config.env.ALLGO_IMPORT_URL, webapp_id)
# use a 1s timeout to avoid blocking the django thread if the rails
# server is not responding
rep = requests.get(url, timeout=1)
if rep.status_code == 404:
raise Http404()
js = rep.json()
if (webapp_id not in (None, js["id"])
or docker_name not in ("", js["docker_name"])):
raise Exception("rails returned a webapp with inconsistent id or docker_name")
return js
except Exception as e:
log.error("webapp import error: failed to get %s (%s)", url, e)
class WebappImport(SuccessMessageMixin, LoginRequiredMixin, IsProviderMixin, FormView):
"""Import a new webapp
This only creates the Webapp entry (along with the tags and webapp
parameters), versions are imported separately.
Once the webapp is imported, the Webapp entry is created with
imported=True which enables the WebappVersionImport view (for import the
actual versions, including the docker images).
A webapp can be imported only if the e-mail of the current user strictly
matches the owner e-mail of the imported app. If not, then the app has to
be imported by a superuser, who can then transfer its ownership to the
requesting user.
An imported webapp will keep the same id and docker_name (to preserve the
published urls). The import fails the id or docker_name is already used by
another webapp.
model = Webapp
form_class = WebappImportForm
success_message = 'Webapp imported successfully.'
template_name = 'webapp_import.html'
def get_success_url(self):
return reverse('main:webapp_version_import', args=(self.object.docker_name,))
def get_context_data(self, **kwargs):
ctx["import_url"] = config.env.ALLGO_IMPORT_URL+"/apps"
return ctx
def form_valid(self, form):
def error(msg = "500 Internal Server Error"):
messages.error(self.request, "Import failed : " + msg)
return self.form_invalid(form)
# parse the form parameters an prepare the import url
webapp_id = form.cleaned_data["webapp_id"]
docker_name = form.cleaned_data["docker_name"]
if bool(webapp_id) == bool(docker_name):
return error("You must provide either a name or an id")
# get the metadata from the rails server an store them in var 'js'
js = get_rails_webapp_metadata(webapp_id=webapp_id,
except Http404:
return error("application not found")
except Exception as e:
return error()
webapp_id = js["id"]
docker_name = js["docker_name"]
# ensure this app does not already exist locally
if Webapp.objects.filter(docker_name=docker_name).exists():
return error("webapp named %r already exists" % docker_name)
if Webapp.objects.filter(id=webapp_id).exists():
return error("webapp id %r already exists" % webapp_id)
current_user = self.request.user
if not current_user.is_superuser:
# ensure this app has the same owner
if != js["user"]:
return error("""this webapp belongs to another user (if you think
it really belongs to you, then you should contact the
# ensure the user email is verified
#TODO support gitlab accounts
if not EmailAddress.objects.filter(user=current_user,, verified=True).exists():
return error("your e-mail address is not yet verified")
# We can import the webapp !
webapp = Webapp(user=current_user, imported=True,
# TODO: import logo+readme (but they are not yet implemented in django)
for field_name in ("id", "docker_name", "name", "description",
"contact", "default_quota", "entrypoint", "private",
setattr(webapp, field_name, js[field_name])
# try to use job queue with same name or fallback to the default queue
webapp.job_queue = JobQueue.objects.filter(name=js["default_job_queue"]
).first() or JobQueue.objects.filter(is_default=True).first()
# import the tags and parameters
for param in js["parameters"]:
if (param["value"], param["name"], param["detail"]) != (
None, None, None):
name=param["name"], value=param["value"],
self.object = webapp
return super().form_valid(form)
class WebappVersionImport(LoginRequiredMixin, DetailView):
"""Import version
This view is enabled only for webapps created with imported=True
The GET view lists the current status of remote version along with the
local version (if any). If the remote version can be imported, it displays
a checkbox to allow requesting its import.
The POST view creates the WebappVersion entries with state=IMPORT and
notifies the controller (which performs the actual import). The import is
considered done as soon as the entry reaches the COMMITTED state.
template_name = 'webapp_version_import.html'
def get_object(self):
"""Returns the object according to its docker name or a 404 error"""
data = self.kwargs.get('docker_name', None)
queryset = get_object_or_404(Webapp, docker_name=data,
return queryset
def get_context_data(self, **kwargs):
ctx = super().get_context_data(**kwargs)
webapp = self.object
if not webapp.imported:
raise Http404()
# url of this webapp on the legacy server
ctx["import_url"] = "%s/app/%s" % (
config.env.ALLGO_IMPORT_URL, webapp.docker_name)
# get the webapp metadata from the legacy server
remote_versions = get_rails_webapp_metadata(
# dict of local webapp versions (indexed by the number)
# (if multiple entries exist with the same number (this happens if a
# commit/push/import is in progress), we keep the one with the highest
# id)
local_versions = {}
for v in WebappVersion.objects.filter(webapp=webapp).exclude(
local_versions[v.number] = v
# list of versions to be displayed on the page
versions = {}
for remote in remote_versions:
number = remote["number"]
assert number not in versions, "rails must not export duplicated versions"
local_version = local_versions.get(number)
in_progress = getattr(local_version, "state", None) == WebappVersion.IMPORT
versions[number] = {
"number": number,
"remote_ts": iso8601.parse_date(remote["updated_at"]),
"local_ts": "-" if in_progress else getattr(local_version, "updated_at", ""),
"local_imported": None if in_progress else getattr(local_version, "imported", None),
"in_progress": in_progress,
ctx["versions"] = natsort.versorted(versions.values(),
key=lambda v: v["number"], reverse=True)
return ctx
def post(self, request, *, docker_name):
webapp = self.get_object()
if not webapp.imported:
raise Http404()
remote_versions = get_rails_webapp_metadata(
for remote in remote_versions:
number = remote["number"]
if request.POST.get("version_"+number):"import version %s", number)
version = WebappVersion.objects.update_or_create({
"imported": True,
"published": remote["published"],
"description": remote["changelog"],
"docker_image_size": remote["docker_image_size"],
version.created_at = remote["created_at"]
version.updated_at = remote["updated_at"]
transaction.on_commit(lambda: notify_controller(webapp))
return HttpResponseRedirect(request.path_info)
class WebappJson(LoginRequiredMixin, DetailView):
"""json variant of the application details
{% extends "base.html" %}
{% load htmlattrs converters %}
{% block title %}Import a webapp{% endblock %}
{% block breadcrumb %}
<li class="breadcrumb-item"><a href="{% url 'main:webapp_list' %}">Applications</a></li>
<li class="breadcrumb-item active" aria-current="page">Import</li>
{% endblock %}
{% block content %}
<div class="container">
<div class="allgo-page">
<h3>Import an application from <a href="{{import_url}}">{{ import_url }}</a></h3>
<p>Please provide either the ID or the short name of the application you
wish to import.</p>
<form method="post">
{% csrf_token %}
<div class="form-group">
{{ form.webapp_id.label_tag }} <i>(as in url {{ import_url }}/webapps/&lt;ID&gt;)</i>
{% if form.webapp_id.errors %}
{{ form.webapp_id | add_class:"form-control is-invalid" }}
{% else %}
{{ form.webapp_id | add_class:"form-control" }}
{% endif %}
<div class="form-group">
{{ form.docker_name.label_tag }} <i>(as in url {{ import_url }}/app/&lt;NAME&gt;)</i>
{% if form.docker_name.errors %}
{{ form.docker_name | add_class:"form-control is-invalid" }}
{% else %}
{{ form.docker_name | add_class:"form-control" }}
{% endif %}
<input class="btn btn-primary" type="submit" value="Import webapp">
{% endblock %}
{% block messages %}
{{ block.super }}
{% include 'partials/_form_messages.html' %}
{% endblock %}
......@@ -32,6 +32,16 @@
<i class="fa-inverse fas fa-plus" data-fa-transform="shrink-7 down-.25 left-.25"></i>
<span class="text-hide">Create application</span>
href="{% url 'main:webapp_import' %}"
title="Import application"
class="fa-layers fa-2x">
<i class="fas fa-square"></i>
<i class="fa-inverse fas fa-cloud-download-alt" data-fa-transform="shrink-7 down-.25 left-1.75"></i>
<span class="text-hide">Import application</span>
{% endif %}
......@@ -16,8 +16,14 @@
<div class="col-10">
<h1 class="page-title">
<span>Application profile</span>
<a class="btn btn-primary float-right"
<span class="float-right">
<a class="btn btn-primary"