Commit 51f51d9c authored by BAIRE Anthony's avatar BAIRE Anthony

Allow importing a webapp from a legacy allgo instance


This adds two views:

- WebappImport for importing the webapp (but without the versions).
  The import is allowed if the requesting user has the same email
  as the owner of the imported app. The webapp is created with
  imported=True, which enables the WebappVersionImport view

- WebappVersionImport for requisting the import of webapp version.
  This only creates the WebappVersion entry with state=IMPORT
  (the actual import is performed by the controller)

A version may be imported multiple times. In that case, the newly
imported version overwrite the local version with the same number.

This features requires:
- that the rails server implements !138
- that the docker daemon hosting the sandboxes is configured with
  credentials for pulling from the legacy registry
parent c9fd7c9e
Pipeline #41192 failed with stage
in 3 minutes and 31 seconds
......@@ -26,6 +26,7 @@ ENV ENV="" \
DOCKER_HOST="unix:///run/docker.sock" \
SWARM_HOST="unix:///run/docker.sock" \
MYSQL_HOST="{ENV}-mysql" \
ALLGO_REDIS_HOST="{ENV}-redis"
ALLGO_REDIS_HOST="{ENV}-redis" \
ALLGO_IMPORT_REGISTRY="cargo.irisa.fr:8003/allgo/prod/webapp"
LABEL dk.migrate_always=1
......@@ -1016,6 +1016,51 @@ exec /.toolbox/bin/sshd -D
log.debug("done sandbox %d", webapp_id)
class ImportManager(Manager):
"""Manager for importing webapp versions"""
def __init__(self, ctrl, nb_threads=1):
super().__init__(nb_threads)
self.ctrl = ctrl
async def _process(self, webapp_id, reset, rescheduled):
ctrl = self.ctrl
ses = ctrl.session
# lock the manager so that we only have a single import running
# concurrently (to avoid overloading the push manager and affecting
# users working on a sandbox)
with await iter(self):
log.debug("process import %d", webapp_id)
with ses.begin():
docker_name, = ses.query(Webapp.docker_name).filter_by(id=webapp_id).one()
versions = [(v.id, v.number) + ctrl.gen_image_tag(v)
for v in ses.query(WebappVersion).filter_by(
webapp_id=webapp_id, state=int(VersionState.IMPORT))]
for version_id, number, dst_repo, dst_tag in versions:
log.info("importing %s:%s (id%s)", docker_name, number, version_id)
# pull the image from the old registry
src_repo = "%s/%s" % (ctrl.import_registry, docker_name)
src_image = "%s:%s" % (src_repo, number)
await ctrl.image_manager.sandbox_pull_manager.process(
(src_repo, number))
# commit the image
log.info("tag %s as %s:%s", src_image, dst_repo, dst_tag)
await self.run_in_executor(ctrl.sandbox.tag, src_image,
dst_repo, dst_tag, lock=False)
# atomically mark the image as committed
with ses.begin():
ses.execute("UPDATE dj_webapp_versions SET state=%d WHERE id=%d AND state=%d"
% (VersionState.COMMITTED, version_id, VersionState.IMPORT))
# push the image to the new registry
await ctrl.image_manager.push(version_id)
class JobManager(Manager):
class JobInfo:
__slots__ = "job_id", "ver_id", "ctr_id", "version", "ctr_name", "client", "cpu", "mem", "node_id", "timeout"
......@@ -1587,7 +1632,7 @@ class PushManager(Manager):
if version.state in (VersionState.READY, VersionState.REPLACED):
# already pushed
return
if version.state == VersionState.SANDBOX:
if version.state in (VersionState.SANDBOX, VersionState.IMPORT):
raise Error("unable to push (image not yet committed)")
raise Error("unable to push (invalid state: %s)" % version.state)
......@@ -1719,6 +1764,7 @@ class DockerController:
def __init__(self, sandbox_host, swarm_host, mysql_host,
registry, env, datastore_path, sandbox_path,
toolbox_path, sandbox_network, redis_host, job_user,
import_registry,
config_file="/vol/ro/config.yml",
):
......@@ -1766,6 +1812,7 @@ class DockerController:
self.image_manager = ImageManager(self, auth_dict=auth_dict)
self.sandbox_manager = SandboxManager(self)
self.job_manager = JobManager(self)
self.import_manager = ImportManager(self)
self.registry = registry
......@@ -1775,6 +1822,7 @@ class DockerController:
self.toolbox_path = toolbox_path
self.sandbox_network= sandbox_network
self.job_user = job_user
self.import_registry= import_registry
self._task = None
self._shutdown_requested = None
......@@ -1926,6 +1974,12 @@ class DockerController:
self.sandbox_manager.process(webapp_id)
for webapp_id, in ses.execute("""SELECT webapp_id
FROM dj_webapp_versions WHERE state=%d
GROUP BY webapp_id""" % VersionState.IMPORT):
self.import_manager.process(webapp_id)
for job_id, in ses.query(Job.id).filter(Job.state.in_(
(JobState.WAITING.value, JobState.RUNNING.value, JobState.ABORTING.value))
).order_by(Job.state):
......@@ -1986,6 +2040,7 @@ class DockerController:
self.job_manager.process(item_id)
elif item_type == b"webapp":
self.sandbox_manager.process(item_id)
self.import_manager.process(item_id)
else:
log.warning("ignored notification for unknown item %r", msg)
......
......@@ -27,6 +27,7 @@ class VersionState(enum.IntEnum):
ERROR = 3
REPLACED = 4
USER = 5
IMPORT = 6
class JobState(enum.IntEnum):
......
......@@ -141,6 +141,10 @@ def main():
redis_host = val.format(ENV=env)
log.info("redis host %s", redis_host)
with get_envvar("ALLGO_IMPORT_REGISTRY") as val:
import_registry = val.format(ENV=env)
log.info("import registry %s", import_registry)
os.makedirs("/vol/cache", exist_ok=True)
lockfile = "/vol/cache/controller.lock"
lock = fasteners.InterProcessLock(lockfile)
......@@ -153,7 +157,7 @@ def main():
return controller.DockerController(docker_host, swarm_host, mysql_host,
registry, env, datastore_path, sandbox_path,
toolbox_path, sandbox_network, redis_host,
job_user).run()
job_user, import_registry).run()
except config_reader.ConfigError:
log.critical("bad config")
sys.exit(1)
......
......@@ -9,7 +9,7 @@ RUN apt-getq update && apt-getq install \
nginx-light zip gcc python3-dev python3-pip python3-wheel python3-mysqldb \
python-mysqldb python3-crypto gunicorn3 python3-redis python-mysqldb \
python3-crypto python3-natsort python3-aiohttp python3-aioredis supervisor \
python3-ipy python3-django-taggit
python3-ipy python3-django-taggit python3-iso8601
COPY requirements.txt /tmp/
RUN cd /tmp && pip3 install -r requirements.txt && rm requirements.txt
......
......@@ -15,6 +15,7 @@ from .models import (
WebappParameter,
WebappVersion,
)
from .validators import docker_name_validator
class UserForm(forms.ModelForm):
......@@ -217,3 +218,10 @@ class WebappSandboxForm(forms.ModelForm):
class Meta:
model = WebappVersion
fields = ('number', 'description')
class WebappImportForm(forms.Form):
webapp_id = forms.IntegerField(label="Webapp ID", required=False)
docker_name = forms.CharField(label="Short name", required=False,
validators=[docker_name_validator])
......@@ -232,6 +232,10 @@ class Webapp(TimeStampModel):
memory_limit = models.BigIntegerField(null=True,
validators=[MinValueValidator(0)])
# flag indicating if this webbapp was imported from rails
# (if True, then we can import webapp versions)
imported = models.BooleanField(default=False)
# Relationships
# A webapp has one docker os type
......@@ -348,6 +352,8 @@ class WebappVersion(TimeStampModel):
USER = 5 # this version is being pushed directly by the user
IMPORT = 6 # this version is being imported from rails
SANDBOX_STATE_CHOICES = (
(SANDBOX, 'SANDBOX'),
(COMMITTED, 'COMMITTED'),
......@@ -355,18 +361,19 @@ class WebappVersion(TimeStampModel):
(ERROR, 'ERROR'),
(REPLACED, 'REPLACED'),
(USER, 'USER'),
(IMPORT, 'IMPORT'),
)
# Notes about WebappVersion states
#
# - Allowed state changes:
# - by django:
# (none) -> SANDBOX
# (none) -> USER
# (none) -> SANDBOX,USER,IMPORT
# USER -> READY,ERROR
# READY -> REPLACED
# - by the controller:
# SANDBOX -> COMMITTED,ERROR
# IMPORT -> COMMITTED,ERROR
# COMMITTED -> READY,REPLACED
# READY -> REPLACED
#
......@@ -401,6 +408,9 @@ class WebappVersion(TimeStampModel):
state = models.IntegerField(choices=SANDBOX_STATE_CHOICES)
published = models.BooleanField()
# flag indicating if this version was imported from rails
imported = models.BooleanField(default=False)
webapp = models.ForeignKey('Webapp', on_delete=models.CASCADE, related_name="webapp")
class Meta:
......
......@@ -49,6 +49,8 @@ urlpatterns = [
url(r'^apps/$', views.WebappList.as_view(), name='webapp_list'),
url(r'^apps/_authors/(?P<username>[\w.@+-]+)/$', views.UserWebappList.as_view(), name='user_webapp_list'),
url(r'^apps/_create/$', views.WebappCreate.as_view(), name='webapp_creation'),
url(r'^apps/_import/$', views.WebappImport.as_view(), name='webapp_import'),
url(r'^apps/(?P<docker_name>[\w-]+)/import$', views.WebappVersionImport.as_view(), name="webapp_version_import"),
url(r'^apps/(?P<docker_name>[\w-]+)/update$', views.WebappUpdate.as_view(), name="webapp_update"),
url(r'^apps/(?P<docker_name>[\w-]+)/sandbox$', views.WebappSandboxPanel.as_view(), name="webapp_sandbox_panel"),
url(r'^apps/(?P<docker_name>[\w-]+)/json$', views.WebappJson.as_view(), name='webapp_json'),
......
This diff is collapsed.
{% extends "base.html" %}
{% load htmlattrs converters %}
{% block title %}Import a webapp{% endblock %}
{% block breadcrumb %}
<li class="breadcrumb-item"><a href="{% url 'main:webapp_list' %}">Applications</a></li>
<li class="breadcrumb-item active" aria-current="page">Import</li>
{% endblock %}
{% block content %}
<div class="container">
<div class="allgo-page">
<h3>Import an application from <a href="{{import_url}}">{{ import_url }}</a></h3>
<p>Please provide either the ID or the short name of the application you
wish to import.</p>
<form method="post">
{% csrf_token %}
<div class="form-group">
{{ form.webapp_id.label_tag }} <i>(as in url {{ import_url }}/webapps/&lt;ID&gt;)</i>
{% if form.webapp_id.errors %}
{{ form.webapp_id | add_class:"form-control is-invalid" }}
{% else %}
{{ form.webapp_id | add_class:"form-control" }}
{% endif %}
</div>
<div class="form-group">
{{ form.docker_name.label_tag }} <i>(as in url {{ import_url }}/app/&lt;NAME&gt;)</i>
{% if form.docker_name.errors %}
{{ form.docker_name | add_class:"form-control is-invalid" }}
{% else %}
{{ form.docker_name | add_class:"form-control" }}
{% endif %}
</div>
<input class="btn btn-primary" type="submit" value="Import webapp">
</form>
</div>
</div>
{% endblock %}
{% block messages %}
{{ block.super }}
{% include 'partials/_form_messages.html' %}
{% endblock %}
......@@ -32,6 +32,16 @@
<i class="fa-inverse fas fa-plus" data-fa-transform="shrink-7 down-.25 left-.25"></i>
<span class="text-hide">Create application</span>
</a>
<a
href="{% url 'main:webapp_import' %}"
data-toggle="tooltip"
data-placement="top"
title="Import application"
class="fa-layers fa-2x">
<i class="fas fa-square"></i>
<i class="fa-inverse fas fa-cloud-download-alt" data-fa-transform="shrink-7 down-.25 left-1.75"></i>
<span class="text-hide">Import application</span>
</a>
{% endif %}
</div>
</div>
......
......@@ -16,8 +16,14 @@
<div class="col-10">
<h1 class="page-title">
<span>Application profile</span>
<a class="btn btn-primary float-right"
<span class="float-right">
<a class="btn btn-primary"
href="{% url 'main:webapp_sandbox_panel' webapp.docker_name %}">Create a new version</a>
{% if webapp.imported %}
<a class="btn btn-primary"
href="{% url 'main:webapp_version_import' webapp.docker_name %}"><i class="fas fa-cloud-download-alt"></i> Import versions</a>
</span>
{% endif %}
</h1>
<div class="row">
......
{% extends "base.html" %}
{% load htmlattrs static %}
{% block title %}Import versions for application {{ webapp.name }}{% endblock %}
{% block breadcrumb %}
<li class="breadcrumb-item"><a href="{% url 'main:user_webapp_list' user.get_username %}">Applications</a></li>
<li class="breadcrumb-item"><a href="{% url 'main:webapp_detail' webapp.docker_name %}">{{ webapp.name }}</a></li>
<li class="breadcrumb-item active" aria-current="page">Import versions</li>
{% endblock %}
{% block content %}
<div class="container">
<div class="allgo-page">
<h3>Import versions for {{webapp.name}}</h3>
<p>In this page you can request the import of versions of the
<b>{{ webapp.name }}</b> application hosted at
<a href="{{import_url}}">{{ import_url }}</a>.</p>
<p>You may import a version that already exists locally (this is especially
useful in case the remote version was updated). However be aware that this
will permanently overwrite the local image.</p>
<p>It is not recommended to import a version while a commit (with the same
version number) is in progress on the remote server. We cannot guarantee
which image would be imported (the version before or the version after the
commit).</p>
<p>The import process is fully asynchronous, you may need to refresh this
page to have some feedback </p>
{% if not versions %}
<p><b>The remote webapp has no versions !</b></p>
{% else %}
<form method="post">
{% csrf_token %}
<table class="table table-striped">
<thead>
<tr>
<th scope="col" rowspan="2">Number</th>
<th scope="col">Remote version</th>
<th scope="col" colspan="2">Local version</th>
<th scope="col" rowspan="2">Select</th>
</tr>
<tr>
<th scope="col">timestamp</th>
<th scope="col">timestamp</th>
<th scope="col">was imported</th>
</tr>
</thead>
<tbody>
{% for version in versions %}
<tr>
<td>{{ version.number }}</td>
<td>{{ version.remote_ts }} </td>
<td>{{ version.local_ts }}</td>
<td>{% if version.local_imported %}<i class="fas fa-check text-success"></i>{% endif %}</td>
<td>
{% if version.in_progress %}
<i class="text-secondary">in progress</i>
{% elif version.local_imported and version.local_ts == version.remote_ts %}
<i class="text-success">up to date</i>
{% else %}
<input type="checkbox" name="version_{{version.number}}" aria-label="import version {{ version.number }}"/>
{% if version.local_ts and not version.local_imported %}
<i class="fas fa-exclamation-triangle text-danger"
data-toggle="tooltip"
data-placememt="top"
title="Version {{ version.number }} was updated locally,
a new import will overwrite the changes"
></i>
{% endif %}
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
<p class="text-right"><input class="btn btn-primary" type="submit" value="Import selected versions"></p>
</form>
{% endif %}
</div>
</div>
{% endblock %}
{% block messages %}
{{ block.super }}
{% include 'partials/_form_messages.html' %}
{% endblock %}
{% block javascript %}
{{ block.super }}
<script defer src="{% static 'js/tooltip.js' %}"></script>
{% endblock %}
......@@ -187,6 +187,11 @@ with env_loader.EnvironmentVarLoader(__name__, "ALLGO_",
env_var("ALLGO_WEBAPP_DEFAULT_MEMORY_LIMIT_MB", default=str(4*1024),
help="default memory limit (in megabytes) for newly created webapps")
env_var("ALLGO_IMPORT_URL", default="https://allgo.inria.fr",
help="url of the legacy allgo instance (for importing webapps)")
env_var("ALLGO_IMPORT_REGISTRY", default="cargo.irisa.fr:8003/allgo/prod/webapp",
help="registry of the legacy allgo instance (for importing webapps)")
#
# allgo authentication tokens
#
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment