Une nouvelle version du portail de gestion des comptes externes sera mise en production lundi 09 août. Elle permettra d'allonger la validité d'un compte externe jusqu'à 3 ans. Pour plus de détails sur cette version consulter : https://doc-si.inria.fr/x/FCeS

Commit 7f216770 authored by GARNIER Laurent's avatar GARNIER Laurent
Browse files

Merge branch 'webapp-version-management' into 'django'

support edition/deletion/restoration of webapp versions

Closes allgo.inria.fr#5, #44, and #266

See merge request !206
parents c5689bf4 869c7984
Pipeline #136772 failed with stages
in 1 second
......@@ -1635,7 +1635,7 @@ class PushManager(Manager):
# get the version object and check its state
version = ses.query(WebappVersion).filter_by(id=version_id).one()
if version.state != VersionState.COMMITTED:
if version.state in (VersionState.READY, VersionState.REPLACED):
if version.state in (VersionState.READY, VersionState.DELETED):
# already pushed
return
if version.state in (VersionState.SANDBOX, VersionState.IMPORT):
......@@ -1675,7 +1675,7 @@ class PushManager(Manager):
#
# We set the READY state to the version with the highest id
# (the one which was committed/pushed last) and put all other
# versions in the REPLACED state.
# versions in the DELETED state.
# select and lock all candidate versions
versions = ses.query(WebappVersion).filter_by(id=version.id).union(
......@@ -1685,11 +1685,14 @@ class PushManager(Manager):
state = int(VersionState.READY))
).with_for_update().all()
# set the latest one to READY and the others to REPLACED
# set the latest one to READY and the others to DELETED
latest_id = max(v.id for v in versions)
for v in versions:
new_state = (VersionState.READY if v.id == latest_id else
VersionState.REPLACED)
if v.id == latest_id:
new_state = VersionState.READY
else:
new_state = VersionState.DELETED
v.deleted_at=datetime.datetime.now()
log.debug("version id %d: %s -> %s", v.id,
VersionState(v.state).name, new_state)
v.state = int(new_state)
......@@ -1736,7 +1739,7 @@ class ImageManager:
with ses.begin():
version = ses.query(WebappVersion).filter_by(id=version_id).one()
if version.state not in (VersionState.READY, VersionState.REPLACED):
if version.state not in (VersionState.READY, VersionState.DELETED):
raise Error("bad version state: %s" % version.state)
yield from self.swarm_pull_manager.process((image, tag))
......@@ -1747,7 +1750,7 @@ class ImageManager:
# nothing to do
return
if version.state not in (VersionState.READY, VersionState.REPLACED):
if version.state not in (VersionState.READY, VersionState.DELETED):
raise Error("bad version state: %s" % version.state)
yield from self.sandbox_pull_manager.process((image, tag))
......
......@@ -25,7 +25,7 @@ class VersionState(enum.IntEnum):
COMMITTED = 1
READY = 2
ERROR = 3
REPLACED = 4
DELETED = 4
USER = 5
IMPORT = 6
......@@ -80,6 +80,7 @@ class WebappVersion(Base):
webapp_id = Column(Integer, ForeignKey('dj_webapps.id'))
created_at = Column(DateTime, default=datetime.datetime.now)
updated_at = Column(DateTime, default=datetime.datetime.now)
deleted_at = Column(DateTime, nullable=True)
number = Column(String)
description = Column(String)
published = Column(Boolean)
......
......@@ -1595,7 +1595,7 @@ class ControllerTestCase(unittest.TestCase):
self.assertFalse(
ses.query(WebappVersion.id, WebappVersion.state)
.filter_by(webapp_id=app.id)
.filter(WebappVersion.state.notin_((int(V.READY), int(V.REPLACED)))).all())
.filter(WebappVersion.state.notin_((int(V.READY), int(V.DELETED)))).all())
with part("case 1: image replaced but not yet committed -> use previous image"), \
interleave("docker.Client.commit") as commit_interleave:
......@@ -1612,7 +1612,7 @@ class ControllerTestCase(unittest.TestCase):
self.check_job_output(job, rnd+"foo\n\n==== ALLGO JOB SUCCESS ====\n")
# wait until the image is replaced
wait_until(lambda: get_version(ver_id).state == V.REPLACED, timeout=10)
wait_until(lambda: get_version(ver_id).state == V.DELETED, timeout=10)
ensure_version_ready()
......@@ -1632,7 +1632,7 @@ class ControllerTestCase(unittest.TestCase):
self.check_job_output(job, rnd+"bar1\n\n==== ALLGO JOB SUCCESS ====\n")
# wait until the image is replaced
wait_until(lambda: get_version(ver_id).state == V.REPLACED, timeout=10)
wait_until(lambda: get_version(ver_id).state == V.DELETED, timeout=10)
ensure_version_ready()
......@@ -1672,8 +1672,8 @@ class ControllerTestCase(unittest.TestCase):
self.add_dummy_version(app, "2.0", state=V.READY)
self.add_dummy_version(app, "2.1", state=V.READY)
replaced = (
self.add_dummy_version(app, "3.0", state=V.REPLACED).id,
self.add_dummy_version(app, "3.1", state=V.REPLACED).id,
self.add_dummy_version(app, "3.0", state=V.DELETED).id,
self.add_dummy_version(app, "3.1", state=V.DELETED).id,
)
with mock.patch("controller.ImageManager.push") as m_push:
......
......@@ -16,6 +16,7 @@ from .models import (
Token,
Webapp,
WebappParameter,
WebappVersion,
)
from .validators import docker_name_validator
......@@ -230,3 +231,15 @@ class WebappTokenForm(forms.ModelForm):
class Meta:
model = Token
fields = ("name", "expires_at")
class WebappVersionForm(forms.ModelForm):
number = forms.CharField(label="Version", help_text="""The version number. It must be a valid
docker tag.""")
description = forms.CharField(help_text="An optional description for this version.",
required=False)
published = forms.BooleanField(required=False, help_text="""Tick this box to make this version
usable by other users.""")
class Meta:
model = WebappVersion
fields = ("number", "description", "published")
import base64
import hashlib
import itertools
import logging
import os
import re
......@@ -7,10 +8,16 @@ import re
import redis
import IPy
from django.conf import settings
import django.db
from django.db.models import Q
import config
from .models import Job, Webapp, AllgoUser
from .models import (
AllgoUser,
Job,
Webapp,
WebappVersion,
)
log = logging.getLogger('allgo')
......@@ -218,6 +225,14 @@ def query_webapps_for_user(user):
if user.is_superuser:
return Webapp.objects.all()
else:
# select webapps that are either public or owned by the user
return Webapp.objects.filter(Q(private=False) | Q(user_id=user.id))
# a webapp is visible in the public index if it is not private (obviously) and if it has at
# least one version published and ready.
with django.db.connection.cursor() as cur:
cur.execute("""SELECT webapp_id FROM dj_webapp_versions WHERE webapp_id IN (
SELECT id FROM dj_webapps WHERE private != 1
) AND published=1 AND state=%s GROUP BY webapp_id""", (WebappVersion.READY,))
public_ids = list(itertools.chain(*cur.fetchall()))
return Webapp.objects.filter(Q(user_id=user.id) | Q(id__in=public_ids))
......@@ -348,9 +348,9 @@ class WebappVersion(TimeStampModel):
# - a separate 'recovery' version may have been
# committed by the controller
REPLACED = 4 # this version was replaced (by another READY version)
# - it is assumed to be deleted
# - it is no longer visible by the user
DELETED = 4 # this version is deleted (either overwritten or directly deleted)
# - it is no longer visible by the user, but may be restored within the
# next `ALLGO_EXPUNGE_DELAY` days
# - it may still be in use by a job or a sandbox
USER = 5 # this version is being pushed directly by the user
......@@ -362,7 +362,7 @@ class WebappVersion(TimeStampModel):
(COMMITTED, 'COMMITTED'),
(READY, 'READY'),
(ERROR, 'ERROR'),
(REPLACED, 'REPLACED'),
(DELETED, 'DELETED'),
(USER, 'USER'),
(IMPORT, 'IMPORT'),
)
......@@ -371,14 +371,15 @@ class WebappVersion(TimeStampModel):
#
# - Allowed state changes:
# - by django:
# (none) -> SANDBOX,USER,IMPORT
# USER -> READY,ERROR
# READY -> REPLACED
# (none) -> SANDBOX,USER,IMPORT (commit/push/import started)
# USER -> READY,ERROR (push terminated)
# READY -> DELETED (deletion/replacement)
# DELETED -> READY (user restore)
# - by the controller:
# SANDBOX -> COMMITTED,ERROR
# IMPORT -> COMMITTED,ERROR
# COMMITTED -> READY,REPLACED
# READY -> REPLACED
# SANDBOX -> COMMITTED,ERROR (sandbox committed)
# IMPORT -> COMMITTED,ERROR (image pulled from the origin registry)
# COMMITTED -> READY,DELETED (image pushed to the internal registry)
# READY -> DELETED (version replaced)
#
# - As soon as the version is in the SANDBOX, the version is considered to
# exist and to be usable for launching a job or a sandbox.
......@@ -387,7 +388,7 @@ class WebappVersion(TimeStampModel):
# if a version is uptated multiples times quickly, then the images may
# not reach the READY state in the same order. If multiple images with
# the same version number reach the READY state, then only the higest id
# is kept at the READY state, older versions are switched to REPLACED.
# is kept at the READY state, older versions are switched to DELETED.
# Note: switching to the READY/REPLACING state must be done atomically
# (by locking the db rows) because it may be done by django (when
# pushing) or the controller (when committing a sandbox).
......@@ -397,6 +398,9 @@ class WebappVersion(TimeStampModel):
# WebappVersion entry is created with a different id. Docker images are never
# overwritten.
#
# - The deletion time (when switching to the DELETED state) is recorded in
# the 'deleted_at' column. The user may recover deleted versions up to
# ALLGO_EXPUNGE_DELAY` days.
# Fields
number = models.CharField(max_length=255, validators=[
......@@ -409,6 +413,12 @@ class WebappVersion(TimeStampModel):
description = models.CharField(max_length=255, blank=True)
docker_image_size = models.FloatField(blank=True, null=True)
state = models.IntegerField(choices=SANDBOX_STATE_CHOICES)
# deletion time, it is meaningful only when in the DELETED state
# Note: old (pre-migration) deleted version have 'deleted_at=None'
deleted_at = models.DateTimeField(blank=True, null=True)
# True if the version is usable by other users.
published = models.BooleanField()
# flag indicating if this version was imported from rails
......
......@@ -7,7 +7,7 @@ from django.utils.safestring import mark_safe
from django.utils.timesince import timesince
from misaka import Markdown, HtmlRenderer, EXT_SUPERSCRIPT, EXT_FENCED_CODE
from main.models import Job
from main.models import Job, WebappVersion
register = template.Library()
......@@ -86,6 +86,27 @@ def status_icon(obj):
raise TypeError(type(obj))
_IN_PROGRESS = ("deployment in progress", "text-primary")
_WEBAPPVERSION_STATUS_RENDER_VALUES = {
WebappVersion.SANDBOX: _IN_PROGRESS,
WebappVersion.COMMITTED: _IN_PROGRESS,
WebappVersion.USER: _IN_PROGRESS,
WebappVersion.IMPORT: _IN_PROGRESS,
WebappVersion.READY: ("ready", "text-success"),
WebappVersion.ERROR: ("error", "text-danger"),
WebappVersion.DELETED: ("deleted", "text-secondary"),
}
@register.filter(name='version_status')
def version_status(webapp_version, verbose):
"""Renders the status of a WebappVersion"""
txt, cls = _WEBAPPVERSION_STATUS_RENDER_VALUES.get(
webapp_version.state, ("unknown", "text-dark"))
result = '<i class="%s">%s</i>' % (cls, txt)
if verbose and txt=="in progress":
result += " (%s)" % webapp_version.get_state_display()
return mark_safe(result)
@register.filter(name='command_multiline')
def command_multiline(cmd):
......@@ -117,3 +138,6 @@ def command_oneline(cmd):
as a single line.
"""
return " ".join(shlex.quote(arg) for arg in cmd if arg is not None)
......@@ -64,6 +64,10 @@ urlpatterns = [
name="webapp_update"),
url(r'^apps/(?P<docker_name>[\w-]+)/sandbox$', views.WebappSandboxPanel.as_view(),
name="webapp_sandbox_panel"),
url(r'^apps/(?P<docker_name>[\w-]+)/versions/$', views.WebappVersionList.as_view(),
name="webapp_version_list"),
url(r'^apps/(?P<docker_name>[\w-]+)/versions/(?P<pk>\d+)/update',
views.WebappVersionUpdate.as_view(), name="webapp_version_update"),
url(r'^apps/(?P<docker_name>[\w-]+)/json$', views.WebappJson.as_view(), name='webapp_json'),
url(r'^apps/(?P<docker_name>[\w-]+)$', views.JobCreate.as_view(), name='webapp_detail'),
......
......@@ -22,6 +22,7 @@ Attributes:
# Python standard libraries
import datetime
import itertools
import json
import logging
import os
import re
......@@ -48,6 +49,7 @@ from django.db.models import Q
from django.http import HttpResponse, JsonResponse, HttpResponseRedirect, FileResponse, Http404
from django.shortcuts import get_object_or_404, redirect
from django.urls import reverse, reverse_lazy
import django.utils
from django.utils.crypto import get_random_string
from django.utils.text import slugify
from django.views.decorators.csrf import csrf_exempt
......@@ -76,6 +78,7 @@ from .forms import (
WebappForm,
WebappImportForm,
WebappTokenForm,
WebappVersionForm,
)
from .helpers import (
get_base_url,
......@@ -534,7 +537,7 @@ class WebappVersionImport(UserAccessMixin, DetailView):
local_versions = {}
for ver in WebappVersion.objects.filter(webapp=webapp).exclude(
state__in=(WebappVersion.ERROR,
WebappVersion.REPLACED)).order_by("id"):
WebappVersion.DELETED)).order_by("id"):
local_versions[ver.number] = ver
# list of versions to be displayed on the page
......@@ -588,6 +591,149 @@ class WebappVersionImport(UserAccessMixin, DetailView):
return HttpResponseRedirect(request.path_info)
class WebappVersionList(AllAccessMixin, ListView):
"""Display the list of versions for a given app
If the current user is not the app owner, the page just list the published versions with limited
information (creation date, description, deployment status)
If the current user is the owner, then the page also includes:
- all unpublised versions
- recently deleted versions
- control buttons for modifyring/deleting/restoring versions
- more details (deletion date, version id)
"""
model = WebappVersion
context_object_name = 'version_list'
template_name = 'webapp_version_list.html'
def get_webapp(self):
return get_object_or_404(Webapp, docker_name=self.kwargs['docker_name'])
def get_queryset(self):
"""get all versions of this webapp"""
webapp = self.get_webapp()
queryset = WebappVersion.objects.filter(webapp=webapp)
if not self.request.user.is_superuser:
if self.request.user == webapp.user:
# filter old deleted version (those that are scheduled for expunged)
# TODO: expunge them for real
limit = datetime.datetime.now() - datetime.timedelta(
int(config.env.ALLGO_EXPUNGE_DELAY))
queryset = queryset.exclude(
Q(state=WebappVersion.DELETED)
& (Q(deleted_at=None) | Q(deleted_at__lt=limit)))
else:
# keep only the versions that are active and published
queryset = queryset.filter(published=True).exclude(
state__in=(WebappVersion.DELETED, WebappVersion.ERROR))
# FIXME: natsort could be better suited here for ordinary users (but order by creation time
# is better in case there are multiple versions with the same number)
return queryset.order_by("-id")
def get_context_data(self, **kwargs):
webapp = Webapp.objects.get(docker_name=self.kwargs['docker_name'])
kwargs['webapp'] = webapp
kwargs['can_edit'] = self.request.user == webapp.user
return super().get_context_data(**kwargs)
def post(self, request, *args, **kwargs):
"""Delete the WebappVersion given in the 'delete' query arg"""
# delete webapp version
version_id = int(request.POST["delete"])
webapp = self.get_webapp()
version = get_object_or_404(WebappVersion, webapp=webapp, id=version_id)
if not(self.request.user == webapp.user or self.request.user.is_superuser):
return HttpResponse(status=403)
if version.state == WebappVersion.READY:
version.deleted_at = django.utils.timezone.now()
version.state = WebappVersion.DELETED
version.save()
elif version.state == WebappVersion.ERROR:
version.delete()
messages.success(request, "Version %s (#%d) deleted" % (version.number, version.id))
return HttpResponseRedirect("")
class WebappVersionUpdate(UserAccessMixin, UpdateView):
"""Update/Restore a webapp version
This views allows editing a webapp, but with two quirks:
- if it is called on a DELETED version, it is assumed that the user wants to restore it
(thus the state is switched to READY on submit)
- if the submitted version collides with an existing READY version with the same number,
then it s assumed that the user wants to replace that version (thus that version is deleted
first)
"""
form_class = WebappVersionForm
template_name = "webapp_version_update.html"
def get_object(self):
self.webapp = get_object_or_404(Webapp, docker_name=self.kwargs["docker_name"],
user=self.request.user)
return get_object_or_404(WebappVersion, id=self.kwargs["pk"], webapp=self.webapp)
def get_context_data(self, **kw):
version = self.object
# list of existing versions (to warn the user before overwriting one)
other_active_versions_json = json.dumps(list(set(v for v, in WebappVersion.objects
.filter(webapp=self.webapp)
.exclude(id=version.id)
.exclude(state__in=(WebappVersion.ERROR, WebappVersion.DELETED))
.values_list("number"))))
return super().get_context_data(webapp=self.webapp, version=version,
other_active_versions_json=other_active_versions_json,
action=("Restore" if version.state == WebappVersion.DELETED else "Update"), **kw)
def get_success_url(self):
return reverse('main:webapp_version_list', args=(self.webapp.docker_name,))
def form_valid(self, form):
versions = WebappVersion.objects.filter(webapp=self.webapp).filter(number=self.object.number)
# lock db rows for all versions of the current app and w/ the same number
# (because we must guarantee that only one WebappVersion is in state READY)
versions.select_for_update().exists()
# current version being updated/restored
version = self.object
version.refresh_from_db(fields=("state",))
# other versions with the same nuber
other_versions = versions.exclude(id=version.id)
# prevent replacing a version not yet deployed (because that would produce a race condition)
if other_versions.filter(state__in=(WebappVersion.SANDBOX, WebappVersion.COMMITTED,
WebappVersion.USER, WebappVersion.IMPORT)).exists():
messages.error(self.request,
"Cannot replace version %r while a deployement is in progress. "
"Please wait until the deployment is complete." % version.number)
return self.form_invalid(form)
if version.state in (WebappVersion.READY, WebappVersion.DELETED):
# delete the version that is currently READY
other_versions.filter(state=WebappVersion.READY
).update(state=WebappVersion.DELETED, deleted_at=django.utils.timezone.now())
if version.state == WebappVersion.DELETED:
# mark the current version as READY
version.state = WebappVersion.READY
version.deleted_at = None
messages.success(self.request, "Version %s (#%d) restored"
% (version.number, version.id))
else:
messages.success(self.request, "Version %s (#%d) updated"
% (version.number, version.id))
return super().form_valid(form)
class WebappJson(UserAccessMixin, DetailView):
"""json variant of the application details
......@@ -652,6 +798,8 @@ class WebappSandboxPanel(UserAccessMixin, TemplateView):
context['versions'] = natsort.versorted(versions.values(), key=lambda v: v.number)
context['versions'].reverse()
# commit form
context['form'] = WebappVersionForm()
# docker parameters
context['repository'] = "%s/%s" % (
......@@ -719,7 +867,7 @@ class WebappSandboxPanel(UserAccessMixin, TemplateView):
webapp=webapp,
number=number,
state=WebappVersion.SANDBOX,
published=True,
published=bool(request.POST.get("published")),
description=request.POST["description"],
**extra)
version.save()
......@@ -1281,6 +1429,7 @@ class JobCreate(AllAccessMixin, SuccessMessageMixin, CreateView):
.filter(webapp=webapp, state__in=( WebappVersion.SANDBOX,
WebappVersion.COMMITTED,
WebappVersion.READY))
.filter(Q() if self.request.user==webapp.user else Q(published=True))
.values_list("number")))
# also list 'sandbox' if the sandbox is running and if the current user
......
......@@ -55,6 +55,16 @@
</a>
{% endif %}
{% endif %}
<a class="fa-layers fa-2x"
href="{% url 'main:webapp_version_list' webapp.docker_name %}"
data-toggle="tooltip"
data-placement="top"
title="List versions">
<i class="fas fa-square"></i>
<i class="fa-inverse fas fa-tasks" data-fa-transform="shrink-7 down-.25 left-.25"></i>
<span class="text-hide">List versions</span>
</a>
</div>
</div>
......
......@@ -126,10 +126,11 @@
<div class="form-group">
<select class="form-control d-none" name="version-select">
{% for version in versions %}
<option data-description="{{version.description}}">{{ version.number }}</option>
<option data-description="{{version.description}}" data-published="{{version.published | yesno:'1,0,0'}}">{{ version.number }}</option>
{% endfor %}
</select>
<input type="text" class="form-control d-none" name="version-new" placeholder="new version number">
<small id="version-help" class="form-text text-muted">{{form.number.help_text}}</small>
</div>
</div>
</div>
......@@ -137,11 +138,23 @@
<div class="form-row">
<div class="col-lg-10">
<div class="form-group">
<label>Description</label>
<input type="text" name="description" class="form-control">
{{form.description.label_tag}}
{{form.description | add_class:"form-control"}}
<small class="form-text text-muted">{{form.description.help_text}}</small>
</div>
</div>
</div>
<div class="form-row">
<div class="form-group">
{{ form.published.label_tag }}
<div class="form-check">
{{ form.published | add_class:"form-check-control" }}
</div>
<small class="form-text text-muted">{{form.published.help_text}}</small>
</div>
</div>
<div class="form-group">
<button type="submit" class="btn btn-primary">
<i class="fas fa-download"></i> Commit</button>
......@@ -299,30 +312,34 @@ json_seq_event_listener("/aio/apps/{{ webapp.docker_name }}/events",
<script>
(function() {
function reset_description(replace) {
function reset_form(replace) {
if (replace) {
$("input[name='description']").val(
$("[name='version-select'] > :selected").attr("data-description"))
var option = $("[name='version-select'] > :selected");
$("input[name='description']").val(option.attr("data-description"));
$("input[name='published']").prop("checked", parseInt(option.attr("data-published")));
} else {
$("input[name='description']").val("")
$("input[name='published']").prop("checked", true)
}
}
function switch_version_display(replace) {
if (replace) {
$("[name='version-select']").removeClass("d-none")
$("[name='version-new']").addClass("d-none")
$("#version-help").addClass("invisible")
} else {
$("#version-help").removeClass("invisible")
$("[name='version-new']").removeClass("d-none")
$("[name='version-select']").addClass("d-none")
}
reset_description(replace);
reset_form(replace);
}