Attention une mise à jour du service Gitlab va être effectuée le mardi 30 novembre entre 17h30 et 18h00. Cette mise à jour va générer une interruption du service dont nous ne maîtrisons pas complètement la durée mais qui ne devrait pas excéder quelques minutes. Cette mise à jour intermédiaire en version 14.0.12 nous permettra de rapidement pouvoir mettre à votre disposition une version plus récente.

Commit 2bfee87a authored by Robin Tissot's avatar Robin Tissot
Browse files

Merge branch 'develop'

parents 0936637d b7aadb72
......@@ -81,14 +81,6 @@ class UserOnboardingSerializer(serializers.ModelSerializer):
model = User
fields = ('onboarding',)
def __init__(self, user, *args, **kwargs):
self.user = user
super().__init__(*args, **kwargs)
def complete(self):
self.user.onboarding = self.validated_data['onboarding']
self.user.save()
class BlockTypeSerializer(serializers.ModelSerializer):
class Meta:
......
......@@ -11,6 +11,26 @@ from django.urls import reverse
from core.models import Block, Line, Transcription, LineTranscription
from core.tests.factory import CoreFactoryTestCase
class UserViewSetTestCase(CoreFactoryTestCase):
def setUp(self):
super().setUp()
def test_onboarding(self):
user = self.factory.make_user()
self.client.force_login(user)
uri = reverse('api:user-onboarding')
resp = self.client.put(uri, {
'onboarding' : 'False',
}, content_type='application/json')
user.refresh_from_db()
self.assertEqual(resp.status_code, 200)
self.assertEqual(user.onboarding, False)
class DocumentViewSetTestCase(CoreFactoryTestCase):
def setUp(self):
......
......@@ -14,7 +14,7 @@ from api.views import (DocumentViewSet,
router = routers.DefaultRouter()
router.register(r'documents', DocumentViewSet)
router.register(r'users', UserViewSet)
router.register(r'user', UserViewSet)
router.register(r'types/block', BlockTypeViewSet)
router.register(r'types/line', LineTypeViewSet)
documents_router = routers.NestedSimpleRouter(router, r'documents', lookup='document')
......
......@@ -50,9 +50,9 @@ class UserViewSet(ModelViewSet):
@action(detail=False, methods=['put'])
def onboarding(self, request):
serializer = UserOnboardingSerializer(data=request.data, user=self.request.user)
serializer = UserOnboardingSerializer(self.request.user,data=request.data, partial=True)
if serializer.is_valid(raise_exception=True):
serializer.complete()
serializer.save()
return Response(status=status.HTTP_200_OK)
......@@ -70,6 +70,9 @@ class DocumentViewSet(ModelViewSet):
def form_error(self, msg):
return Response({'status': 'error', 'error': msg}, status=400)
def perform_create(self, serializer):
serializer.save(owner=self.request.user)
@action(detail=True, methods=['post'])
def imports(self, request, pk=None):
document = self.get_object()
......@@ -79,7 +82,7 @@ class DocumentViewSet(ModelViewSet):
form.save() # create the import
try:
form.process()
except ParseError as e:
except ParseError:
return self.form_error("Incorrectly formated file, couldn't parse it.")
return Response({'status': 'ok'})
else:
......
# Generated by Django 2.1.4 on 2020-09-17 13:02
import uuid
from django.db import migrations
def forward(apps, se):
Block = apps.get_model('core', 'Block')
Line = apps.get_model('core', 'Line')
for block in Block.objects.filter(external_id=None):
block.external_id = 'eSc_textblock_%s' % str(uuid.uuid4())[:8]
block.save()
for line in Line.objects.filter(external_id=None):
line.external_id = 'eSc_line_%s' % str(uuid.uuid4())[:8]
line.save()
def backward(apps, se):
# no need to do anything
pass
class Migration(migrations.Migration):
dependencies = [
('core', '0040_link_default_typology'),
]
operations = [
migrations.RunPython(forward, backward),
]
# Generated by Django 2.1.4 on 2020-10-09 10:15
import core.models
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('core', '0041_create_external_ids'),
]
operations = [
migrations.AlterField(
model_name='block',
name='box',
field=django.contrib.postgres.fields.jsonb.JSONField(validators=[core.models.validate_polygon, core.models.validate_3_points]),
),
migrations.AlterField(
model_name='line',
name='baseline',
field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, validators=[core.models.validate_polygon, core.models.validate_2_points]),
),
migrations.AlterField(
model_name='line',
name='block',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='lines', to='core.Block'),
),
migrations.AlterField(
model_name='line',
name='mask',
field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, validators=[core.models.validate_polygon, core.models.validate_3_points]),
),
]
......@@ -5,11 +5,12 @@ import os
import json
import functools
import subprocess
import uuid
from PIL import Image
from datetime import datetime
from django.db import models, transaction
from django.db.models import Q
from django.db.models import Q, Prefetch
from django.db.models.signals import pre_delete
from django.conf import settings
from django.contrib.auth import get_user_model
......@@ -332,9 +333,6 @@ class DocumentPart(OrderedModel):
except IndexError:
return False
def make_external_id(self):
return 'eSc_page_%d' % self.pk
@property
def filename(self):
return self.original_filename or os.path.split(self.image.path)[1]
......@@ -641,15 +639,6 @@ class DocumentPart(OrderedModel):
res = blla.segment(im, **options)
if steps in ['lines', 'both']:
# line_types = {t.name: t for t in self.document.valid_line_types.all()}
for line in res['lines']:
mask = line['boundary'] if line['boundary'] is not None else None
Line.objects.create(
document_part=self,
# typology=line_types.get(line['type']),
baseline=line['baseline'],
mask=mask)
if steps in ['regions', 'both']:
block_types = {t.name: t for t in self.document.valid_block_types.all()}
for region_type, regions in res['regions'].items():
......@@ -658,6 +647,18 @@ class DocumentPart(OrderedModel):
document_part=self,
typology=block_types.get(region_type),
box=region)
if steps in ['lines', 'both']:
line_types = {t.name: t for t in self.document.valid_line_types.all()}
for line in res['lines']:
mask = line['boundary'] if line['boundary'] is not None else None
Line.objects.create(
document_part=self,
typology=line_types.get(line['script']),
# region=region_map.get(line['region']),
baseline=line['baseline'],
mask=mask)
im.close()
self.workflow_state = self.WORKFLOW_STATE_SEGMENTED
......@@ -784,13 +785,27 @@ def validate_polygon(value):
params={'value': value})
def validate_2_points(value):
if len(value) < 2:
raise ValidationError(
_('Polygon needs to have at least 2 points, it has %(length)d: %(value)s.'),
params={'length': len(value), 'value': value})
def validate_3_points(value):
if len(value) < 3:
raise ValidationError(
_('Polygon needs to have at least 3 points, it has %(length)d: %(value)s.'),
params={'length': len(value), 'value': value})
class Block(OrderedModel, models.Model):
"""
Represents a visualy close group of graphemes (characters) bound by the same semantic
example: a paragraph, a margin note or floating text
"""
# box = models.BoxField() # in case we use PostGIS
box = JSONField(validators=[validate_polygon])
box = JSONField(validators=[validate_polygon, validate_3_points])
typology = models.ForeignKey(BlockType, null=True, blank=True,
on_delete=models.SET_NULL)
document_part = models.ForeignKey(DocumentPart, on_delete=models.CASCADE,
......@@ -805,8 +820,8 @@ class Block(OrderedModel, models.Model):
@property
def coordinates_box(self):
"""
Cast the box field to the format [xmin,ymin,xmax,ymax]
to make it usable to calculate VPOS,HPOS,WIDTH, HEIGHT for Alto
Cast the box field to the format [xmin, ymin, xmax, ymax]
to make it usable to calculate VPOS, HPOS, WIDTH, HEIGHT for Alto
"""
return [*map(min, *self.box), *map(max, *self.box)]
......@@ -819,7 +834,22 @@ class Block(OrderedModel, models.Model):
return self.coordinates_box[3] - self.coordinates_box[1]
def make_external_id(self):
return self.external_id or 'eSc_textblock_%d' % self.pk
self.external_id = 'eSc_textblock_%s' % str(uuid.uuid4())[:8]
def save(self, *args, **kwargs):
if self.external_id is None:
self.make_external_id()
return super().save(*args, **kwargs)
class LineManager(models.Manager):
def prefetch_transcription(self, transcription):
return (self.get_queryset().order_by('order')
.prefetch_related(
Prefetch('transcriptions',
to_attr='transcription',
queryset=LineTranscription.objects.filter(
transcription=transcription))))
class Line(OrderedModel): # Versioned,
......@@ -828,13 +858,13 @@ class Line(OrderedModel): # Versioned,
"""
# box = gis_models.PolygonField() # in case we use PostGIS
# Closed Polygon: [[x1, y1], [x2, y2], ..]
mask = JSONField(null=True, blank=True, validators=[validate_polygon])
mask = JSONField(null=True, blank=True, validators=[validate_polygon, validate_3_points])
# Polygon: [[x1, y1], [x2, y2], ..]
baseline = JSONField(null=True, blank=True, validators=[validate_polygon])
baseline = JSONField(null=True, blank=True, validators=[validate_polygon, validate_2_points])
document_part = models.ForeignKey(DocumentPart,
on_delete=models.CASCADE,
related_name='lines')
block = models.ForeignKey(Block, null=True, blank=True, on_delete=models.SET_NULL)
block = models.ForeignKey(Block, null=True, blank=True, on_delete=models.SET_NULL, related_name='lines')
script = models.CharField(max_length=8, null=True, blank=True) # choices ??
# text direction
order_with_respect_to = 'document_part'
......@@ -845,6 +875,8 @@ class Line(OrderedModel): # Versioned,
external_id = models.CharField(max_length=128, blank=True, null=True)
objects = LineManager()
class Meta(OrderedModel.Meta):
pass
......@@ -871,10 +903,15 @@ class Line(OrderedModel): # Versioned,
(box[2], box[3]),
(box[2], box[1])]
box = property(get_box, set_box)
box = cached_property(get_box, set_box)
def make_external_id(self):
return self.external_id or 'eSc_line_%d' % self.pk
self.external_id = 'eSc_line_%s' % str(uuid.uuid4())[:8]
def save(self, *args, **kwargs):
if self.external_id is None:
self.make_external_id()
return super().save(*args, **kwargs)
class Transcription(models.Model):
......
......@@ -163,6 +163,6 @@ const TranscriptionModal = Vue.component('transcriptionmodal', {
} else {
overlay.style.display = 'none';
}
},
}
},
});
......@@ -55,7 +55,9 @@ var partVM = new Vue({
}
},
selectedTranscription: function(n, o) {
userProfile.set('default-transcription-' + DOCUMENT_ID, n);
let itrans = userProfile.get('initialTranscriptions') || {};
itrans[DOCUMENT_ID] = n;
userProfile.set('initialTranscriptions', itrans);
this.getCurrentContent(n);
},
comparedTranscriptions: function(n, o) {
......@@ -82,7 +84,8 @@ var partVM = new Vue({
created() {
// this.fetch();
this.part.fetchPart(PART_ID, function() {
let tr = userProfile.get('default-transcription-' + DOCUMENT_ID)
let tr = userProfile.get('initialTranscriptions')
&& userProfile.get('initialTranscriptions')[DOCUMENT_ID]
|| this.part.transcriptions[0].pk;
this.selectedTranscription = tr;
}.bind(this));
......
......@@ -3,16 +3,16 @@
var API = {
'document': '/api/documents/' + DOCUMENT_ID,
'parts': '/api/documents/' + DOCUMENT_ID + '/parts/',
'part': '/api/documents/' + DOCUMENT_ID + '/parts/{part_pk}/'
'part': '/api/documents/' + DOCUMENT_ID + '/parts/{part_pk}/'
};
Dropzone.autoDiscover = false;
var g_dragged = null; // Note: chrome doesn't understand dataTransfer very well
var lastSelected = null;
function openWizard(proc) {
function openWizard(proc) {
var selected_num = partCard.getSelectedPks().length;
if(!proc.startsWith('import') && selected_num < 1) {
alert('Select at least one image.');
return;
......@@ -22,16 +22,17 @@ function openWizard(proc) {
// can't send more than one binarized image at a time
if (selected_num != 1) { $('#id_bw_image').attr('disabled', true); }
else { $('#id_bw_image').attr('disabled', false); }
// Reset the form
$('.process-part-form', '#'+proc+'-wizard').get(0).reset();
// initialize transcription field with user's last edited transcription
let itrans = userProfile.get('initialTranscriptions');
if (itrans && itrans[DOCUMENT_ID]) {
$('#process-part-form-export #id_transcription').val(itrans[DOCUMENT_ID]);
$('#process-part-form-train #id_transcription').val(itrans[DOCUMENT_ID]);
}
$('#'+proc+'-wizard').modal('show');
}
......@@ -50,7 +51,7 @@ class partCard {
this.locked = false;
this.api = API.part.replace('{part_pk}', this.pk);
var $new = $('.card', '#card-template').clone();
this.$element = $new;
this.domElement = this.$element.get(0);
......@@ -58,7 +59,7 @@ class partCard {
this.selectButton = $('.js-card-select-hdl', $new);
this.deleteButton = $('.js-card-delete', $new);
this.dropAfter = $('.js-drop', '#card-template').clone();
// fill template
$new.attr('id', $new.attr('id').replace('{pk}', this.pk));
this.updateThumbnail();
......@@ -73,7 +74,7 @@ class partCard {
}, this)).on('mouseout', $.proxy(function(ev) {
this.$element.attr('draggable', true);
}, this));
// add to the dom
$('#cards-container').append($new);
$('#cards-container').append(this.dropAfter);
......@@ -96,7 +97,7 @@ class partCard {
this.cancelTasksButton.click($.proxy(function(ev) {
this.cancelTasks();
}, this));
this.binarizedButton.click($.proxy(function(ev) {
this.select();
partCard.refreshSelectedCount();
......@@ -112,16 +113,16 @@ class partCard {
partCard.refreshSelectedCount();
openWizard('transcribe');
}, this));
this.index = $('.card', '#cards-container').index(this.$element);
// save a reference to this object in the card dom element
$new.data('partCard', this);
// add the image element to the lazy loader
imageObserver.observe($('img', $new).get(0));
this.defaultColor = this.$element.css('color');
//************* events **************
this.selectButton.on('click', $.proxy(function(ev) {
if (ev.shiftKey) {
......@@ -142,12 +143,12 @@ class partCard {
this.delete();
partCard.refreshSelectedCount();
}, this));
this.$element.on('dblclick', $.proxy(function(ev) {
this.toggleSelect();
partCard.refreshSelectedCount();
}, this));
// drag'n'drop
this.$element.on('dragstart', $.proxy(function(ev) {
if (this.locked) return;
......@@ -157,7 +158,7 @@ class partCard {
}, this));
this.$element.on('dragend', $.proxy(function(ev) {
$('.js-drop').removeClass('drop-target');
}, this));
}, this));
}
inQueue() {
......@@ -174,7 +175,7 @@ class partCard {
this.workflow['segment'] == 'ongoing' ||
this.workflow['transcribe'] == 'ongoing');
}
isCancelable() {
return (this.workflow['binarize'] == 'ongoing' ||
this.workflow['segment'] == 'ongoing' ||
......@@ -186,7 +187,7 @@ class partCard {
updateThumbnail() {
let uri, img = $('img.card-img-top', this.$element);
if (this.image.thumbnails && this.image.thumbnails['card'] != undefined) {
uri = this.image.thumbnails['card'];
} else {
......@@ -196,7 +197,7 @@ class partCard {
if (img.attr('src')) img.attr('src', uri);
img.attr('data-src', uri);
}
updateWorkflowIcons() {
var map = [
['convert', this.convertIcon],
......@@ -212,9 +213,9 @@ class partCard {
btn.removeClass('pending').removeClass('ongoing').removeClass('error').removeClass('done');
btn.addClass(this.workflow[proc]);
btn.attr('title', btn.data('title') + ' ('+this.workflow[proc]+')');
}
}
}
if (this.inQueue() || this.working()) {
this.lock();
} else {
......@@ -232,7 +233,7 @@ class partCard {
this.dropAfter.remove();
this.$element.remove();
}
select(scroll=true) {
if (this.locked) return;
lastSelected = this;
......@@ -266,7 +267,7 @@ class partCard {
this.$element.removeClass('locked');
this.$element.attr('draggable', true);
}
moveTo(index, upload) {
if (upload === undefined) upload = true;
// store the previous index in case of error
......@@ -297,7 +298,7 @@ class partCard {
this.updateWorkflowIcons();
}, this)).fail($.proxy(function(data){console.log("Couldn't cancel the task.");}));
}
delete() {
var posting = $.ajax({url:this.api, type: 'DELETE'})
.done($.proxy(function(data) {
......@@ -346,12 +347,12 @@ $(document).ready(function() {
ev.preventDefault();
}
});
$('#cards-container').on('dragleave','.js-drop', function(ev) {
ev.preventDefault();
$(ev.target).removeClass('drop-accept');
});
$('#cards-container').on('drop', '.js-drop', function(ev) {
ev.preventDefault();
$(ev.target).removeClass('drop-accept');
......@@ -363,7 +364,7 @@ $(document).ready(function() {
card.moveTo(index);
g_dragged = null;
});
// update workflow icons, send by notification through web socket
var workflow_order = ['pending', 'ongoing', 'error', 'done'];
$('#alerts-container').on('part:workflow', function(ev, data) {
......@@ -377,7 +378,7 @@ $(document).ready(function() {
if (data.task_id) card.task_ids[data.process] = data.task_id;
card.updateWorkflowIcons();
// special case, done with thumbnails:
if (data.process == 'generate_part_thumbnails' && data.status == 'done') {
card.image.thumbnails = data.data;
......@@ -428,15 +429,16 @@ $(document).ready(function() {
Alert.add(Date.now(), data.reason, 'warning');
});
$alertsContainer.on('import:fail', function(ev, data) {
$('#import-counter').text('failed');
$('#import-counter').text('Failed.');
$('#import-selected').removeClass('blink');
$('#cancel-import').hide();
Alert.add('import-failed', "Import failed because '"+data.reason+"'", 'danger');
});
$alertsContainer.on('import:done', function(ev, data) {
$('#import-counter').text('Done.');
$('#import-counter').parent().removeClass('ongoing');
$('#import-selected').removeClass('blink');
Alert.add('import-done', "Import finished!", 'success');
$('#cancel-import').hide();
});
$('#cancel-import').click(function(ev, data) {
let url = API.document + '/cancel_import/';
......@@ -450,7 +452,7 @@ $(document).ready(function() {
console.log("Couldn't cancel import");
});
});
// Exports
var $exportBtn = $('#document-export');
$alertsContainer.on('export:start', function(ev, data) {
......@@ -463,7 +465,18 @@ $(document).ready(function() {
$alertsContainer.on('export:done', function(ev, data) {
$exportBtn.removeClass('blink');
});
// disables including images for text export
$("#process-part-form-export #id_file_format").on('change', function(ev) {
let sel = ev.target;