Commit 42e1fcba authored by Robin Tissot's avatar Robin Tissot
Browse files

Merge branch 'develop' into feature/one-block

parents 6b82840a 57af9eb3
CELERY_MAIN_CORES=2
CELERY_LOW_CORES=2
FLOWER_BASIC_AUTH=flower:whatever
\ No newline at end of file
......@@ -2,6 +2,7 @@ import bleach
import logging
from django.conf import settings
from django.db.utils import IntegrityError
from rest_framework import serializers
from easy_thumbnails.files import get_thumbnailer
......@@ -65,6 +66,14 @@ class TranscriptionSerializer(serializers.ModelSerializer):
model = Transcription
fields = ('pk', 'name')
def create(self, data):
document = Document.objects.get(pk=self.context["view"].kwargs["document_pk"])
data['document'] = document
try:
return super().create(data)
except IntegrityError:
return Transcription.objects.get(name=data['name'])
class UserOnboardingSerializer(serializers.ModelSerializer):
class Meta:
......@@ -214,7 +223,6 @@ class LineOrderSerializer(serializers.ModelSerializer):
class DetailedLineSerializer(LineSerializer):
region = BlockSerializer(required=False)
transcriptions = LineTranscriptionSerializer(many=True, required=False)
class Meta(LineSerializer.Meta):
......
from django.urls import include, path
from rest_framework_nested import routers
from rest_framework.authtoken import views
from api.views import (DocumentViewSet,
UserViewSet,
......@@ -29,5 +30,6 @@ urlpatterns = [
path('', include(router.urls)),
path('', include(documents_router.urls)),
path('', include(parts_router.urls)),
path('api-auth/', include('rest_framework.urls', namespace='rest_framework'))
path('api-auth/', include('rest_framework.urls', namespace='rest_framework')),
path('token-auth/', views.obtain_auth_token)
]
......@@ -206,7 +206,7 @@ class LineTypeViewSet(ModelViewSet):
class BlockViewSet(ModelViewSet):
queryset = Block.objects.all()
queryset = Block.objects.select_related('typology')
serializer_class = BlockSerializer
def get_queryset(self):
......@@ -214,15 +214,16 @@ class BlockViewSet(ModelViewSet):
class LineViewSet(ModelViewSet):
queryset = (Line.objects.all()
.select_related('block')
.prefetch_related('transcriptions__transcription'))
serializer_class = DetailedLineSerializer
queryset = (Line.objects.select_related('block')
.select_related('typology'))
def get_queryset(self):
return super().get_queryset().filter(document_part=self.kwargs['part_pk'])
def get_serializer_class(self):
if self.action in ['retrieve', 'list']:
if self.action == 'retrieve':
return DetailedLineSerializer
else: # create
else: # create, list
return LineSerializer
@action(detail=False, methods=['post'])
......@@ -268,7 +269,15 @@ class LineViewSet(ModelViewSet):
return Response(errors,
status=status.HTTP_400_BAD_REQUEST)
return Response(status=200, data=response)
# return Response(status=200, data=response)
# line = get_object_or_404(Line, pk=pk)
# serializer = LineMoveSerializer(line=line, data=request.data)
# if serializer.is_valid():
# serializer.move()
# return Response({'status': 'moved'})
# else:
# return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
class LargeResultsSetPagination(PageNumberPagination):
......
import re
import logging
import math
import os
import json
import functools
......@@ -350,33 +351,54 @@ class DocumentPart(OrderedModel):
Re-order the lines of the DocumentPart depending on read direction.
"""
read_direction = read_direction or self.document.read_direction
imgbox = ((0, 0), (self.image.width, self.image.height))
# imgbox = ((0, 0), (self.image.width, self.image.height))
if read_direction == Document.READ_DIRECTION_RTL:
origin_box = [self.image.width, 0]
else:
origin_box = [0, 0]
def distance(x, y):
return math.sqrt(sum([(a - b) ** 2 for a, b in zip(x, y)]))
def poly_origin_pt(shape):
return min(shape, key=lambda pt: distance(pt, origin_box))
def origin_pt(shape):
def line_origin_pt(line):
if read_direction == Document.READ_DIRECTION_RTL:
return max(shape, key=lambda pt: pt[0])
return line[-1]
else:
return min(shape, key=lambda pt: pt[0])
return line[0]
# fetch all lines and regroup them by block
qs = self.lines.select_related('block').all()
ls = list(qs)
if len(ls) == 0:
return
ords = list(map(lambda l: origin_pt(l.baseline or l.mask)[0], ls))
ords = list(map(lambda l: (line_origin_pt(l.baseline) if l.baseline
else poly_origin_pt(l.mask))[0], ls))
averageLineHeight = (max(ords) - min(ords)) / len(ords)
def cmp_lines(a, b):
# cache origin pts for efficiency
if not hasattr(a, 'origin_pt'):
a.origin_pt = line_origin_pt(a.baseline) if a.baseline else poly_origin_pt(a.mask)
if not hasattr(b, 'origin_pt'):
b.origin_pt = line_origin_pt(b.baseline) if b.baseline else poly_origin_pt(b.mask)
try:
if a.block != b.block:
pt1 = origin_pt(a.block.box) if a.block else origin_pt(a.baseline or a.mask)
pt2 = origin_pt(b.block.box) if b.block else origin_pt(b.baseline or a.mask)
pt1 = poly_origin_pt(a.block.box) if a.block else a.origin_pt
pt2 = poly_origin_pt(b.block.box) if b.block else b.origin_pt
# when comparing blocks we can use the distance
return distance(pt1, origin_box) - distance(pt2, origin_box)
else:
pt1 = origin_pt(a.baseline or a.mask)
pt2 = origin_pt(b.baseline or a.mask)
pt1 = a.origin_pt
pt2 = b.origin_pt
# 2 lines more or less on the same level
if abs(pt1[1] - pt2[1]) < averageLineHeight:
return abs(pt1[0] - origin_pt(imgbox)[0]) - abs(pt2[0] - origin_pt(imgbox)[0])
# # 2 lines more or less on the same level
if abs(pt1[1] - pt2[1]) < averageLineHeight:
return distance(pt1, origin_box) - distance(pt2, origin_box)
return pt1[1] - pt2[1]
except TypeError as e: # invalid line
return 0
......@@ -620,17 +642,21 @@ class DocumentPart(OrderedModel):
res = blla.segment(im, **options)
if steps in ['lines', 'both']:
# line_types = {t.name: t for t in self.document.valid_line_types.all()}
for line in res['lines']:
mask = line['boundary'] if line['boundary'] is not None else None
Line.objects.create(
document_part=self,
# typology=line_types.get(line['type']),
baseline=line['baseline'],
mask=mask)
if steps in ['regions', 'both']:
for region_type in res['regions'].items():
for region in region_type:
block_types = {t.name: t for t in self.document.valid_block_types.all()}
for region_type, regions in res['regions'].items():
for region in regions:
Block.objects.create(
document_part=self,
typology=block_types.get(region_type),
box=region)
im.close()
......
const LineVersion = Vue.extend({
props: ['version', 'previous'],
created() {
this.timeZone = moment.tz.guess();
this.timeZone = this.$parent.timeZone;
},
beoforeDestroy() {
this.timeZone = null; // make sure it's garbage collected
},
computed: {
momentDate() {
......@@ -17,12 +20,10 @@ const LineVersion = Vue.extend({
if (!this.previous) return this.version.data.content;
let diff = Diff.diffChars(this.previous.data.content, this.version.data.content);
return diff.map(function(part){
let color = part.added ? 'green' :
part.removed ? 'red' : '';
if (part.removed) {
return '<small><font color="'+color+'" class="collapse show history-deletion">'+part.value+'</font></small>';
return '<span class="cmp-del">'+part.value+'</span>';
} else if (part.added) {
return '<font color="'+color+'">'+part.value+'</font>';
return '<span class="cmp-add">'+part.value+'</span>';
} else {
return part.value;
}
......
......@@ -72,12 +72,10 @@ const TranscriptionModal = Vue.component('transcriptionmodal', {
if (!this.line.currentTrans) return;
let diff = Diff.diffChars(this.line.currentTrans.content, content);
return diff.map(function(part){
let color = part.added ? 'green' :
part.removed ? 'red' : '';
if (part.removed) {
return '<small><font color="'+color+'" class="collapse show history-deletion">'+part.value+'</font></small>';
return '<span class="cmp-del">'+part.value+'</span>';
} else if (part.added) {
return '<font color="'+color+'">'+part.value+'</font>';
return '<span class="cmp-add">'+part.value+'</span>';
} else {
return part.value;
}
......
......@@ -32,9 +32,11 @@ const VisuPanel = BasePanel.extend({
updateView() {
this.$el.querySelector('svg').style.height = Math.round(this.part.image.size[1] * this.ratio) + 'px';
Vue.nextTick(function() {
this.$refs.visulines.forEach(function(line) {
line.reset();
});
if (this.part.lines.length) {
this.$refs.visulines.forEach(function(line) {
line.reset();
});
}
}.bind(this));
}
}
......
......@@ -48,10 +48,10 @@ var partVM = new Vue({
'$1'+this.part.pk+'$2'));
// set the 'image' tab btn to select the corresponding image
var tabUrl = new URL($('#images-tab-link').attr('href'),
var tabUrl = new URL($('#nav-img-tab').attr('href'),
window.location.origin);
tabUrl.searchParams.set('select', this.part.pk);
$('#images-tab-link').attr('href', tabUrl);
$('#nav-img-tab').attr('href', tabUrl);
}
},
selectedTranscription: function(n, o) {
......@@ -142,6 +142,8 @@ var partVM = new Vue({
(event.keyCode == (READ_DIRECTION == 'rtl'?37:39) && event.ctrlKey)) { // arrow right
this.getNext();
event.preventDefault();
} else if (event.keyCode == 8 && event.ctrlKey) { // backspace
this.zoom.reset();
}
}.bind(this));
......
......@@ -54,7 +54,6 @@ const partStore = {
// actions
fetchPart(pk, callback) {
this.reset();
this.pk = pk;
this.fetchDocument(function() {
let uri = this.getApiPart(pk);
......@@ -284,7 +283,7 @@ const partStore = {
let deletedLines = [];
for (let i=0; i<pks.length; i++) {
let index = this.lines.findIndex(l=>l.pk==pks[i]);
if(index) {
if(index != -1) {
deletedLines.push(pks[i]);
Vue.delete(this.lines, index);
}
......@@ -476,11 +475,13 @@ const partStore = {
getPrevious(cb) {
if (this.loaded && this.previous) {
this.reset();
this.fetchPart(this.previous, cb);
}
},
getNext(cb) {
if (this.loaded && this.next) {
this.reset();
this.fetchPart(this.next, cb);
}
}
......
......@@ -115,7 +115,7 @@ def make_segmentation_training_data(part):
'image': part.image.path,
'baselines': [{'script': 'default', 'baseline': bl}
for bl in part.lines.values_list('baseline', flat=True) if bl],
'regions': {'test': [r.box for r in part.blocks.all().only('box')]}
'regions': {'default': [r.box for r in part.blocks.all().only('box')]}
}
......@@ -146,8 +146,7 @@ def segtrain(task, model_pk, document_pk, part_pks, user_pk=None):
load = model.file.path
upload_to = model.file.field.upload_to(model, model.name + '.mlmodel')
except ValueError: # model is empty
# load = settings.KRAKEN_DEFAULT_SEGMENTATION_MODEL
load = None
load = settings.KRAKEN_DEFAULT_SEGMENTATION_MODEL
upload_to = model.file.field.upload_to(model, model.name + '.mlmodel')
model.file = upload_to
......@@ -185,6 +184,7 @@ def segtrain(task, model_pk, document_pk, part_pks, user_pk=None):
evaluation_data=evaluation_data,
threads=LOAD_THREADS,
augment=True,
resize='both',
load_hyper_parameters=True)
if not os.path.exists(os.path.split(modelpath)[0]):
......
......@@ -32,7 +32,7 @@ class InvitationAcceptForm(BootstrapFormMixin, UserCreationForm):
"""
This is a registration form since a user is created.
"""
class Meta(UserCreationForm.Meta):
model = User
fields = ('email',
......@@ -41,8 +41,14 @@ class InvitationAcceptForm(BootstrapFormMixin, UserCreationForm):
'last_name',
'password1',
'password2')
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fields['username'].help_text = None
self.fields['password1'].help_text = None
class ProfileForm(BootstrapFormMixin, forms.ModelForm):
class Meta:
model = User
fields = ('email', 'first_name', 'last_name')
from django.urls import path, include
from users.views import SendInvitation, AcceptInvitation
from users.views import SendInvitation, AcceptInvitation, Profile
from django.contrib.auth.decorators import permission_required
urlpatterns = [
path('', include('django.contrib.auth.urls')),
path('profile/', Profile.as_view(), name='user_profile'),
path('invite/',
permission_required('users.can_invite', raise_exception=True)(SendInvitation.as_view()),
name='send-invitation'),
......
from os import listdir, stat
from os.path import isfile, join, relpath
from django.conf import settings
from django.contrib.auth.mixins import LoginRequiredMixin
from django.contrib.messages.views import SuccessMessageMixin
from django.core.paginator import Paginator
from django.http import Http404
from django.utils.functional import cached_property
from django.utils.translation import gettext as _
from django.views.generic.edit import CreateView
from django.views.generic.edit import CreateView, UpdateView
from rest_framework.authtoken.models import Token
from users.models import User, Invitation
from users.forms import InvitationForm, InvitationAcceptForm
from users.forms import InvitationForm, InvitationAcceptForm, ProfileForm
class SendInvitation(LoginRequiredMixin, SuccessMessageMixin, CreateView):
......@@ -68,3 +74,32 @@ class AcceptInvitation(CreateView):
return response
# request invitation
class Profile(SuccessMessageMixin, UpdateView):
model = User
form_class = ProfileForm
success_url = '/profile/'
success_message = _('Profile successfully saved.')
template_name = 'users/profile.html'
def get_object(self):
return self.request.user
def get_context_data(self):
context = super().get_context_data()
context['api_auth_token'], created = Token.objects.get_or_create(user=self.object)
# files directory
upath = self.object.get_document_store_path() + '/'
files = listdir(upath)
files.sort(key=lambda x: stat(join(upath, x)).st_mtime)
files.reverse()
files = [(relpath(join(upath, f), settings.MEDIA_ROOT), f)
for f in files
if isfile(join(upath, f))]
paginator = Paginator(files, 25) # Show 25 files per page.
page_number = self.request.GET.get('page')
context['is_paginated'] = paginator.count != 0
context['page_obj'] = paginator.get_page(page_number)
return context
......@@ -34,6 +34,9 @@ sys.path.append(APPS_DIR)
SITE_ID = 1
SECRET_KEY = os.getenv('SECRET_KEY', 'a-beautiful-snowflake')
# SECURE_SSL_REDIRECT = os.getenv('SECURE_SSL_REDIRECT', False) == 'True' # should be done by nginx
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = os.getenv('DEBUG', False)
......@@ -57,6 +60,7 @@ INSTALLED_APPS = [
'easy_thumbnails.optimize',
'channels',
'rest_framework',
'rest_framework.authtoken',
'compressor',
'bootstrap',
......@@ -149,8 +153,8 @@ LANGUAGES = [
('de', _('French')),
]
EMAIL_HOST = 'mail'
EMAIL_PORT = 25
EMAIL_HOST = os.getenv('EMAIL_HOST', 'mail')
EMAIL_PORT = os.getenv('EMAIL_PORT', 25)
DEFAULT_FROM_EMAIL = os.getenv('DJANGO_FROM_EMAIL', 'noreply@escriptorium.fr')
REDIS_HOST = os.getenv('REDIS_HOST', 'localhost')
......@@ -314,6 +318,11 @@ REST_FRAMEWORK = {
# 'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
'rest_framework.permissions.IsAuthenticated'
],
'DEFAULT_AUTHENTICATION_CLASSES': [
# 'rest_framework.authentication.BasicAuthentication', # Only for testing
'rest_framework.authentication.TokenAuthentication',
'rest_framework.authentication.SessionAuthentication',
],
'DEFAULT_PAGINATION_CLASS': 'core.pagination.CustomPagination',
'PAGE_SIZE': 10,
}
......
......@@ -608,7 +608,15 @@ i.panel-icon {
overflow-wrap: break-word;
flex-basis: 85%;
}
.selected {
.line-selected {
background-color:#33A2FF;
border: solid #33A2FF 1px;
}
.cmp-add {
background-color: lightgreen;
}
.cmp-del {
color: red;
}
......@@ -8,8 +8,7 @@
segmenter.load([{baseline: [[0,0],[10,10]], mask: null}]);
Options:
lengthTreshold=15
lengthTreshold=15,
lengthThreshold=15
delayInit=false,
deletePointBtn=null,
deleteSelectionBtn=null,
......@@ -73,6 +72,7 @@ class SegmenterRegion {
this.polygonPath.selected = false;
this.segmenter.removeFromSelection(this);
this.selected = false;
}
toggleSelect() {
......@@ -93,9 +93,15 @@ class SegmenterRegion {
update(polygon) {
if (polygon && polygon.length) {
this.polygon = polygon;
this.polygonPath.removeSegments();
this.polygonPath.addSegments(polygon);
this.segmenter.bindRegionEvents(this);
if (polygon.length == this.polygonPath.segments.length) {
// if number of points didn't change make sure to keep selection
for (let i in this.polygon) {
this.polygonPath.segments[i].point = this.polygon[i];
}
} else {
this.polygonPath.removeSegments();
this.polygonPath.addSegments(this.polygon);
}
}
}
......@@ -107,7 +113,6 @@ class SegmenterRegion {
}
delete() {
this.unselect();
this.remove();
}
......@@ -252,19 +257,33 @@ class SegmenterLine {
update(baseline, mask, region, order) {
if (baseline && baseline.length) {
this.baseline = baseline;
this.baselinePath.removeSegments();
this.baselinePath.addSegments(baseline);
this.baselinePath.strokeWidth = 5/this.segmenter.getRatio();
this.segmenter.bindLineEvents(this);
if (baseline.length == this.baselinePath.segments.length) {
// make sure to keep selection
for (let i in this.baseline) {
this.baselinePath.segments[i].point = this.baseline[i];
}
} else {
this.baselinePath.removeSegments();
this.baselinePath.addSegments(baseline);
}
/* this.baselinePath.strokeWidth = 5/this.segmenter.getRatio();
* this.segmenter.bindLineEvents(this); */
}
if (mask && mask.length) {
if (! this.maskPath) {
this.makeMaskPath();
}
this.mask = mask;
this.maskPath.removeSegments();
this.maskPath.addSegments(mask);
this.segmenter.bindMaskEvents(this);
if (mask.length == this.maskPath.segments.length) {
// make sure to keep selection
for (let i in this.mask) {
this.maskPath.segments[i].point = this.mask[i];
}
} else {
this.maskPath.removeSegments();
this.maskPath.addSegments(mask);
}
// this.segmenter.bindMaskEvents(this);
}
if (region !== undefined) {
this.region = region;
......@@ -297,6 +316,8 @@ class SegmenterLine {
}
extend(point) {
// make sure the point is inside img boundaries
this.segmenter.movePointInView(point, {x:0, y:0});
return this.baselinePath.add(point);
}
......@@ -310,7 +331,6 @@ class SegmenterLine {
}
delete() {
this.unselect();
this.remove();
}
......@@ -436,7 +456,8 @@ class SegmenterLine {
}
class Segmenter {
constructor(image, {lengthTreshold=10,
constructor(image, {lengthThreshold=10,
regionAreaThreshold=20,
// scale = real coordinates to image coordinates
// for example if drawing on a 1000px wide thumbnail for a 'real' 3000px wide image,