Commit 25a15f5e authored by EL HASSANE GARGEM's avatar EL HASSANE GARGEM
Browse files

merge onboarding

parents 7cf307b6 7b901a6a
import bleach
import logging
import html
from django.conf import settings
from django.db.utils import IntegrityError
......@@ -154,7 +155,9 @@ class LineTranscriptionSerializer(serializers.ModelSerializer):
'versions', 'version_author', 'version_source', 'version_updated_at')
def cleanup(self, data):
return bleach.clean(data, tags=['em', 'strong', 's', 'u'], strip=True)
nd = bleach.clean(data, tags=['em', 'strong', 's', 'u'], strip=True)
nd = html.unescape(nd)
return nd
def validate_content(self, content):
return self.cleanup(content)
......@@ -192,25 +195,39 @@ class LineSerializer(serializers.ModelSerializer):
list_serializer_class = LineListSerializer
class LineMoveSerializer(serializers.ModelSerializer):
index = serializers.IntegerField()
class LineOrderListSerializer(serializers.ListSerializer):
def update(self, qs, validated_data):
# Maps for id->instance and id->data item.
line_mapping = {line.pk: line for line in qs}
data_mapping = {item['pk']: item for item in validated_data}
class Meta:
model = Line
fields = ('index',)
# we can only go down or up (not both)
first_ = qs[0]
down = first_.order < data_mapping[first_.pk]['order']
lines = list(data_mapping.items())
lines.sort(key=lambda l: l[1]['order'])
if down:
# reverse to avoid pushing up already moved lines
lines.reverse()
def __init__(self, *args, line=None, **kwargs):
self.line = line
super().__init__(*args, **kwargs)
for i, (line_id, data) in enumerate(lines):
line = line_mapping.get(line_id, None)
line.to(data['order'])
def move(self):
self.line.to(self.validated_data['index'])
line.document_part.enforce_line_order()
# returns all new ordering for the whole page
data = self.child.__class__(line.document_part.lines.all(), many=True).data
return data
class LineOrderSerializer(serializers.ModelSerializer):
pk = serializers.IntegerField()
order = serializers.IntegerField()
class Meta:
model = Line
fields = ('pk', 'order')
list_serializer_class = LineOrderListSerializer
class DetailedLineSerializer(LineSerializer):
......
......@@ -164,7 +164,7 @@ class BlockViewSetTestCase(CoreFactoryTestCase):
kwargs={'document_pk': self.part.document.pk,
'part_pk': self.part.pk,
'pk': self.block.pk})
with self.assertNumQueries(3):
with self.assertNumQueries(6):
resp = self.client.get(uri)
self.assertEqual(resp.status_code, 200)
......@@ -173,7 +173,7 @@ class BlockViewSetTestCase(CoreFactoryTestCase):
uri = reverse('api:block-list',
kwargs={'document_pk': self.part.document.pk,
'part_pk': self.part.pk})
with self.assertNumQueries(4):
with self.assertNumQueries(7):
resp = self.client.get(uri)
self.assertEqual(resp.status_code, 200)
......@@ -199,7 +199,7 @@ class BlockViewSetTestCase(CoreFactoryTestCase):
kwargs={'document_pk': self.part.document.pk,
'part_pk': self.part.pk,
'pk': self.block.pk})
with self.assertNumQueries(4):
with self.assertNumQueries(7):
resp = self.client.patch(uri, {
'box': '[[100,100], [150,150]]'
}, content_type='application/json')
......@@ -250,7 +250,7 @@ class LineViewSetTestCase(CoreFactoryTestCase):
kwargs={'document_pk': self.part.document.pk,
'part_pk': self.part.pk,
'pk': self.line.pk})
with self.assertNumQueries(5):
with self.assertNumQueries(7):
resp = self.client.patch(uri, {
'baseline': '[[100,100], [150,150]]'
}, content_type='application/json')
......@@ -272,7 +272,7 @@ class LineViewSetTestCase(CoreFactoryTestCase):
self.client.force_login(self.user)
uri = reverse('api:line-bulk-update',
kwargs={'document_pk': self.part.document.pk, 'part_pk': self.part.pk})
with self.assertNumQueries(7):
with self.assertNumQueries(9):
resp = self.client.put(uri, {'lines': [
{'pk': self.line.pk,
'mask': '[[60, 40], [60, 50], [90, 50], [90, 40]]',
......@@ -320,7 +320,7 @@ class LineTranscriptionViewSetTestCase(CoreFactoryTestCase):
kwargs={'document_pk': self.part.document.pk,
'part_pk': self.part.pk,
'pk': self.lt.pk})
with self.assertNumQueries(5):
with self.assertNumQueries(8):
resp = self.client.patch(uri, {
'content': 'update'
}, content_type='application/json')
......@@ -347,7 +347,7 @@ class LineTranscriptionViewSetTestCase(CoreFactoryTestCase):
'part_pk': self.part.pk,
'pk': self.lt.pk})
with self.assertNumQueries(7):
with self.assertNumQueries(10):
resp = self.client.put(uri, {'content': 'test',
'transcription': self.lt.transcription.pk,
'line': self.lt.line.pk},
......
......@@ -23,7 +23,6 @@ from api.serializers import (UserOnboardingSerializer,
BlockTypeSerializer,
LineTypeSerializer,
DetailedLineSerializer,
LineMoveSerializer,
LineOrderSerializer,
TranscriptionSerializer,
LineTranscriptionSerializer)
......@@ -205,20 +204,24 @@ class LineTypeViewSet(ModelViewSet):
serializer_class = LineTypeSerializer
class BlockViewSet(ModelViewSet):
class BlockViewSet(DocumentPermissionMixin, ModelViewSet):
queryset = Block.objects.select_related('typology')
serializer_class = BlockSerializer
def get_queryset(self):
return Block.objects.filter(document_part=self.kwargs['part_pk'])
return (super().get_queryset()
.filter(document_part=self.kwargs['part_pk'])
.filter(document_part__document=self.kwargs['document_pk']))
class LineViewSet(ModelViewSet):
class LineViewSet(DocumentPermissionMixin, ModelViewSet):
queryset = (Line.objects.select_related('block')
.select_related('typology'))
def get_queryset(self):
return super().get_queryset().filter(document_part=self.kwargs['part_pk'])
return (super().get_queryset()
.filter(document_part=self.kwargs['part_pk'])
.filter(document_part__document=self.kwargs['document_pk']))
def get_serializer_class(self):
if self.action == 'retrieve':
......@@ -250,13 +253,14 @@ class LineViewSet(ModelViewSet):
qs.delete()
return Response(status=status.HTTP_204_NO_CONTENT)
@action(detail=True, methods=['post'])
@action(detail=False, methods=['post'])
def move(self, request, document_pk=None, part_pk=None, pk=None):
line = get_object_or_404(Line, pk=pk)
serializer = LineMoveSerializer(line=line, data=request.data)
data = request.data.get('lines')
qs = Line.objects.filter(pk__in=[l['pk'] for l in data])
serializer = LineOrderSerializer(qs, data=data, many=True)
if serializer.is_valid():
serializer.move()
return Response({'status': 'moved'})
resp = serializer.save()
return Response(resp, status=200)
else:
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
......@@ -265,14 +269,15 @@ class LargeResultsSetPagination(PageNumberPagination):
page_size = 100
class LineTranscriptionViewSet(ModelViewSet):
class LineTranscriptionViewSet(DocumentPermissionMixin, ModelViewSet):
queryset = LineTranscription.objects.all()
serializer_class = LineTranscriptionSerializer
pagination_class = LargeResultsSetPagination
def get_queryset(self):
qs = (self.queryset
qs = (super().get_queryset()
.filter(line__document_part=self.kwargs['part_pk'])
.filter(line__document_part__document=self.kwargs['document_pk'])
.select_related('line', 'transcription')
.order_by('line__order'))
transcription = self.request.GET.get('transcription')
......@@ -322,19 +327,35 @@ class LineTranscriptionViewSet(ModelViewSet):
@action(detail=False, methods=['PUT'])
def bulk_update(self, request, document_pk=None, part_pk=None, pk=None):
lines = request.data.get("lines")
lines = request.data.get('lines')
response = []
errors = []
for line in lines:
lt = get_object_or_404(LineTranscription, pk=line["pk"])
lt.new_version(author=request.user.username,
source=settings.VERSIONING_DEFAULT_SOURCE)
serializer = LineTranscriptionSerializer(lt, data=line, partial=True)
serializer.is_valid(raise_exception=True)
serializer.save()
return Response({'status': 'ok'}, status=200)
if serializer.is_valid():
try:
lt.new_version(author=request.user.username,
source=settings.VERSIONING_DEFAULT_SOURCE)
except NoChangeException:
pass
serializer.save()
response.append(serializer.data)
else:
errors.append(errors)
if errors:
return Response(errors,
status=status.HTTP_400_BAD_REQUEST)
return Response(status=200, data=response)
@action(detail=False, methods=['POST'])
def bulk_delete(self, request, document_pk=None, part_pk=None, pk=None):
lines = request.data.get("lines")
qs = LineTranscription.objects.filter(pk__in=lines)
qs.update(content='')
return Response(status=status.HTTP_204_NO_CONTENT)
return Response(status=status.HTTP_204_NO_CONTENT, )
......@@ -3,6 +3,7 @@ import logging
from PIL import Image
from django import forms
from django.conf import settings
from django.core.validators import FileExtensionValidator, MinValueValidator, MaxValueValidator
from django.db.models import Q
from django.forms.models import inlineformset_factory
......@@ -146,9 +147,10 @@ class DocumentProcessForm(BootstrapFormMixin, forms.Form):
)
segmentation_steps = forms.ChoiceField(choices=SEGMENTATION_STEPS_CHOICES,
initial='both', required=False)
seg_model = forms.ModelChoiceField(queryset=OcrModel.objects
.filter(job=OcrModel.MODEL_JOB_SEGMENT),
label=_("Model"), required=False)
seg_model = forms.ModelChoiceField(queryset=OcrModel.objects.filter(job=OcrModel.MODEL_JOB_SEGMENT),
label=_("Model"), empty_label="default ({name})".format(
name=settings.KRAKEN_DEFAULT_SEGMENTATION_MODEL.rsplit('/')[-1]),
required=False)
override = forms.BooleanField(required=False, initial=True,
help_text=_("If checked, deletes existing segmentation <b>and bound transcriptions</b> first!"))
TEXT_DIRECTION_CHOICES = (('horizontal-lr', _("Horizontal l2r")),
......
......@@ -763,6 +763,15 @@ class DocumentPart(OrderedModel):
return to_calc
def enforce_line_order(self):
# django-ordered-model doesn't care about unicity and linearity...
lines = self.lines.order_by('order', 'pk')
for i, line in enumerate(lines):
if line.order != i:
logger.debug('Enforcing line order %d : %d' % (line.pk, i))
line.order = i
line.save()
def validate_polygon(value):
if value is None:
......@@ -829,7 +838,7 @@ class Line(OrderedModel): # Versioned,
script = models.CharField(max_length=8, null=True, blank=True) # choices ??
# text direction
order_with_respect_to = 'document_part'
version_ignore_fields = ('document_part', 'order')
# version_ignore_fields = ('document_part', 'order')
typology = models.ForeignKey(LineType, null=True, blank=True,
on_delete=models.SET_NULL)
......
var diploLine = LineBase.extend({
props: ['line', 'ratio'],
computed: {
showregion() {
let idx = this.$parent.part.lines.indexOf(this.line);
if (idx) {
let pr = this.$parent.part.lines[idx - 1].region;
if (this.line.region == pr)
return "";
else
return this.getRegion() + 1 ;
} else {
return this.getRegion() + 1 ;
}
}
},
mounted() {
this.$content = this.$refs.content[0];
Vue.nextTick(function() {
this.$parent.appendLine();
}.bind(this));
},
beforeDestroy() {
let el = this.getEl();
if (el != null) {
el.remove();
}
},
watch: {
'line.order': function(o, n) {
'line.order': function(n, o) {
// make sure it's at the right place,
// in case it was just created or the ordering got recalculated
let index = Array.from(this.$el.parentNode.children).indexOf(this.$el);
......@@ -12,76 +34,25 @@ var diploLine = LineBase.extend({
this.$el.parentNode.insertBefore(
this.$el,
this.$el.parentNode.children[this.line.order]);
this.setElContent(this.line.currentTrans.content);
}
},
'line.currentTrans': function(n, o) {
if (n!=undefined) {
this.setElContent(n.content);
}
}
},
methods: {
startEdit(ev) {
// if we are selecting text we don't want to start editing
// to be able to do multiline selection
if (document.getSelection().toString()) {
return true;
}
this.$content.setAttribute('contenteditable', true);
this.$content.focus(); // needed in case we edit from the panel
this.$parent.setEditLine(this.line);
this.$content.style.backgroundColor = '#F8F8F8';
this.$parent.$parent.blockShortcuts = true;
},
stopEdit(ev) {
this.$content.setAttribute('contenteditable', false);
this.$content.style.backgroundColor = 'white';
this.$parent.$parent.blockShortcuts = false;
this.pushUpdate();
},
pushUpdate(){
// set content of input to line content
if (this.line.currentTrans.content != this.$content.textContent) {
this.line.currentTrans.content = this.$content.textContent;
this.addToList();
// call save of parent method
this.$parent.toggleSave();
}
getEl() {
return this.$parent.editor.querySelector('div:nth-child('+parseInt(this.line.order+1)+')');
},
setContent(content){
let id = this.line.pk;
$("#" + id).text(content);
this.line.currentTrans.content = content;
},
onPaste(e) {
let pastedData = e.clipboardData.getData('text/plain');
let pasted_data_split = pastedData.split('\n');
if (pasted_data_split.length < 2) {
return
} else {
e.preventDefault();
if (pasted_data_split[pasted_data_split.length - 1] == "")
pasted_data_split.pop();
let index = this.$parent.$children.indexOf(this);
for (let i = 1; i < pasted_data_split.length; i++) {
if (this.$parent.$children[index + i]) {
let content = pasted_data_split[i];
let child = this.$parent.$children[index + i];
child.setContent(content);
child.addToList();
} else {
let content = pasted_data_split.slice(i - 1).join('\n');
let child = this.$parent.$children[index + 1];
child.setContent(content);
child.addToList();
}
}
}
setElContent(content) {
let line = this.getEl();
if (line) line.textContent = content;
},
addToList(){
if(this.line.currentTrans.pk)
this.$parent.$emit('update:transcription:content', this.line.currentTrans);
else
this.$parent.$emit('create:transcription', this.line.currentTrans);
getRegion() {
return this.$parent.part.regions.findIndex(r => r.pk == this.line.region);
}
}
});
var DiploPanel = BasePanel.extend({
data() { return {
editLine: null,
save: false, //show save button
updatedLines : [],
createdLines : [],
dragging: -1
movedLines:[],
};},
components: {
'diploline': diploLine,
},
watch: {
'part.loaded': function(isLoaded, wasLoaded) {
if (!isLoaded) {
// changed page probably
this.empty();
}
}
},
created() {
this.$on('update:transcription:content', function(linetranscription) {
this.addToUpdatedLines(linetranscription);
});
this.$on('create:transcription', function(linetranscription) {
this.createdLines.push(linetranscription);
});
// vue.js quirck, have to dinamically create the event handler
// call save every 10 seconds after last change
this.debouncedSave = _.debounce(function() {
this.save();
}.bind(this), 10000);
},
mounted() {
Vue.nextTick(function() {
var vm = this ;
vm.sortable = Sortable.create(this.editor, {
disabled: true,
multiDrag: true,
multiDragKey : 'CTRL',
selectedClass: "selected",
ghostClass: "ghost",
dragClass: "info",
animation: 150,
onEnd: function(evt) {
vm.onDraggingEnd(evt);
}
});
}.bind(this));
this.editor = this.$el.querySelector('#diplomatic-lines');
this.sortModeBtn = this.$el.querySelector('#sortMode');
this.saveNotif = this.$el.querySelector('.tools #save-notif');
},
methods:{
toggleSave(){
methods: {
empty() {
while (this.editor.hasChildNodes()) {
this.editor.removeChild(this.editor.lastChild);
}
},
toggleSort() {
if (this.editor.contentEditable === 'true') {
this.editor.contentEditable = 'false';
this.sortable.option('disabled', false);
this.sortModeBtn.classList.remove('btn-info');
this.sortModeBtn.classList.add('btn-success');
} else {
this.editor.contentEditable = 'true';
this.sortable.option('disabled', true);
this.sortModeBtn.classList.remove('btn-success');
this.sortModeBtn.classList.add('btn-info');
}
},
changed() {
this.saveNotif.classList.remove('hide');
this.debouncedSave();
},
appendLine(pos) {
let div = document.createElement('div');
div.appendChild(document.createElement('br'));
if (pos === undefined) {
this.editor.appendChild(div);
} else {
this.editor.insertBefore(div, pos);
}
return div;
},
constrainLineNumber() {
// add lines untill we have enough of them
while (this.editor.childElementCount < this.part.lines.length) {
this.appendLine();
}
// need to add/remove danger indicators
for (let i=0; i<this.editor.childElementCount; i++) {
let line = this.editor.querySelector('div:nth-child('+parseInt(i+1)+')');
if (line === null) {
this.editor.children[i].remove();
continue;
}
if (i<this.part.lines.length) {
line.classList.remove('alert-danger');
line.setAttribute('title', '');
} else if (i>=this.part.lines.length) {
if (line.textContent == '') { // just remove empty lines
line.remove();
} else {
line.classList.add('alert-danger');
line.setAttribute('title', 'More lines than there is in the segmentation!');
}
}
}
},
startEdit(ev) {
this.$parent.blockShortcuts = true;
},
stopEdit(ev) {
this.$parent.blockShortcuts = false;
this.constrainLineNumber();
this.save();
},
onDraggingEnd(ev) {
/*
Finish dragging lines, save new positions
*/
if(ev.newIndicies.length == 0 && ev.newIndex != ev.oldIndex) {
let diploLine = this.$children.find(dl=>dl.line.order==ev.oldIndex);
this.movedLines.push({
"pk": diploLine.line.pk,
"order": ev.newIndex
});
} else {
for(let i=0; i< ev.newIndicies.length; i++) {
let diploLine = this.$children.find(dl=>dl.line.order==ev.oldIndicies[i].index);
this.movedLines.push({
"pk": diploLine.line.pk,
"order": ev.newIndicies[i].index
});
}
}
this.moveLines();
},
moveLines() {
if(this.movedLines.length != 0){
this.$parent.$emit('move:line', this.movedLines, function () {
this.movedLines = [];
}.bind(this));
}
},
save() {
/*
if some lines are modified add them to updatedlines,
new lines add them to createdLines then save
*/
this.saveNotif.classList.add('hide');
this.addToList();
this.bulkUpdate();
this.bulkCreate();
},
setHeight() {
this.$el.querySelector('.content-container').style.maxHeight = Math.round(this.part.image.size[1] * this.ratio) + 'px';