Skip to content

GitLab

Commit 25a15f5e authored by EL HASSANE GARGEM's avatar EL HASSANE GARGEM
Browse files

merge onboarding

parents 7cf307b6 7b901a6a
Expand all Hide whitespace changes
Inline Side-by-side
app/apps/api/serializers.py
View file @ 25a15f5e
import bleach
import logging
import html
from django.conf import settings
from django.db.utils import IntegrityError
......@@ -154,7 +155,9 @@ class LineTranscriptionSerializer(serializers.ModelSerializer):
'versions', 'version_author', 'version_source', 'version_updated_at')
def cleanup(self, data):
return bleach.clean(data, tags=['em', 'strong', 's', 'u'], strip=True)
nd = bleach.clean(data, tags=['em', 'strong', 's', 'u'], strip=True)
nd = html.unescape(nd)
return nd
def validate_content(self, content):
return self.cleanup(content)
......@@ -192,25 +195,39 @@ class LineSerializer(serializers.ModelSerializer):
list_serializer_class = LineListSerializer
class LineMoveSerializer(serializers.ModelSerializer):
index = serializers.IntegerField()
class LineOrderListSerializer(serializers.ListSerializer):
def update(self, qs, validated_data):
# Maps for id->instance and id->data item.
line_mapping = {line.pk: line for line in qs}
data_mapping = {item['pk']: item for item in validated_data}
class Meta:
model = Line
fields = ('index',)
# we can only go down or up (not both)
first_ = qs[0]
down = first_.order < data_mapping[first_.pk]['order']
lines = list(data_mapping.items())
lines.sort(key=lambda l: l[1]['order'])
if down:
# reverse to avoid pushing up already moved lines
lines.reverse()
def __init__(self, *args, line=None, **kwargs):
self.line = line
super().__init__(*args, **kwargs)
for i, (line_id, data) in enumerate(lines):
line = line_mapping.get(line_id, None)
line.to(data['order'])
def move(self):
self.line.to(self.validated_data['index'])
line.document_part.enforce_line_order()
# returns all new ordering for the whole page
data = self.child.__class__(line.document_part.lines.all(), many=True).data
return data
class LineOrderSerializer(serializers.ModelSerializer):
pk = serializers.IntegerField()
order = serializers.IntegerField()
class Meta:
model = Line
fields = ('pk', 'order')
list_serializer_class = LineOrderListSerializer
class DetailedLineSerializer(LineSerializer):
......
app/apps/api/tests.py
View file @ 25a15f5e
......@@ -164,7 +164,7 @@ class BlockViewSetTestCase(CoreFactoryTestCase):
kwargs={'document_pk': self.part.document.pk,
'part_pk': self.part.pk,
'pk': self.block.pk})
with self.assertNumQueries(3):
with self.assertNumQueries(6):
resp = self.client.get(uri)
self.assertEqual(resp.status_code, 200)
......@@ -173,7 +173,7 @@ class BlockViewSetTestCase(CoreFactoryTestCase):
uri = reverse('api:block-list',
kwargs={'document_pk': self.part.document.pk,
'part_pk': self.part.pk})
with self.assertNumQueries(4):
with self.assertNumQueries(7):
resp = self.client.get(uri)
self.assertEqual(resp.status_code, 200)
......@@ -199,7 +199,7 @@ class BlockViewSetTestCase(CoreFactoryTestCase):
kwargs={'document_pk': self.part.document.pk,
'part_pk': self.part.pk,
'pk': self.block.pk})
with self.assertNumQueries(4):
with self.assertNumQueries(7):
resp = self.client.patch(uri, {
'box': '[[100,100], [150,150]]'
}, content_type='application/json')
......@@ -250,7 +250,7 @@ class LineViewSetTestCase(CoreFactoryTestCase):
kwargs={'document_pk': self.part.document.pk,
'part_pk': self.part.pk,
'pk': self.line.pk})
with self.assertNumQueries(5):
with self.assertNumQueries(7):
resp = self.client.patch(uri, {
'baseline': '[[100,100], [150,150]]'
}, content_type='application/json')
......@@ -272,7 +272,7 @@ class LineViewSetTestCase(CoreFactoryTestCase):
self.client.force_login(self.user)
uri = reverse('api:line-bulk-update',
kwargs={'document_pk': self.part.document.pk, 'part_pk': self.part.pk})
with self.assertNumQueries(7):
with self.assertNumQueries(9):
resp = self.client.put(uri, {'lines': [
{'pk': self.line.pk,
'mask': '[[60, 40], [60, 50], [90, 50], [90, 40]]',
......@@ -320,7 +320,7 @@ class LineTranscriptionViewSetTestCase(CoreFactoryTestCase):
kwargs={'document_pk': self.part.document.pk,
'part_pk': self.part.pk,
'pk': self.lt.pk})
with self.assertNumQueries(5):
with self.assertNumQueries(8):
resp = self.client.patch(uri, {
'content': 'update'
}, content_type='application/json')
......@@ -347,7 +347,7 @@ class LineTranscriptionViewSetTestCase(CoreFactoryTestCase):
'part_pk': self.part.pk,
'pk': self.lt.pk})
with self.assertNumQueries(7):
with self.assertNumQueries(10):
resp = self.client.put(uri, {'content': 'test',
'transcription': self.lt.transcription.pk,
'line': self.lt.line.pk},
......
app/apps/api/views.py
View file @ 25a15f5e
......@@ -23,7 +23,6 @@ from api.serializers import (UserOnboardingSerializer,
BlockTypeSerializer,
LineTypeSerializer,
DetailedLineSerializer,
LineMoveSerializer,
LineOrderSerializer,
TranscriptionSerializer,
LineTranscriptionSerializer)
......@@ -205,20 +204,24 @@ class LineTypeViewSet(ModelViewSet):
serializer_class = LineTypeSerializer
class BlockViewSet(ModelViewSet):
class BlockViewSet(DocumentPermissionMixin, ModelViewSet):
queryset = Block.objects.select_related('typology')
serializer_class = BlockSerializer
def get_queryset(self):
return Block.objects.filter(document_part=self.kwargs['part_pk'])
return (super().get_queryset()
.filter(document_part=self.kwargs['part_pk'])
.filter(document_part__document=self.kwargs['document_pk']))
class LineViewSet(ModelViewSet):
class LineViewSet(DocumentPermissionMixin, ModelViewSet):
queryset = (Line.objects.select_related('block')
.select_related('typology'))
def get_queryset(self):
return super().get_queryset().filter(document_part=self.kwargs['part_pk'])
return (super().get_queryset()
.filter(document_part=self.kwargs['part_pk'])
.filter(document_part__document=self.kwargs['document_pk']))
def get_serializer_class(self):
if self.action == 'retrieve':
......@@ -250,13 +253,14 @@ class LineViewSet(ModelViewSet):
qs.delete()
return Response(status=status.HTTP_204_NO_CONTENT)
@action(detail=True, methods=['post'])
@action(detail=False, methods=['post'])
def move(self, request, document_pk=None, part_pk=None, pk=None):
line = get_object_or_404(Line, pk=pk)
serializer = LineMoveSerializer(line=line, data=request.data)
data = request.data.get('lines')
qs = Line.objects.filter(pk__in=[l['pk'] for l in data])
serializer = LineOrderSerializer(qs, data=data, many=True)
if serializer.is_valid():
serializer.move()
return Response({'status': 'moved'})
resp = serializer.save()
return Response(resp, status=200)
else:
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
......@@ -265,14 +269,15 @@ class LargeResultsSetPagination(PageNumberPagination):
page_size = 100
class LineTranscriptionViewSet(ModelViewSet):
class LineTranscriptionViewSet(DocumentPermissionMixin, ModelViewSet):
queryset = LineTranscription.objects.all()
serializer_class = LineTranscriptionSerializer
pagination_class = LargeResultsSetPagination
def get_queryset(self):
qs = (self.queryset
qs = (super().get_queryset()
.filter(line__document_part=self.kwargs['part_pk'])
.filter(line__document_part__document=self.kwargs['document_pk'])
.select_related('line', 'transcription')
.order_by('line__order'))
transcription = self.request.GET.get('transcription')
......@@ -322,19 +327,35 @@ class LineTranscriptionViewSet(ModelViewSet):
@action(detail=False, methods=['PUT'])
def bulk_update(self, request, document_pk=None, part_pk=None, pk=None):
lines = request.data.get("lines")
lines = request.data.get('lines')
response = []
errors = []
for line in lines:
lt = get_object_or_404(LineTranscription, pk=line["pk"])
lt.new_version(author=request.user.username,
source=settings.VERSIONING_DEFAULT_SOURCE)
serializer = LineTranscriptionSerializer(lt, data=line, partial=True)
serializer.is_valid(raise_exception=True)
serializer.save()
return Response({'status': 'ok'}, status=200)
if serializer.is_valid():
try:
lt.new_version(author=request.user.username,
source=settings.VERSIONING_DEFAULT_SOURCE)
except NoChangeException:
pass
serializer.save()
response.append(serializer.data)
else:
errors.append(errors)
if errors:
return Response(errors,
status=status.HTTP_400_BAD_REQUEST)
return Response(status=200, data=response)
@action(detail=False, methods=['POST'])
def bulk_delete(self, request, document_pk=None, part_pk=None, pk=None):
lines = request.data.get("lines")
qs = LineTranscription.objects.filter(pk__in=lines)
qs.update(content='')
return Response(status=status.HTTP_204_NO_CONTENT)
return Response(status=status.HTTP_204_NO_CONTENT, )
app/apps/core/forms.py
View file @ 25a15f5e
......@@ -3,6 +3,7 @@ import logging
from PIL import Image
from django import forms
from django.conf import settings
from django.core.validators import FileExtensionValidator, MinValueValidator, MaxValueValidator
from django.db.models import Q
from django.forms.models import inlineformset_factory
......@@ -146,9 +147,10 @@ class DocumentProcessForm(BootstrapFormMixin, forms.Form):
)
segmentation_steps = forms.ChoiceField(choices=SEGMENTATION_STEPS_CHOICES,
initial='both', required=False)
seg_model = forms.ModelChoiceField(queryset=OcrModel.objects
.filter(job=OcrModel.MODEL_JOB_SEGMENT),
label=_("Model"), required=False)
seg_model = forms.ModelChoiceField(queryset=OcrModel.objects.filter(job=OcrModel.MODEL_JOB_SEGMENT),
label=_("Model"), empty_label="default ({name})".format(
name=settings.KRAKEN_DEFAULT_SEGMENTATION_MODEL.rsplit('/')[-1]),
required=False)
override = forms.BooleanField(required=False, initial=True,
help_text=_("If checked, deletes existing segmentation <b>and bound transcriptions</b> first!"))
TEXT_DIRECTION_CHOICES = (('horizontal-lr', _("Horizontal l2r")),
......
app/apps/core/models.py
View file @ 25a15f5e
......@@ -763,6 +763,15 @@ class DocumentPart(OrderedModel):
return to_calc
def enforce_line_order(self):
# django-ordered-model doesn't care about unicity and linearity...
lines = self.lines.order_by('order', 'pk')
for i, line in enumerate(lines):
if line.order != i:
logger.debug('Enforcing line order %d : %d' % (line.pk, i))
line.order = i
line.save()
def validate_polygon(value):
if value is None:
......@@ -829,7 +838,7 @@ class Line(OrderedModel): # Versioned,
script = models.CharField(max_length=8, null=True, blank=True) # choices ??
# text direction
order_with_respect_to = 'document_part'
version_ignore_fields = ('document_part', 'order')
# version_ignore_fields = ('document_part', 'order')
typology = models.ForeignKey(LineType, null=True, blank=True,
on_delete=models.SET_NULL)
......
app/apps/core/static/js/edit/components/diplo_line.js
View file @ 25a15f5e
var diploLine = LineBase.extend({
props: ['line', 'ratio'],
computed: {
showregion() {
let idx = this.$parent.part.lines.indexOf(this.line);
if (idx) {
let pr = this.$parent.part.lines[idx - 1].region;
if (this.line.region == pr)
return "";
else
return this.getRegion() + 1 ;
} else {
return this.getRegion() + 1 ;
}
}
},
mounted() {
this.$content = this.$refs.content[0];
Vue.nextTick(function() {
this.$parent.appendLine();
}.bind(this));
},
beforeDestroy() {
let el = this.getEl();
if (el != null) {
el.remove();
}
},
watch: {
'line.order': function(o, n) {
'line.order': function(n, o) {
// make sure it's at the right place,
// in case it was just created or the ordering got recalculated
let index = Array.from(this.$el.parentNode.children).indexOf(this.$el);
......@@ -12,76 +34,25 @@ var diploLine = LineBase.extend({
this.$el.parentNode.insertBefore(
this.$el,
this.$el.parentNode.children[this.line.order]);
this.setElContent(this.line.currentTrans.content);
}
},
'line.currentTrans': function(n, o) {
if (n!=undefined) {
this.setElContent(n.content);
}
}
},
methods: {
startEdit(ev) {
// if we are selecting text we don't want to start editing
// to be able to do multiline selection
if (document.getSelection().toString()) {
return true;
}
this.$content.setAttribute('contenteditable', true);
this.$content.focus(); // needed in case we edit from the panel
this.$parent.setEditLine(this.line);
this.$content.style.backgroundColor = '#F8F8F8';
this.$parent.$parent.blockShortcuts = true;
},
stopEdit(ev) {
this.$content.setAttribute('contenteditable', false);
this.$content.style.backgroundColor = 'white';
this.$parent.$parent.blockShortcuts = false;
this.pushUpdate();
},
pushUpdate(){
// set content of input to line content
if (this.line.currentTrans.content != this.$content.textContent) {
this.line.currentTrans.content = this.$content.textContent;
this.addToList();
// call save of parent method
this.$parent.toggleSave();
}
getEl() {
return this.$parent.editor.querySelector('div:nth-child('+parseInt(this.line.order+1)+')');
},
setContent(content){
let id = this.line.pk;
$("#" + id).text(content);
this.line.currentTrans.content = content;
},
onPaste(e) {
let pastedData = e.clipboardData.getData('text/plain');
let pasted_data_split = pastedData.split('\n');
if (pasted_data_split.length < 2) {
return
} else {
e.preventDefault();
if (pasted_data_split[pasted_data_split.length - 1] == "")
pasted_data_split.pop();
let index = this.$parent.$children.indexOf(this);
for (let i = 1; i < pasted_data_split.length; i++) {
if (this.$parent.$children[index + i]) {
let content = pasted_data_split[i];
let child = this.$parent.$children[index + i];
child.setContent(content);
child.addToList();
} else {
let content = pasted_data_split.slice(i - 1).join('\n');
let child = this.$parent.$children[index + 1];
child.setContent(content);
child.addToList();
}
}
}
setElContent(content) {
let line = this.getEl();
if (line) line.textContent = content;
},
addToList(){
if(this.line.currentTrans.pk)
this.$parent.$emit('update:transcription:content', this.line.currentTrans);
else
this.$parent.$emit('create:transcription', this.line.currentTrans);
getRegion() {
return this.$parent.part.regions.findIndex(r => r.pk == this.line.region);
}
}
});
app/apps/core/static/js/edit/components/diplo_panel.js
View file @ 25a15f5e
var DiploPanel = BasePanel.extend({
data() { return {
editLine: null,
save: false, //show save button
updatedLines : [],
createdLines : [],
dragging: -1
movedLines:[],
};},
components: {
'diploline': diploLine,
},
watch: {
'part.loaded': function(isLoaded, wasLoaded) {
if (!isLoaded) {
// changed page probably
this.empty();
}
}
},
created() {
this.$on('update:transcription:content', function(linetranscription) {
this.addToUpdatedLines(linetranscription);
});
this.$on('create:transcription', function(linetranscription) {
this.createdLines.push(linetranscription);
});
// vue.js quirck, have to dinamically create the event handler
// call save every 10 seconds after last change
this.debouncedSave = _.debounce(function() {
this.save();
}.bind(this), 10000);
},
mounted() {
Vue.nextTick(function() {
var vm = this ;
vm.sortable = Sortable.create(this.editor, {
disabled: true,
multiDrag: true,
multiDragKey : 'CTRL',
selectedClass: "selected",
ghostClass: "ghost",
dragClass: "info",
animation: 150,
onEnd: function(evt) {
vm.onDraggingEnd(evt);
}
});
}.bind(this));
this.editor = this.$el.querySelector('#diplomatic-lines');
this.sortModeBtn = this.$el.querySelector('#sortMode');
this.saveNotif = this.$el.querySelector('.tools #save-notif');
},
methods:{
toggleSave(){
methods: {
empty() {
while (this.editor.hasChildNodes()) {
this.editor.removeChild(this.editor.lastChild);
}
},
toggleSort() {
if (this.editor.contentEditable === 'true') {
this.editor.contentEditable = 'false';
this.sortable.option('disabled', false);
this.sortModeBtn.classList.remove('btn-info');
this.sortModeBtn.classList.add('btn-success');
} else {
this.editor.contentEditable = 'true';
this.sortable.option('disabled', true);
this.sortModeBtn.classList.remove('btn-success');
this.sortModeBtn.classList.add('btn-info');
}
},
changed() {
this.saveNotif.classList.remove('hide');
this.debouncedSave();
},
appendLine(pos) {
let div = document.createElement('div');
div.appendChild(document.createElement('br'));
if (pos === undefined) {
this.editor.appendChild(div);
} else {
this.editor.insertBefore(div, pos);
}
return div;
},
constrainLineNumber() {
// add lines untill we have enough of them
while (this.editor.childElementCount < this.part.lines.length) {
this.appendLine();
}
// need to add/remove danger indicators
for (let i=0; i<this.editor.childElementCount; i++) {
let line = this.editor.querySelector('div:nth-child('+parseInt(i+1)+')');
if (line === null) {
this.editor.children[i].remove();
continue;
}
if (i<this.part.lines.length) {
line.classList.remove('alert-danger');
line.setAttribute('title', '');
} else if (i>=this.part.lines.length) {
if (line.textContent == '') { // just remove empty lines
line.remove();
} else {
line.classList.add('alert-danger');
line.setAttribute('title', 'More lines than there is in the segmentation!');
}
}
}
},
startEdit(ev) {
this.$parent.blockShortcuts = true;
},
stopEdit(ev) {
this.$parent.blockShortcuts = false;
this.constrainLineNumber();
this.save();
},
onDraggingEnd(ev) {
/*
Finish dragging lines, save new positions
*/
if(ev.newIndicies.length == 0 && ev.newIndex != ev.oldIndex) {
let diploLine = this.$children.find(dl=>dl.line.order==ev.oldIndex);
this.movedLines.push({
"pk": diploLine.line.pk,
"order": ev.newIndex
});
} else {
for(let i=0; i< ev.newIndicies.length; i++) {
let diploLine = this.$children.find(dl=>dl.line.order==ev.oldIndicies[i].index);
this.movedLines.push({
"pk": diploLine.line.pk,
"order": ev.newIndicies[i].index
});
}
}
this.moveLines();
},
moveLines() {
if(this.movedLines.length != 0){
this.$parent.$emit('move:line', this.movedLines, function () {
this.movedLines = [];
}.bind(this));
}
},
save() {
/*
if some lines are modified add them to updatedlines,
new lines add them to createdLines then save
*/
this.saveNotif.classList.add('hide');
this.addToList();
this.bulkUpdate();
this.bulkCreate();
},
setHeight() {
this.$el.querySelector('.content-container').style.maxHeight = Math.round(this.part.image.size[1] * this.ratio) + 'px';