Commit b7aadb72 authored by Robin Tissot's avatar Robin Tissot
Browse files

Merge branch 'feature/import-export-improvements' into develop

parents bd893bcd ecd1915b
......@@ -82,7 +82,7 @@ class DocumentViewSet(ModelViewSet):
form.save() # create the import
try:
form.process()
except ParseError as e:
except ParseError:
return self.form_error("Incorrectly formated file, couldn't parse it.")
return Response({'status': 'ok'})
else:
......
# Generated by Django 2.1.4 on 2020-09-17 13:02
import uuid
from django.db import migrations
def forward(apps, se):
Block = apps.get_model('core', 'Block')
Line = apps.get_model('core', 'Line')
for block in Block.objects.filter(external_id=None):
block.external_id = 'eSc_textblock_%s' % str(uuid.uuid4())[:8]
block.save()
for line in Line.objects.filter(external_id=None):
line.external_id = 'eSc_line_%s' % str(uuid.uuid4())[:8]
line.save()
def backward(apps, se):
# no need to do anything
pass
class Migration(migrations.Migration):
dependencies = [
('core', '0040_link_default_typology'),
]
operations = [
migrations.RunPython(forward, backward),
]
# Generated by Django 2.1.4 on 2020-10-09 10:15
import core.models
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('core', '0041_create_external_ids'),
]
operations = [
migrations.AlterField(
model_name='block',
name='box',
field=django.contrib.postgres.fields.jsonb.JSONField(validators=[core.models.validate_polygon, core.models.validate_3_points]),
),
migrations.AlterField(
model_name='line',
name='baseline',
field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, validators=[core.models.validate_polygon, core.models.validate_2_points]),
),
migrations.AlterField(
model_name='line',
name='block',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='lines', to='core.Block'),
),
migrations.AlterField(
model_name='line',
name='mask',
field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, validators=[core.models.validate_polygon, core.models.validate_3_points]),
),
]
......@@ -5,11 +5,12 @@ import os
import json
import functools
import subprocess
import uuid
from PIL import Image
from datetime import datetime
from django.db import models, transaction
from django.db.models import Q
from django.db.models import Q, Prefetch
from django.db.models.signals import pre_delete
from django.conf import settings
from django.contrib.auth import get_user_model
......@@ -332,9 +333,6 @@ class DocumentPart(OrderedModel):
except IndexError:
return False
def make_external_id(self):
return 'eSc_page_%d' % self.pk
@property
def filename(self):
return self.original_filename or os.path.split(self.image.path)[1]
......@@ -787,13 +785,27 @@ def validate_polygon(value):
params={'value': value})
def validate_2_points(value):
if len(value) < 2:
raise ValidationError(
_('Polygon needs to have at least 2 points, it has %(length)d: %(value)s.'),
params={'length': len(value), 'value': value})
def validate_3_points(value):
if len(value) < 3:
raise ValidationError(
_('Polygon needs to have at least 3 points, it has %(length)d: %(value)s.'),
params={'length': len(value), 'value': value})
class Block(OrderedModel, models.Model):
"""
Represents a visualy close group of graphemes (characters) bound by the same semantic
example: a paragraph, a margin note or floating text
"""
# box = models.BoxField() # in case we use PostGIS
box = JSONField(validators=[validate_polygon])
box = JSONField(validators=[validate_polygon, validate_3_points])
typology = models.ForeignKey(BlockType, null=True, blank=True,
on_delete=models.SET_NULL)
document_part = models.ForeignKey(DocumentPart, on_delete=models.CASCADE,
......@@ -808,8 +820,8 @@ class Block(OrderedModel, models.Model):
@property
def coordinates_box(self):
"""
Cast the box field to the format [xmin,ymin,xmax,ymax]
to make it usable to calculate VPOS,HPOS,WIDTH, HEIGHT for Alto
Cast the box field to the format [xmin, ymin, xmax, ymax]
to make it usable to calculate VPOS, HPOS, WIDTH, HEIGHT for Alto
"""
return [*map(min, *self.box), *map(max, *self.box)]
......@@ -822,7 +834,22 @@ class Block(OrderedModel, models.Model):
return self.coordinates_box[3] - self.coordinates_box[1]
def make_external_id(self):
return self.external_id or 'eSc_textblock_%d' % self.pk
self.external_id = 'eSc_textblock_%s' % str(uuid.uuid4())[:8]
def save(self, *args, **kwargs):
if self.external_id is None:
self.make_external_id()
return super().save(*args, **kwargs)
class LineManager(models.Manager):
def prefetch_transcription(self, transcription):
return (self.get_queryset().order_by('order')
.prefetch_related(
Prefetch('transcriptions',
to_attr='transcription',
queryset=LineTranscription.objects.filter(
transcription=transcription))))
class Line(OrderedModel): # Versioned,
......@@ -831,13 +858,13 @@ class Line(OrderedModel): # Versioned,
"""
# box = gis_models.PolygonField() # in case we use PostGIS
# Closed Polygon: [[x1, y1], [x2, y2], ..]
mask = JSONField(null=True, blank=True, validators=[validate_polygon])
mask = JSONField(null=True, blank=True, validators=[validate_polygon, validate_3_points])
# Polygon: [[x1, y1], [x2, y2], ..]
baseline = JSONField(null=True, blank=True, validators=[validate_polygon])
baseline = JSONField(null=True, blank=True, validators=[validate_polygon, validate_2_points])
document_part = models.ForeignKey(DocumentPart,
on_delete=models.CASCADE,
related_name='lines')
block = models.ForeignKey(Block, null=True, blank=True, on_delete=models.SET_NULL)
block = models.ForeignKey(Block, null=True, blank=True, on_delete=models.SET_NULL, related_name='lines')
script = models.CharField(max_length=8, null=True, blank=True) # choices ??
# text direction
order_with_respect_to = 'document_part'
......@@ -848,6 +875,8 @@ class Line(OrderedModel): # Versioned,
external_id = models.CharField(max_length=128, blank=True, null=True)
objects = LineManager()
class Meta(OrderedModel.Meta):
pass
......@@ -874,10 +903,15 @@ class Line(OrderedModel): # Versioned,
(box[2], box[3]),
(box[2], box[1])]
box = property(get_box, set_box)
box = cached_property(get_box, set_box)
def make_external_id(self):
return self.external_id or 'eSc_line_%d' % self.pk
self.external_id = 'eSc_line_%s' % str(uuid.uuid4())[:8]
def save(self, *args, **kwargs):
if self.external_id is None:
self.make_external_id()
return super().save(*args, **kwargs)
class Transcription(models.Model):
......
......@@ -55,7 +55,9 @@ var partVM = new Vue({
}
},
selectedTranscription: function(n, o) {
userProfile.set('default-transcription-' + DOCUMENT_ID, n);
let itrans = userProfile.get('initialTranscriptions') || {};
itrans[DOCUMENT_ID] = n;
userProfile.set('initialTranscriptions', itrans);
this.getCurrentContent(n);
},
comparedTranscriptions: function(n, o) {
......@@ -82,7 +84,8 @@ var partVM = new Vue({
created() {
// this.fetch();
this.part.fetchPart(PART_ID, function() {
let tr = userProfile.get('default-transcription-' + DOCUMENT_ID)
let tr = userProfile.get('initialTranscriptions')
&& userProfile.get('initialTranscriptions')[DOCUMENT_ID]
|| this.part.transcriptions[0].pk;
this.selectedTranscription = tr;
}.bind(this));
......
......@@ -3,16 +3,16 @@
var API = {
'document': '/api/documents/' + DOCUMENT_ID,
'parts': '/api/documents/' + DOCUMENT_ID + '/parts/',
'part': '/api/documents/' + DOCUMENT_ID + '/parts/{part_pk}/'
'part': '/api/documents/' + DOCUMENT_ID + '/parts/{part_pk}/'
};
Dropzone.autoDiscover = false;
var g_dragged = null; // Note: chrome doesn't understand dataTransfer very well
var lastSelected = null;
function openWizard(proc) {
function openWizard(proc) {
var selected_num = partCard.getSelectedPks().length;
if(!proc.startsWith('import') && selected_num < 1) {
alert('Select at least one image.');
return;
......@@ -22,16 +22,17 @@ function openWizard(proc) {
// can't send more than one binarized image at a time
if (selected_num != 1) { $('#id_bw_image').attr('disabled', true); }
else { $('#id_bw_image').attr('disabled', false); }
// Reset the form
$('.process-part-form', '#'+proc+'-wizard').get(0).reset();
// initialize transcription field with user's last edited transcription
let itrans = userProfile.get('initialTranscriptions');
if (itrans && itrans[DOCUMENT_ID]) {
$('#process-part-form-export #id_transcription').val(itrans[DOCUMENT_ID]);
$('#process-part-form-train #id_transcription').val(itrans[DOCUMENT_ID]);
}
$('#'+proc+'-wizard').modal('show');
}
......@@ -50,7 +51,7 @@ class partCard {
this.locked = false;
this.api = API.part.replace('{part_pk}', this.pk);
var $new = $('.card', '#card-template').clone();
this.$element = $new;
this.domElement = this.$element.get(0);
......@@ -58,7 +59,7 @@ class partCard {
this.selectButton = $('.js-card-select-hdl', $new);
this.deleteButton = $('.js-card-delete', $new);
this.dropAfter = $('.js-drop', '#card-template').clone();
// fill template
$new.attr('id', $new.attr('id').replace('{pk}', this.pk));
this.updateThumbnail();
......@@ -73,7 +74,7 @@ class partCard {
}, this)).on('mouseout', $.proxy(function(ev) {
this.$element.attr('draggable', true);
}, this));
// add to the dom
$('#cards-container').append($new);
$('#cards-container').append(this.dropAfter);
......@@ -96,7 +97,7 @@ class partCard {
this.cancelTasksButton.click($.proxy(function(ev) {
this.cancelTasks();
}, this));
this.binarizedButton.click($.proxy(function(ev) {
this.select();
partCard.refreshSelectedCount();
......@@ -112,16 +113,16 @@ class partCard {
partCard.refreshSelectedCount();
openWizard('transcribe');
}, this));
this.index = $('.card', '#cards-container').index(this.$element);
// save a reference to this object in the card dom element
$new.data('partCard', this);
// add the image element to the lazy loader
imageObserver.observe($('img', $new).get(0));
this.defaultColor = this.$element.css('color');
//************* events **************
this.selectButton.on('click', $.proxy(function(ev) {
if (ev.shiftKey) {
......@@ -142,12 +143,12 @@ class partCard {
this.delete();
partCard.refreshSelectedCount();
}, this));
this.$element.on('dblclick', $.proxy(function(ev) {
this.toggleSelect();
partCard.refreshSelectedCount();
}, this));
// drag'n'drop
this.$element.on('dragstart', $.proxy(function(ev) {
if (this.locked) return;
......@@ -157,7 +158,7 @@ class partCard {
}, this));
this.$element.on('dragend', $.proxy(function(ev) {
$('.js-drop').removeClass('drop-target');
}, this));
}, this));
}
inQueue() {
......@@ -174,7 +175,7 @@ class partCard {
this.workflow['segment'] == 'ongoing' ||
this.workflow['transcribe'] == 'ongoing');
}
isCancelable() {
return (this.workflow['binarize'] == 'ongoing' ||
this.workflow['segment'] == 'ongoing' ||
......@@ -186,7 +187,7 @@ class partCard {
updateThumbnail() {
let uri, img = $('img.card-img-top', this.$element);
if (this.image.thumbnails && this.image.thumbnails['card'] != undefined) {
uri = this.image.thumbnails['card'];
} else {
......@@ -196,7 +197,7 @@ class partCard {
if (img.attr('src')) img.attr('src', uri);
img.attr('data-src', uri);
}
updateWorkflowIcons() {
var map = [
['convert', this.convertIcon],
......@@ -212,9 +213,9 @@ class partCard {
btn.removeClass('pending').removeClass('ongoing').removeClass('error').removeClass('done');
btn.addClass(this.workflow[proc]);
btn.attr('title', btn.data('title') + ' ('+this.workflow[proc]+')');
}
}
}
if (this.inQueue() || this.working()) {
this.lock();
} else {
......@@ -232,7 +233,7 @@ class partCard {
this.dropAfter.remove();
this.$element.remove();
}
select(scroll=true) {
if (this.locked) return;
lastSelected = this;
......@@ -266,7 +267,7 @@ class partCard {
this.$element.removeClass('locked');
this.$element.attr('draggable', true);
}
moveTo(index, upload) {
if (upload === undefined) upload = true;
// store the previous index in case of error
......@@ -297,7 +298,7 @@ class partCard {
this.updateWorkflowIcons();
}, this)).fail($.proxy(function(data){console.log("Couldn't cancel the task.");}));
}
delete() {
var posting = $.ajax({url:this.api, type: 'DELETE'})
.done($.proxy(function(data) {
......@@ -346,12 +347,12 @@ $(document).ready(function() {
ev.preventDefault();
}
});
$('#cards-container').on('dragleave','.js-drop', function(ev) {
ev.preventDefault();
$(ev.target).removeClass('drop-accept');
});
$('#cards-container').on('drop', '.js-drop', function(ev) {
ev.preventDefault();
$(ev.target).removeClass('drop-accept');
......@@ -363,7 +364,7 @@ $(document).ready(function() {
card.moveTo(index);
g_dragged = null;
});
// update workflow icons, send by notification through web socket
var workflow_order = ['pending', 'ongoing', 'error', 'done'];
$('#alerts-container').on('part:workflow', function(ev, data) {
......@@ -377,7 +378,7 @@ $(document).ready(function() {
if (data.task_id) card.task_ids[data.process] = data.task_id;
card.updateWorkflowIcons();
// special case, done with thumbnails:
if (data.process == 'generate_part_thumbnails' && data.status == 'done') {
card.image.thumbnails = data.data;
......@@ -428,15 +429,16 @@ $(document).ready(function() {
Alert.add(Date.now(), data.reason, 'warning');
});
$alertsContainer.on('import:fail', function(ev, data) {
$('#import-counter').text('failed');
$('#import-counter').text('Failed.');
$('#import-selected').removeClass('blink');
$('#cancel-import').hide();
Alert.add('import-failed', "Import failed because '"+data.reason+"'", 'danger');
});
$alertsContainer.on('import:done', function(ev, data) {
$('#import-counter').text('Done.');
$('#import-counter').parent().removeClass('ongoing');
$('#import-selected').removeClass('blink');
Alert.add('import-done', "Import finished!", 'success');
$('#cancel-import').hide();
});
$('#cancel-import').click(function(ev, data) {
let url = API.document + '/cancel_import/';
......@@ -450,7 +452,7 @@ $(document).ready(function() {
console.log("Couldn't cancel import");
});
});
// Exports
var $exportBtn = $('#document-export');
$alertsContainer.on('export:start', function(ev, data) {
......@@ -463,7 +465,18 @@ $(document).ready(function() {
$alertsContainer.on('export:done', function(ev, data) {
$exportBtn.removeClass('blink');
});
// disables including images for text export
$("#process-part-form-export #id_file_format").on('change', function(ev) {
let sel = ev.target;
if ($(sel).val() == 'text') {
$("#process-part-form-export #include_images").prop('checked', false);
$("#process-part-form-export #include_images").prop('disabled', true);
} else {
$("#process-part-form-export #include_images").prop('disabled', false);
}
});
// training
var max_accuracy = 0;
$alertsContainer.on('training:start', function(ev, data) {
......@@ -497,7 +510,7 @@ $(document).ready(function() {
console.log("Couldn't cancel training");
});
});
// create & configure dropzone
var imageDropzone = new Dropzone('.dropzone', {
paramName: "image",
......@@ -506,7 +519,7 @@ $(document).ready(function() {
// retryChunks: true,
parallelUploads: 1 // ! important or the 'order' field gets duplicates
});
//************* New card creation **************
imageDropzone.on("success", function(file, data) {
var card = new partCard(data);
......@@ -520,7 +533,7 @@ $(document).ready(function() {
}
}
});
// processor buttons
$('#select-all').click(function(ev) {
var cards = partCard.getRange(0, $('#cards-container .card').length);
......@@ -536,7 +549,7 @@ $(document).ready(function() {
});
partCard.refreshSelectedCount();
});
$('.js-proc-selected').click(function(ev) {
openWizard($(ev.target).data('proc'));
});
......@@ -544,7 +557,7 @@ $(document).ready(function() {
$('#process-part-form-binarize #id_threshold').on('input', function() {
$(this).attr('title', this.value);
});
$('.process-part-form').submit(function(ev) {
$('input[name=parts]', $form).val(JSON.stringify(partCard.getSelectedPks()));
ev.preventDefault();
......@@ -574,13 +587,13 @@ $(document).ready(function() {
/* Select card if coming from edit page */
var tabUrl = new URL(window.location);
var select = tabUrl.searchParams.get('select');
/* fetch the images and create the cards */
var counter=0;
var getNextParts = function(page) {
var uri = API.parts + '?paginate_by=50&page=' + page;
$.get(uri, function(data) {
counter += data.results.length;
counter += data.results.length;
$('#loading-counter').html(counter+'/'+data.count);
for (var i=0; i<data.results.length; i++) {
var pc = new partCard(data.results[i]);
......
from datetime import datetime
import json
import os
import io
import requests
from zipfile import ZipFile
from django import forms
from django.core.files.base import ContentFile
from django.db.models import Prefetch
from django.http import StreamingHttpResponse, HttpResponse
from django.template import loader
from django.utils.translation import gettext as _
from django.utils.text import slugify
from bootstrap.forms import BootstrapFormMixin
from core.models import Transcription, LineTranscription
from core.models import Transcription
from imports.models import DocumentImport
from imports.parsers import make_parser, ParseError
from imports.tasks import document_import, document_export
from reporting.models import TaskReport
from users.consumers import send_event
class ImportForm(BootstrapFormMixin, forms.Form):
......@@ -31,7 +26,7 @@ class ImportForm(BootstrapFormMixin, forms.Form):
override = forms.BooleanField(
initial=True, required=False,
label=_("Override existing segmentation."),
help_text=_("Destroys existing regions and lines before importing."))
help_text=_("Destroys existing regions, lines and any bound transcription before importing."))
iiif_uri = forms.URLField(
required=False,
label=_("IIIF manifesto uri"),
......@@ -98,11 +93,11 @@ class ImportForm(BootstrapFormMixin, forms.Form):
self.instance = self.current_import
else:
imp = DocumentImport(
document = self.document,
document=self.document,