Commit 2777da42 authored by Robin Tissot's avatar Robin Tissot
Browse files

Lot of fixes, better messages, code cleanup.

parent 4002a570
......@@ -783,17 +783,17 @@ def validate_polygon(value):
def validate_2_points(value):
if len(value) <= 2:
if len(value) < 2:
raise ValidationError(
_('Polygon needs to have at least 2 points %(value)s.'),
params={'value': value})
_('Polygon needs to have at least 2 points, it has %(length)d: %(value)s.'),
params={'length': len(value), 'value': value})
def validate_3_points(value):
if len(value) <= 3:
if len(value) < 3:
raise ValidationError(
_('Polygon needs to have at least 2 points %(value)s.'),
params={'value': value})
_('Polygon needs to have at least 3 points, it has %(length)d: %(value)s.'),
params={'length': len(value), 'value': value})
class Block(OrderedModel, models.Model):
......
......@@ -429,15 +429,16 @@ $(document).ready(function() {
Alert.add(Date.now(), data.reason, 'warning');
});
$alertsContainer.on('import:fail', function(ev, data) {
$('#import-counter').text('failed');
$('#import-counter').text('Failed.');
$('#import-selected').removeClass('blink');
$('#cancel-import').hide();
Alert.add('import-failed', "Import failed because '"+data.reason+"'", 'danger');
});
$alertsContainer.on('import:done', function(ev, data) {
$('#import-counter').text('Done.');
$('#import-counter').parent().removeClass('ongoing');
$('#import-selected').removeClass('blink');
Alert.add('import-done', "Import finished!", 'success');
$('#cancel-import').hide();
});
$('#cancel-import').click(function(ev, data) {
let url = API.document + '/cancel_import/';
......
......@@ -108,7 +108,7 @@ class ImportForm(BootstrapFormMixin, forms.Form):
# create a report and link to it
report = TaskReport.objects.create(
label='Import',
label=_('Import in %(document_name)s') % {'document_name': self.document.name},
user=self.user)
imp.report = report
......@@ -159,7 +159,10 @@ class ExportForm(BootstrapFormMixin, forms.Form):
file_format = self.cleaned_data['file_format']
transcription = self.cleaned_data['transcription']
report = TaskReport.objects.create(user=self.user, label='Export')
report = TaskReport.objects.create(
user=self.user,
label=_('Export %(document_name)s') % {
'document_name': self.document.name})
document_export.delay(file_format, self.user.pk, self.document.pk,
parts, transcription.pk, report.pk,
include_images=self.cleaned_data['include_images'])
......@@ -40,7 +40,8 @@ class DocumentImport(models.Model):
validators=[FileExtensionValidator(
allowed_extensions=XML_EXTENSIONS + ['json'])])
report = models.ForeignKey(TaskReport, max_length=64, null=True, blank=True,
report = models.ForeignKey(TaskReport, max_length=64,
null=True, blank=True,
on_delete=models.CASCADE)
processed = models.PositiveIntegerField(default=0)
total = models.PositiveIntegerField(default=None, null=True, blank=True)
......
......@@ -11,6 +11,7 @@ from lxml import etree
from django.conf import settings
from django.core.files.base import ContentFile
from django.core.validators import get_available_image_extensions
from django.db import transaction
from django.forms import ValidationError
from django.utils.translation import gettext as _
......@@ -97,18 +98,41 @@ class ZipParser(ParserDocument):
if index < start_at:
continue
with zfh.open(finfo) as zipedfh:
parser = make_parser(self.document, zipedfh,
name=self.name, report=self.report)
try:
for part in parser.parse(override=override):
yield part
file_extension = os.path.splitext(zipedfh.name)[1][1:]
# image
if file_extension in get_available_image_extensions():
try:
part = DocumentPart.objects.filter(
document=self.document,
original_filename=zipedfh.name
)[0]
except IndexError:
part = DocumentPart(
document=self.document,
original_filename=zipedfh.name
)
part.image.save(zipedfh.name, ContentFile(zipedfh.read()))
part.save()
# xml
elif file_extension == 'xml':
parser = make_parser(self.document, zipedfh,
name=self.name, report=self.report)
for part in parser.parse(override=override):
yield part
except IndexError:
# no file extension!?
pass
except ParseError as e:
# we let go to try other documents
msg = _("Parse error in {filename}: {error}, skipping it.").format(
filename=self.file.name, error=e.args[0]
)
logger.warning(msg)
self.report.append(msg)
if self.report:
self.report.append(msg)
if user:
user.notify(msg, id="import:warning", level="warning")
......@@ -117,6 +141,10 @@ class XMLParser(ParserDocument):
ACCEPTED_SCHEMAS = ()
def __init__(self, document, file_handler, report, transcription_name=None, xml_root=None):
super().__init__(document,
file_handler,
transcription_name=transcription_name,
report=report)
if xml_root is not None:
self.root = xml_root
try:
......@@ -135,8 +163,6 @@ class XMLParser(ParserDocument):
self.root = etree.parse(self.file).getroot()
except (AttributeError, etree.XMLSyntaxError) as e:
raise ParseError("Invalid XML. %s" % e.args[0])
super().__init__(document, file_handler, transcription_name=transcription_name,
report=report)
def validate(self):
if self.schema_location in self.ACCEPTED_SCHEMAS:
......@@ -146,8 +172,9 @@ class XMLParser(ParserDocument):
schema_root = etree.XML(content)
except requests.exceptions.RequestException as e:
logger.exception(e)
self.report.append("Can't reach validation document %s, %s" % (self.schema_location, OWN_RISK))
if self.report:
self.report.append("Can't reach validation document %s, %s" % (
self.schema_location, OWN_RISK))
else:
try:
xmlschema = etree.XMLSchema(schema_root)
......@@ -157,10 +184,12 @@ class XMLParser(ParserDocument):
etree.DocumentInvalid,
etree.XMLSyntaxError,
) as e:
self.report.append("Document didn't validate. %s, %s" % (e.args[0], OWN_RISK))
if self.report:
self.report.append("Document didn't validate. %s, %s" % (e.args[0], OWN_RISK))
else:
self.report.append("Document Schema %s is not in the accepted escriptiium list. Valid schemas are: %s, %s" %
(self.schema_location, self.ACCEPTED_SCHEMAS, OWN_RISK))
if self.report:
self.report.append("Document Schema %s is not in the accepted escriptiium list. Valid schemas are: %s, %s" %
(self.schema_location, self.ACCEPTED_SCHEMAS, OWN_RISK))
def get_filename(self, pageTag):
raise NotImplementedError
......@@ -213,11 +242,12 @@ class XMLParser(ParserDocument):
)[0]
except IndexError:
# TODO: check for the image in the zip
self.report.append(
_("No match found for file {} with filename \"{}\".").format(
self.file.name, filename
if self.report:
self.report.append(
_("No match found for file {} with filename \"{}\".").format(
self.file.name, filename
)
)
)
else:
# if something fails, revert everything for this document part
with transaction.atomic():
......@@ -240,9 +270,10 @@ class XMLParser(ParserDocument):
try:
block.full_clean()
except ValidationError as e:
self.report.append(
"Block in '{filen}' line N°{line} was skipped because: {error}".format(
filen=self.file.name, line=blockTag.sourceline, error=e))
if self.report:
self.report.append(
"Block in '{filen}' line N°{line} was skipped because: {error}".format(
filen=self.file.name, line=blockTag.sourceline, error=e))
else:
block.save()
else:
......@@ -265,8 +296,9 @@ class XMLParser(ParserDocument):
try:
line.full_clean()
except ValidationError as e:
self.report.append("Line in '{filen}' line N°{line} was skipped because: {error}".format(
filen=self.file.name, line=blockTag.sourceline, error=e))
if self.report:
self.report.append("Line in '{filen}' line N°{line} was skipped because: {error}".format(
filen=self.file.name, line=blockTag.sourceline, error=e))
else:
line.save()
......@@ -275,10 +307,10 @@ class XMLParser(ParserDocument):
if tc:
self.make_transcription(line, lineTag, tc, user=user)
# TODO: store glyphs too
logger.info("Uncompressed and parsed %s" % self.file.name)
part.calculate_progress()
yield part
# TODO: store glyphs too
logger.info("Uncompressed and parsed %s" % self.file.name)
part.calculate_progress()
yield part
class AltoParser(XMLParser):
......@@ -367,7 +399,8 @@ The alto file should contain a Description/sourceImageInformation/fileName tag f
except ValueError:
msg = "Invalid baseline %s in {filen} line {linen}" % (baseline, self.file.name, lineTag.sourceline)
logger.warning(msg)
self.report.append(msg)
if self.report:
self.report.append(msg)
polygon = lineTag.find("Shape/Polygon", self.root.nsmap)
if polygon is not None:
......@@ -377,7 +410,8 @@ The alto file should contain a Description/sourceImageInformation/fileName tag f
except ValueError:
msg = "Invalid polygon %s in {filen} line {linen}" % (polygon, self.file.name, lineTag.sourceline)
logger.warning(msg)
self.report.append(msg)
if self.report:
self.report.append(msg)
else:
line.box = [
int(lineTag.get("HPOS")),
......@@ -590,13 +624,12 @@ class TranskribusPageXmlParser(PagexmlParser):
A Pagexml Parser for documents exported from Transkribus to handle data
"""
def validate(self):
warnings.warn(
"Validation is skipped for Transkribus Files but coordinates will be corrected",
SyntaxWarning,
2,
)
# def validate(self):
# warnings.warn(
# "Validation is skipped for Transkribus Files but coordinates will be corrected",
# SyntaxWarning,
# 2,
# )
def clean_coords(self, coordTag):
return [
......
......@@ -5,10 +5,10 @@ from zipfile import ZipFile
from django.apps import apps
from django.conf import settings
from django.urls import reverse
from django.db.models import Q, Prefetch
from django.template import loader
from django.utils.text import slugify
from django.utils.translation import gettext as _
from celery import shared_task
......@@ -110,6 +110,9 @@ def document_export(task, file_format, user_pk, document_pk, part_pks,
parts = DocumentPart.objects.filter(document=document, pk__in=part_pks)
with ZipFile(filepath, 'w') as zip_:
for part in parts:
if include_images:
# Note adds image before the xml file
zip_.write(part.image.path, part.filename)
try:
page = tplt.render({
'valid_block_types': document.valid_block_types.all(),
......@@ -131,19 +134,14 @@ def document_export(task, file_format, user_pk, document_pk, part_pks,
))
else:
zip_.writestr('%s.xml' % os.path.splitext(part.filename)[0], page)
if include_images:
zip_.write(part.image.path, part.filename)
report.end()
zip_.close()
# send websocket msg
rel_path = os.path.relpath(filepath, settings.MEDIA_ROOT)
report_uri = reverse('report-detail', kwargs={'pk': report.pk})
user.notify('Export ready!', level='success', links=[
{'text': 'Download', 'src': settings.MEDIA_URL + rel_path},
{'text': 'Report', 'src': report_uri},
])
report.end(extra_links=[{'text': _('Download'),
'src': settings.MEDIA_URL + rel_path}])
# send websocket msg
send_event('document', document.pk, "export:done", {
"id": document.pk
})
......
from datetime import datetime, timezone
from django.contrib.auth import get_user_model
from django.db import models
from django.urls import reverse
from django.utils.translation import gettext_lazy as _
from django.contrib.auth import get_user_model
User = get_user_model()
......@@ -38,6 +39,10 @@ class TaskReport(models.Model):
def append(self, text):
self.messages += text + '\n'
@property
def uri(self):
return reverse('report-detail', kwargs={'pk': self.pk})
def start(self, task_id):
self.task_id = task_id
self.workflow_state = self.WORKFLOW_STATE_STARTED
......@@ -50,8 +55,18 @@ class TaskReport(models.Model):
self.append(message)
self.save()
def end(self):
self.user.notify(_('%(task_label)s error!') % {'task_label': self.label},
level='error',
links=[{'text': 'Report', 'src': self.uri}])
def end(self, extra_links=None):
self.workflow_state = self.WORKFLOW_STATE_DONE
self.done_at = datetime.now(timezone.utc)
self.append('Done.')
self.save()
links = extra_links or []
if self.messages != '':
links.append({'text': 'Report', 'src': self.uri})
self.user.notify(_('%(task_label)s done!') % {'task_label': self.label},
level='success',
links=links)
......@@ -10,7 +10,9 @@ class ReportList(LoginRequiredMixin, ListView):
def get_queryset(self):
qs = super().get_queryset()
return qs.filter(user=self.request.user)
return (qs.filter(user=self.request.user)
.exclude(messages='')
.order_by('-queued_at'))
class ReportDetail(LoginRequiredMixin, DetailView):
......
......@@ -11,10 +11,11 @@ class Alert {
var $new = $('.alert', '#alert-tplt').clone();
$new.addClass('alert-' + this.level);
$('.message', $new).html(message);
for (let i=0; i<this.links.length; i++) {
let link = $('<div>').html('<a href="'+this.links[i].src+'" >'+this.links[i].text+'</a>');
$('.additional', $new).append(link).css('display', 'block');
if (this.links !== undefined) {
for (let i=0; i<this.links.length; i++) {
let link = $('<div>').html('<a href="'+this.links[i].src+'" >'+this.links[i].text+'</a>');
$('.additional', $new).append(link).css('display', 'block');
}
}
this.$element = $new;
$('#alerts-container').append($new);
......
......@@ -8,6 +8,6 @@
<div>{% trans "Queued at:" %} {{ report.queued_at }}</div>
{% if report.workflow_state > 0 %}<div>{% trans "Started at:" %} {{ report.queued_at }}</div>{% endif %}
{% if report.workflow_state > 1 %}<div>{% trans "Ended at:" %} {{ report.done_at }}</div>{% endif %}
<div class="jumbotron">{{ report.messages }}</div>
<div class="jumbotron">{{ report.messages|linebreaks }}</div>
</div>
{% endblock %}
......@@ -3,15 +3,17 @@
{% block body %}
<div class="container">
<table class="table table-hover">
{% for report in object_list %}
<div>
{{ report.label }}
{{ report.get_workflow_state_display }}
{{ report.queued_at }}
<a href="{% url 'report-detail' report.pk %}">{% trans "Details" %}</a>
</div>
<tr>
<td>{{ report.label }}</td>
<td>{{ report.get_workflow_state_display }}</td>
<td>{{ report.queued_at }}</td>
<td><a href="{% url 'report-detail' report.pk %}">{% trans "Details" %}</a></td>
</tr>
{% empty %}
{% trans "You don't have any task report yet." %}
<tr><td>{% trans "You don't have any task report yet." %}</td></tr>
{% endfor %}
</table>
</div>
{% endblock %}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment