From 5ac4e7df28e1c5031b7a2368704234d8c89b58c3 Mon Sep 17 00:00:00 2001
From: Robin Tissot <tissotrobin@gmail.com>
Date: Tue, 2 Jun 2020 10:51:30 +0200
Subject: [PATCH 01/14] Rm env file not removed by merge.
---
.env | 3 ---
1 file changed, 3 deletions(-)
delete mode 100644 .env
diff --git a/.env b/.env
deleted file mode 100644
index 00b6a4c6..00000000
--- a/.env
+++ /dev/null
@@ -1,3 +0,0 @@
-CELERY_MAIN_CORES=2
-CELERY_LOW_CORES=2
-FLOWER_BASIC_AUTH=flower:whatever
\ No newline at end of file
--
GitLab
From 7a0747311d58a8f193e06c38ac8f6d48a1f77af4 Mon Sep 17 00:00:00 2001
From: elhassane <elhassanegargem@gmail.com>
Date: Tue, 17 Nov 2020 15:15:03 +0100
Subject: [PATCH 02/14] tests for model endpoint
---
app/apps/api/tests.py | 68 +++++++++++++++++++++++++++++++++-
app/apps/core/tests/factory.py | 2 +-
2 files changed, 67 insertions(+), 3 deletions(-)
diff --git a/app/apps/api/tests.py b/app/apps/api/tests.py
index 7de43df7..802a8ace 100644
--- a/app/apps/api/tests.py
+++ b/app/apps/api/tests.py
@@ -8,9 +8,9 @@ from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import override_settings
from django.urls import reverse
-from core.models import Block, Line, Transcription, LineTranscription
+from core.models import Block, Line, Transcription, LineTranscription, OcrModel
from core.tests.factory import CoreFactoryTestCase
-
+from api.serializers import DocumentProcessSerializer
class UserViewSetTestCase(CoreFactoryTestCase):
def setUp(self):
@@ -37,6 +37,31 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
super().setUp()
self.doc = self.factory.make_document()
self.doc2 = self.factory.make_document(owner=self.doc.owner)
+ self.part = self.factory.make_part(document=self.doc)
+ self.part2 = self.factory.make_part(document=self.doc)
+ self.model_uri = reverse('api:document-model',kwargs={'pk': self.doc.pk})
+
+
+ self.line = Line.objects.create(
+ box=[10, 10, 50, 50],
+ document_part=self.part)
+ self.line2 = Line.objects.create(
+ box=[10, 60, 50, 100],
+ document_part=self.part)
+ self.transcription = Transcription.objects.create(
+ document=self.part.document,
+ name='test')
+ self.transcription2 = Transcription.objects.create(
+ document=self.part.document,
+ name='tr2')
+ self.lt = LineTranscription.objects.create(
+ transcription=self.transcription,
+ line=self.line,
+ content='test')
+ self.lt2 = LineTranscription.objects.create(
+ transcription=self.transcription2,
+ line=self.line2,
+ content='test2')
def test_list(self):
self.client.force_login(self.doc.owner)
@@ -62,6 +87,45 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
# Note: raises a 404 instead of 403 but its fine
self.assertEqual(resp.status_code, 404)
+ def test_segtrain_less_two_parts(self):
+ self.client.force_login(self.doc.owner)
+
+
+ resp = self.client.post(self.model_uri,data={
+ 'parts': [self.part.pk],
+ 'transcription':self.transcription.pk,
+ 'task': DocumentProcessSerializer.TASK_SEGTRAIN,
+ })
+ self.assertEqual(resp.status_code, 400)
+
+ def test_segtrain_new_model(self):
+ self.client.force_login(self.doc.owner)
+
+
+ resp = self.client.post(self.model_uri,data={
+ 'parts': [self.part.pk, self.part2.pk],
+ 'transcription':self.transcription.pk,
+ 'task': DocumentProcessSerializer.TASK_SEGTRAIN,
+ 'new_model':'new model'
+ })
+ self.assertEqual(resp.status_code, 200)
+ self.assertEqual(OcrModel.objects.count(),1)
+ self.assertEqual(OcrModel.objects.first().name,"new model")
+
+ def test_segment(self):
+
+ self.client.force_login(self.doc.owner)
+ model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT,document=self.doc)
+
+ resp = self.client.post(self.model_uri, data={
+ 'parts': [self.part.pk,self.part2.pk],
+ 'task': DocumentProcessSerializer.TASK_SEGMENT,
+ 'segmentation_steps':'both',
+ 'seg_model': model.pk,
+ })
+
+ self.assertEqual(resp.status_code, 200)
+
# not used
# def test_update
# def test_create
diff --git a/app/apps/core/tests/factory.py b/app/apps/core/tests/factory.py
index 44bb89ba..bb2bb138 100644
--- a/app/apps/core/tests/factory.py
+++ b/app/apps/core/tests/factory.py
@@ -94,7 +94,7 @@ class CoreFactory():
def make_model(self, job=OcrModel.MODEL_JOB_RECOGNIZE, document=None):
spec = '[1,48,0,1 Lbx100 Do O1c10]'
nn = vgsl.TorchVGSLModel(spec)
- model_name = 'test-model'
+ model_name = 'test-model.mlmodel'
model = OcrModel.objects.create(name=model_name,
document=document,
job=job)
--
GitLab
From 355db50c3df3fcfbf46522f8b9b6fb4b9fb69163 Mon Sep 17 00:00:00 2001
From: elhassane <elhassanegargem@gmail.com>
Date: Tue, 17 Nov 2020 15:16:05 +0100
Subject: [PATCH 03/14] model endpoint serializer and view
---
app/apps/api/serializers.py | 210 +++++++++++++++++++++++++++++++++++-
app/apps/api/views.py | 27 ++++-
2 files changed, 234 insertions(+), 3 deletions(-)
diff --git a/app/apps/api/serializers.py b/app/apps/api/serializers.py
index 705c455a..3d73b8b9 100644
--- a/app/apps/api/serializers.py
+++ b/app/apps/api/serializers.py
@@ -1,9 +1,14 @@
import bleach
import logging
import html
+import json
from django.conf import settings
from django.db.utils import IntegrityError
+from django.utils.translation import gettext_lazy as _
+from django.core.validators import FileExtensionValidator, MinValueValidator, MaxValueValidator
+from django.utils.functional import cached_property
+from django.shortcuts import get_object_or_404
from rest_framework import serializers
from easy_thumbnails.files import get_thumbnailer
@@ -16,7 +21,9 @@ from core.models import (Document,
Transcription,
LineTranscription,
BlockType,
- LineType)
+ LineType,
+ OcrModel,
+ AlreadyProcessingException)
logger = logging.getLogger(__name__)
@@ -259,3 +266,204 @@ class PartDetailSerializer(PartSerializer):
nex = DocumentPart.objects.filter(
document=instance.document, order__gt=instance.order).order_by('order').first()
return nex and nex.pk or None
+
+
+class OcrModelSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = OcrModel
+ fields = ('pk', 'name','file','job','owner','training','training_epoch',
+ 'training_accuracy','training_total','training_errors','document','script',)
+
+
+class DocumentProcessSerializer(serializers.Serializer):
+
+ TASK_BINARIZE = 'binarize'
+ TASK_SEGMENT = 'segment'
+ TASK_TRAIN = 'train'
+ TASK_SEGTRAIN = 'segtrain'
+ TASK_TRANSCRIBE = 'transcribe'
+
+ parts = serializers.ListField(
+ child=serializers.IntegerField()
+ )
+
+ task = serializers.ChoiceField(required=True,
+ choices = (
+ (TASK_BINARIZE, TASK_BINARIZE),
+ (TASK_SEGMENT, TASK_BINARIZE),
+ (TASK_TRAIN, TASK_BINARIZE),
+ (TASK_TRANSCRIBE, TASK_BINARIZE),
+ (TASK_SEGTRAIN, TASK_BINARIZE),
+ )
+ )
+
+ # binarization
+ bw_image = serializers.ImageField(required=False)
+ BINARIZER_CHOICES = (('kraken', _("Kraken")),)
+ binarizer = serializers.ChoiceField(required=False,
+ choices=BINARIZER_CHOICES,
+ initial='kraken')
+ threshold = serializers.FloatField(
+ required=False, initial=0.5,
+ validators=[MinValueValidator(0.1), MaxValueValidator(1)],
+ help_text=_('Increase it for low contrast documents, if the letters are not visible enough.'),
+ )
+ # segment
+ SEGMENTATION_STEPS_CHOICES = (
+ ('both', _('Lines and regions')),
+ ('lines', _('Lines Baselines and Masks')),
+ ('masks', _('Only lines Masks')),
+ ('regions', _('Regions')),
+ )
+
+ segmentation_steps = serializers.ChoiceField(choices=SEGMENTATION_STEPS_CHOICES,
+ initial='both', required=False)
+ seg_model = serializers.IntegerField(required=False)
+
+ override = serializers.BooleanField(required=False, initial=True,
+ help_text=_(
+ "If checked, deletes existing segmentation <b>and bound transcriptions</b> first!"))
+ TEXT_DIRECTION_CHOICES = (('horizontal-lr', _("Horizontal l2r")),
+ ('horizontal-rl', _("Horizontal r2l")),
+ ('vertical-lr', _("Vertical l2r")),
+ ('vertical-rl', _("Vertical r2l")))
+ text_direction = serializers.ChoiceField(initial='horizontal-lr', required=False,
+ choices=TEXT_DIRECTION_CHOICES)
+ # transcribe
+ upload_model = serializers.FileField(required=False,
+ validators=[FileExtensionValidator(
+ allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
+ ocr_model = OcrModelSerializer(required=False)
+
+ # train
+ new_model = serializers.CharField(required=False, label=_('Model name'))
+ train_model = OcrModelSerializer(required=False)
+ transcription = serializers.PrimaryKeyRelatedField(many=False,read_only=True)
+
+ # segtrain
+ segtrain_model = OcrModelSerializer(required=False)
+
+
+ def __init__(self, document, user, *args, **kwargs):
+ self.document = document
+ self.user = user
+ if self.document.read_direction == self.document.READ_DIRECTION_RTL:
+ self.initial['text_direction'] = 'horizontal-rl'
+ super().__init__(*args, **kwargs)
+
+ def validate_bw_image(self,img):
+ if not img:
+ return
+ if len(self.document_parts) != 1:
+ raise serializers.ValidationError({'bw_image':_("Uploaded image with more than one selected image.")})
+ # Beware: don't close the file here !
+ fh = Image.open(img)
+ if fh.mode not in ['1', 'L']:
+ raise serializers.ValidationError({'bw_image':_("Uploaded image should be black and white.")})
+ isize = (self.document_parts[0].image.width, self.document_parts[0].image.height)
+ if fh.size != isize:
+ raise serializers.ValidationError({'bw_image':_("Uploaded image should be the same size as original image {size}.").format(size=isize)})
+ return img
+
+ def validate_train_model(self,train_model):
+
+ if train_model and train_model.training:
+ raise AlreadyProcessingException
+ return train_model
+
+ def validate_seg_model(self,value):
+ model = get_object_or_404(OcrModel,pk=value)
+ return model
+
+
+ def validate(self,data):
+
+ task = data['task']
+ parts = data['parts']
+
+ self.document_parts = DocumentPart.objects.filter(
+ document=self.document, pk__in=parts)
+
+ if task == self.TASK_SEGMENT:
+ model_job = OcrModel.MODEL_JOB_SEGMENT
+ elif task == self.TASK_SEGTRAIN:
+ model_job = OcrModel.MODEL_JOB_SEGMENT
+ if task == self.TASK_SEGTRAIN and len(parts) < 2:
+ raise serializers.ValidationError("Segmentation training requires at least 2 images.")
+ else:
+ model_job = OcrModel.MODEL_JOB_RECOGNIZE
+
+ if task == self.TASK_TRAIN and data.get('train_model'):
+ model = data.get('train_model')
+ elif task == self.TASK_SEGTRAIN and data.get('segtrain_model'):
+ model = data.get('segtrain_model')
+ elif data.get('upload_model'):
+ model = OcrModel.objects.create(
+ document=self.document_parts[0].document,
+ owner=self.user,
+ name=data['upload_model'].name.rsplit('.', 1)[0],
+ job=model_job)
+ # Note: needs to save the file in a second step because the path needs the db PK
+ model.file = data['upload_model']
+ model.save()
+
+ elif data.get('new_model'):
+ # file will be created by the training process
+ model = OcrModel.objects.create(
+ document=self.document_parts[0].document,
+ owner=self.user,
+ name=data['new_model'],
+ job=model_job)
+ elif data.get('ocr_model'):
+ model = data.get('ocr_model')
+ elif data.get('seg_model'):
+ model = data.get('seg_model')
+ else:
+ if task in (self.TASK_TRAIN, self.TASK_SEGTRAIN):
+ raise serializers.ValidationError(
+ _("Either select a name for your new model or an existing one."))
+ else:
+ model = None
+
+ data['model'] = model
+ return data
+
+ def process(self):
+
+ task = self.validated_data.get('task')
+
+ model = self.validated_data.get('model')
+ if task == self.TASK_BINARIZE:
+ if len(self.document_parts) == 1 and self.validated_data.get('bw_image'):
+ self.document_parts[0].bw_image = self.validated_data['bw_image']
+ self.document_parts[0].save()
+ else:
+ for part in self.document_parts:
+ part.task('binarize',
+ user_pk=self.user.pk,
+ threshold=self.validated_data.get('threshold'))
+
+ elif task == self.TASK_SEGMENT:
+ for part in self.document_parts:
+ part.task('segment',
+ user_pk=self.user.pk,
+ steps=self.validated_data.get('segmentation_steps'),
+ text_direction=self.validated_data.get('text_direction'),
+ model_pk=model and model.pk or None,
+ override=self.validated_data.get('override'))
+
+ elif task == self.TASK_TRANSCRIBE:
+ for part in self.document_parts:
+ part.task('transcribe',
+ user_pk=self.user.pk,
+ model_pk=model and model.pk or None)
+
+ elif task == self.TASK_TRAIN:
+ model.train(self.document_parts,
+ self.validated_data['transcription'],
+ user=self.user)
+
+ elif task == self.TASK_SEGTRAIN:
+ model.segtrain(self.document,
+ self.document_parts,
+ user=self.user)
\ No newline at end of file
diff --git a/app/apps/api/views.py b/app/apps/api/views.py
index e41e1939..e26dc26b 100644
--- a/app/apps/api/views.py
+++ b/app/apps/api/views.py
@@ -25,7 +25,9 @@ from api.serializers import (UserOnboardingSerializer,
DetailedLineSerializer,
LineOrderSerializer,
TranscriptionSerializer,
- LineTranscriptionSerializer)
+ LineTranscriptionSerializer,
+ DocumentProcessSerializer)
+
from core.models import (Document,
DocumentPart,
Block,
@@ -33,8 +35,11 @@ from core.models import (Document,
BlockType,
LineType,
Transcription,
- LineTranscription)
+ LineTranscription,
+ AlreadyProcessingException)
+
from core.tasks import recalculate_masks
+from core.forms import DocumentProcessForm
from users.models import User
from imports.forms import ImportForm, ExportForm
from imports.parsers import ParseError
@@ -120,6 +125,24 @@ class DocumentViewSet(ModelViewSet):
else:
return self.form_error(json.dumps(form.errors))
+ @action(detail=True, methods=['post'])
+ def model(self, request, pk=None):
+ document = self.get_object()
+ self.serializer_class = DocumentProcessSerializer
+ serializer = DocumentProcessSerializer(document=document, user=request.user,data=request.data)
+ if serializer.is_valid():
+ try:
+ serializer.process()
+ except AlreadyProcessingException:
+ return Response(status=status.HTTP_400_BAD_REQUEST,data={'status': 'error', 'error':'Already processing.'})
+
+ return Response(status=status.HTTP_200_OK,data={'status': 'ok'})
+ else:
+ return Response(status=status.HTTP_400_BAD_REQUEST,data={'status': 'error', 'error':serializer.errors})
+
+
+
+
class DocumentPermissionMixin():
def get_queryset(self):
--
GitLab
From 8200e3c1bf9a40b70a3b5c6111304f16dd996e5e Mon Sep 17 00:00:00 2001
From: elhassane <elhassanegargem@gmail.com>
Date: Thu, 19 Nov 2020 15:53:56 +0100
Subject: [PATCH 04/14] test_segment_file_upload
---
app/apps/api/tests.py | 27 +++++++++++++++++++--------
1 file changed, 19 insertions(+), 8 deletions(-)
diff --git a/app/apps/api/tests.py b/app/apps/api/tests.py
index 802a8ace..c85fe17c 100644
--- a/app/apps/api/tests.py
+++ b/app/apps/api/tests.py
@@ -43,10 +43,10 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
self.line = Line.objects.create(
- box=[10, 10, 50, 50],
+ mask=[10, 10, 50, 50],
document_part=self.part)
self.line2 = Line.objects.create(
- box=[10, 60, 50, 100],
+ mask=[10, 60, 50, 100],
document_part=self.part)
self.transcription = Transcription.objects.create(
document=self.part.document,
@@ -123,7 +123,18 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
'segmentation_steps':'both',
'seg_model': model.pk,
})
+ self.assertEqual(resp.status_code, 200)
+
+ def test_segment_file_upload(self):
+ self.client.force_login(self.doc.owner)
+ model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT,document=self.doc)
+ resp = self.client.post(self.model_uri, data={
+ 'parts': [self.part.pk, self.part2.pk],
+ 'task': DocumentProcessSerializer.TASK_SEGMENT,
+ 'segmentation_steps': 'both',
+ 'upload_model': SimpleUploadedFile(model.name,model.file.read())
+ })
self.assertEqual(resp.status_code, 200)
# not used
@@ -279,15 +290,15 @@ class LineViewSetTestCase(CoreFactoryTestCase):
box=[10, 10, 200, 200],
document_part=self.part)
self.line = Line.objects.create(
- box=[60, 10, 100, 50],
+ mask=[60, 10, 100, 50],
document_part=self.part,
block=self.block)
self.line2 = Line.objects.create(
- box=[90, 10, 70, 50],
+ mask=[90, 10, 70, 50],
document_part=self.part,
block=self.block)
self.orphan = Line.objects.create(
- box=[0, 0, 10, 10],
+ mask=[0, 0, 10, 10],
document_part=self.part,
block=None)
@@ -358,10 +369,10 @@ class LineTranscriptionViewSetTestCase(CoreFactoryTestCase):
self.part = self.factory.make_part()
self.user = self.part.document.owner
self.line = Line.objects.create(
- box=[10, 10, 50, 50],
+ mask=[10, 10, 50, 50],
document_part=self.part)
self.line2 = Line.objects.create(
- box=[10, 60, 50, 100],
+ mask=[10, 60, 50, 100],
document_part=self.part)
self.transcription = Transcription.objects.create(
document=self.part.document,
@@ -425,7 +436,7 @@ class LineTranscriptionViewSetTestCase(CoreFactoryTestCase):
uri = reverse('api:linetranscription-bulk-create',
kwargs={'document_pk': self.part.document.pk, 'part_pk': self.part.pk})
ll = Line.objects.create(
- box=[10, 10, 50, 50],
+ mask=[10, 10, 50, 50],
document_part=self.part)
with self.assertNumQueries(10):
resp = self.client.post(
--
GitLab
From b971a5c591a6aa743d573649b61d095f9ba18ca4 Mon Sep 17 00:00:00 2001
From: elhassane <elhassanegargem@gmail.com>
Date: Fri, 20 Nov 2020 11:48:18 +0100
Subject: [PATCH 05/14] rename seg_steps
---
app/apps/api/serializers.py | 19 ++++++++++++++-----
app/apps/api/tests.py | 4 ++--
2 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/app/apps/api/serializers.py b/app/apps/api/serializers.py
index 3d73b8b9..cc1c1475 100644
--- a/app/apps/api/serializers.py
+++ b/app/apps/api/serializers.py
@@ -309,14 +309,14 @@ class DocumentProcessSerializer(serializers.Serializer):
help_text=_('Increase it for low contrast documents, if the letters are not visible enough.'),
)
# segment
- SEGMENTATION_STEPS_CHOICES = (
+ SEG_STEPS_CHOICES = (
('both', _('Lines and regions')),
('lines', _('Lines Baselines and Masks')),
('masks', _('Only lines Masks')),
('regions', _('Regions')),
)
- segmentation_steps = serializers.ChoiceField(choices=SEGMENTATION_STEPS_CHOICES,
+ seg_steps = serializers.ChoiceField(choices=SEG_STEPS_CHOICES,
initial='both', required=False)
seg_model = serializers.IntegerField(required=False)
@@ -333,7 +333,8 @@ class DocumentProcessSerializer(serializers.Serializer):
upload_model = serializers.FileField(required=False,
validators=[FileExtensionValidator(
allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
- ocr_model = OcrModelSerializer(required=False)
+
+ ocr_model = serializers.IntegerField(required=False)
# train
new_model = serializers.CharField(required=False, label=_('Model name'))
@@ -341,7 +342,7 @@ class DocumentProcessSerializer(serializers.Serializer):
transcription = serializers.PrimaryKeyRelatedField(many=False,read_only=True)
# segtrain
- segtrain_model = OcrModelSerializer(required=False)
+ segtrain_model = serializers.IntegerField(required=False)
def __init__(self, document, user, *args, **kwargs):
@@ -375,6 +376,14 @@ class DocumentProcessSerializer(serializers.Serializer):
model = get_object_or_404(OcrModel,pk=value)
return model
+ def validate_ocr_model(self,value):
+ model = get_object_or_404(OcrModel,pk=value)
+ return model
+
+ def validate_segtrain_model(self,value):
+ model = get_object_or_404(OcrModel,pk=value)
+ return model
+
def validate(self,data):
@@ -447,7 +456,7 @@ class DocumentProcessSerializer(serializers.Serializer):
for part in self.document_parts:
part.task('segment',
user_pk=self.user.pk,
- steps=self.validated_data.get('segmentation_steps'),
+ steps=self.validated_data.get('seg_steps'),
text_direction=self.validated_data.get('text_direction'),
model_pk=model and model.pk or None,
override=self.validated_data.get('override'))
diff --git a/app/apps/api/tests.py b/app/apps/api/tests.py
index c85fe17c..8f5ce6a2 100644
--- a/app/apps/api/tests.py
+++ b/app/apps/api/tests.py
@@ -120,7 +120,7 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
resp = self.client.post(self.model_uri, data={
'parts': [self.part.pk,self.part2.pk],
'task': DocumentProcessSerializer.TASK_SEGMENT,
- 'segmentation_steps':'both',
+ 'seg_steps':'both',
'seg_model': model.pk,
})
self.assertEqual(resp.status_code, 200)
@@ -132,7 +132,7 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
resp = self.client.post(self.model_uri, data={
'parts': [self.part.pk, self.part2.pk],
'task': DocumentProcessSerializer.TASK_SEGMENT,
- 'segmentation_steps': 'both',
+ 'seg_steps': 'both',
'upload_model': SimpleUploadedFile(model.name,model.file.read())
})
self.assertEqual(resp.status_code, 200)
--
GitLab
From 3757f7204a4e7ec05f3f983326659f151f854fd3 Mon Sep 17 00:00:00 2001
From: elhassane <elhassanegargem@gmail.com>
Date: Fri, 20 Nov 2020 11:52:39 +0100
Subject: [PATCH 06/14] pep8
---
app/apps/api/serializers.py | 17 ++++++++---------
1 file changed, 8 insertions(+), 9 deletions(-)
diff --git a/app/apps/api/serializers.py b/app/apps/api/serializers.py
index cc1c1475..1c67c8a6 100644
--- a/app/apps/api/serializers.py
+++ b/app/apps/api/serializers.py
@@ -344,10 +344,10 @@ class DocumentProcessSerializer(serializers.Serializer):
# segtrain
segtrain_model = serializers.IntegerField(required=False)
-
def __init__(self, document, user, *args, **kwargs):
self.document = document
self.user = user
+ self.document_parts = []
if self.document.read_direction == self.document.READ_DIRECTION_RTL:
self.initial['text_direction'] = 'horizontal-rl'
super().__init__(*args, **kwargs)
@@ -372,20 +372,19 @@ class DocumentProcessSerializer(serializers.Serializer):
raise AlreadyProcessingException
return train_model
- def validate_seg_model(self,value):
- model = get_object_or_404(OcrModel,pk=value)
+ def validate_seg_model(self, value):
+ model = get_object_or_404(OcrModel, pk=value)
return model
- def validate_ocr_model(self,value):
- model = get_object_or_404(OcrModel,pk=value)
+ def validate_ocr_model(self, value):
+ model = get_object_or_404(OcrModel, pk=value)
return model
- def validate_segtrain_model(self,value):
- model = get_object_or_404(OcrModel,pk=value)
+ def validate_segtrain_model(self, value):
+ model = get_object_or_404(OcrModel, pk=value)
return model
-
- def validate(self,data):
+ def validate(self, data):
task = data['task']
parts = data['parts']
--
GitLab
From 7f1fddb8f6f6eb6ae6639821b7879feffbead5cc Mon Sep 17 00:00:00 2001
From: elhassane <elhassanegargem@gmail.com>
Date: Thu, 3 Dec 2020 11:45:19 +0100
Subject: [PATCH 07/14] split serializer DocumentProcessSerializer
---
app/apps/api/serializers.py | 337 ++++++++++++++++++++++--------------
app/apps/api/tests.py | 53 ++++--
2 files changed, 240 insertions(+), 150 deletions(-)
diff --git a/app/apps/api/serializers.py b/app/apps/api/serializers.py
index 1c67c8a6..8bb3ee1e 100644
--- a/app/apps/api/serializers.py
+++ b/app/apps/api/serializers.py
@@ -268,47 +268,23 @@ class PartDetailSerializer(PartSerializer):
return nex and nex.pk or None
-class OcrModelSerializer(serializers.ModelSerializer):
- class Meta:
- model = OcrModel
- fields = ('pk', 'name','file','job','owner','training','training_epoch',
- 'training_accuracy','training_total','training_errors','document','script',)
+class OcrModelSerializer(serializers.Serializer):
+ def __init__(self, document, user, *args, **kwargs):
+ self.document = document
+ self.user = user
+ self.document_parts = []
+ if self.document.read_direction == self.document.READ_DIRECTION_RTL:
+ self.initial['text_direction'] = 'horizontal-rl'
+ super().__init__(*args, **kwargs)
-class DocumentProcessSerializer(serializers.Serializer):
- TASK_BINARIZE = 'binarize'
- TASK_SEGMENT = 'segment'
- TASK_TRAIN = 'train'
- TASK_SEGTRAIN = 'segtrain'
- TASK_TRANSCRIBE = 'transcribe'
+class SegmentSerializer(OcrModelSerializer):
parts = serializers.ListField(
- child=serializers.IntegerField()
- )
-
- task = serializers.ChoiceField(required=True,
- choices = (
- (TASK_BINARIZE, TASK_BINARIZE),
- (TASK_SEGMENT, TASK_BINARIZE),
- (TASK_TRAIN, TASK_BINARIZE),
- (TASK_TRANSCRIBE, TASK_BINARIZE),
- (TASK_SEGTRAIN, TASK_BINARIZE),
- )
+ child=serializers.IntegerField()
)
- # binarization
- bw_image = serializers.ImageField(required=False)
- BINARIZER_CHOICES = (('kraken', _("Kraken")),)
- binarizer = serializers.ChoiceField(required=False,
- choices=BINARIZER_CHOICES,
- initial='kraken')
- threshold = serializers.FloatField(
- required=False, initial=0.5,
- validators=[MinValueValidator(0.1), MaxValueValidator(1)],
- help_text=_('Increase it for low contrast documents, if the letters are not visible enough.'),
- )
- # segment
SEG_STEPS_CHOICES = (
('both', _('Lines and regions')),
('lines', _('Lines Baselines and Masks')),
@@ -317,94 +293,172 @@ class DocumentProcessSerializer(serializers.Serializer):
)
seg_steps = serializers.ChoiceField(choices=SEG_STEPS_CHOICES,
- initial='both', required=False)
+ initial='both', required=False)
seg_model = serializers.IntegerField(required=False)
override = serializers.BooleanField(required=False, initial=True,
- help_text=_(
- "If checked, deletes existing segmentation <b>and bound transcriptions</b> first!"))
+ help_text=_(
+ "If checked, deletes existing segmentation <b>and bound transcriptions</b> first!"))
TEXT_DIRECTION_CHOICES = (('horizontal-lr', _("Horizontal l2r")),
('horizontal-rl', _("Horizontal r2l")),
('vertical-lr', _("Vertical l2r")),
('vertical-rl', _("Vertical r2l")))
text_direction = serializers.ChoiceField(initial='horizontal-lr', required=False,
- choices=TEXT_DIRECTION_CHOICES)
- # transcribe
+ choices=TEXT_DIRECTION_CHOICES)
+
upload_model = serializers.FileField(required=False,
- validators=[FileExtensionValidator(
- allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
+ validators=[FileExtensionValidator(
+ allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
- ocr_model = serializers.IntegerField(required=False)
+ def validate_seg_model(self, value):
+ model = get_object_or_404(OcrModel, pk=value)
+ return model
- # train
- new_model = serializers.CharField(required=False, label=_('Model name'))
- train_model = OcrModelSerializer(required=False)
- transcription = serializers.PrimaryKeyRelatedField(many=False,read_only=True)
+ def validate(self, data):
- # segtrain
- segtrain_model = serializers.IntegerField(required=False)
+ parts = data['parts']
- def __init__(self, document, user, *args, **kwargs):
- self.document = document
- self.user = user
- self.document_parts = []
- if self.document.read_direction == self.document.READ_DIRECTION_RTL:
- self.initial['text_direction'] = 'horizontal-rl'
- super().__init__(*args, **kwargs)
+ self.document_parts = DocumentPart.objects.filter(
+ document=self.document, pk__in=parts)
- def validate_bw_image(self,img):
- if not img:
- return
- if len(self.document_parts) != 1:
- raise serializers.ValidationError({'bw_image':_("Uploaded image with more than one selected image.")})
- # Beware: don't close the file here !
- fh = Image.open(img)
- if fh.mode not in ['1', 'L']:
- raise serializers.ValidationError({'bw_image':_("Uploaded image should be black and white.")})
- isize = (self.document_parts[0].image.width, self.document_parts[0].image.height)
- if fh.size != isize:
- raise serializers.ValidationError({'bw_image':_("Uploaded image should be the same size as original image {size}.").format(size=isize)})
- return img
-
- def validate_train_model(self,train_model):
+ model_job = OcrModel.MODEL_JOB_SEGMENT
- if train_model and train_model.training:
- raise AlreadyProcessingException
- return train_model
+ if data.get('upload_model'):
+ model = OcrModel.objects.create(
+ document=self.document_parts[0].document,
+ owner=self.user,
+ name=data['upload_model'].name.rsplit('.', 1)[0],
+ job=model_job)
- def validate_seg_model(self, value):
- model = get_object_or_404(OcrModel, pk=value)
- return model
+ model.file = data['upload_model']
+ model.save()
- def validate_ocr_model(self, value):
- model = get_object_or_404(OcrModel, pk=value)
- return model
+ elif data.get('seg_model'):
+ model = data.get('seg_model')
+
+ else:
+ model = None
+ data['model'] = model
+ return data
+
+ def process(self):
+
+ model = self.validated_data.get('model')
+
+ for part in self.document_parts:
+ part.task('segment',
+ user_pk=self.user.pk,
+ steps=self.validated_data.get('seg_steps'),
+ text_direction=self.validated_data.get('text_direction'),
+ model_pk=model and model.pk or None,
+ override=self.validated_data.get('override'))
+
+
+
+
+class SegTrainSerializer(OcrModelSerializer):
+
+ parts = serializers.ListField(
+ child=serializers.IntegerField()
+ )
+
+ upload_model = serializers.FileField(required=False,
+ validators=[FileExtensionValidator(
+ allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
+
+ segtrain_model = serializers.IntegerField(required=False)
+
+ new_model = serializers.CharField(required=False, label=_('Model name'))
def validate_segtrain_model(self, value):
model = get_object_or_404(OcrModel, pk=value)
return model
+
def validate(self, data):
- task = data['task']
parts = data['parts']
-
self.document_parts = DocumentPart.objects.filter(
document=self.document, pk__in=parts)
+ model_job = OcrModel.MODEL_JOB_SEGMENT
+
+ if len(parts) < 2:
+ raise serializers.ValidationError("Segmentation training requires at least 2 images.")
+
+ if data.get('segtrain_model'):
+ model = data.get('segtrain_model')
+
+ elif data.get('new_model'):
+ # file will be created by the training process
+ model = OcrModel.objects.create(
+ document=self.document_parts[0].document,
+ owner=self.user,
+ name=data['new_model'],
+ job=model_job)
+
+ elif data.get('upload_model'):
+ model = OcrModel.objects.create(
+ document=self.document_parts[0].document,
+ owner=self.user,
+ name=data['upload_model'].name.rsplit('.', 1)[0],
+ job=model_job)
+ # Note: needs to save the file in a second step because the path needs the db PK
+ model.file = data['upload_model']
+ model.save()
- if task == self.TASK_SEGMENT:
- model_job = OcrModel.MODEL_JOB_SEGMENT
- elif task == self.TASK_SEGTRAIN:
- model_job = OcrModel.MODEL_JOB_SEGMENT
- if task == self.TASK_SEGTRAIN and len(parts) < 2:
- raise serializers.ValidationError("Segmentation training requires at least 2 images.")
else:
- model_job = OcrModel.MODEL_JOB_RECOGNIZE
+ raise serializers.ValidationError(
+ _("Either select a name for your new model or an existing one."))
+
+ data['model'] = model
+ return data
+
+
+ def process(self):
+
+ model = self.validated_data.get('model')
+ model.segtrain(self.document,
+ self.document_parts,
+ user=self.user)
+
+
+class TrainSerializer(OcrModelSerializer):
+
+ parts = serializers.ListField(
+ child=serializers.IntegerField()
+ )
+
+ upload_model = serializers.FileField(required=False,
+ validators=[FileExtensionValidator(
+ allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
+
+ # train
+ new_model = serializers.CharField(required=False, label=_('Model name'))
+ train_model = serializers.IntegerField(required=False)
+ transcription = serializers.IntegerField(required=False)
+
+ def validate_transcription(self, value):
+ model = get_object_or_404(Transcription, pk=value)
+ return model
+
+ def validate_train_model(self,value):
+
+ train_model = get_object_or_404(OcrModel, pk=value)
+
+ if train_model and train_model.training:
+ raise AlreadyProcessingException
+ return train_model
+
+ def validate(self, data):
- if task == self.TASK_TRAIN and data.get('train_model'):
+ parts = data['parts']
+ model_job = OcrModel.MODEL_JOB_RECOGNIZE
+
+ self.document_parts = DocumentPart.objects.filter(
+ document=self.document, pk__in=parts)
+
+ if data.get('train_model'):
model = data.get('train_model')
- elif task == self.TASK_SEGTRAIN and data.get('segtrain_model'):
- model = data.get('segtrain_model')
elif data.get('upload_model'):
model = OcrModel.objects.create(
document=self.document_parts[0].document,
@@ -422,56 +476,71 @@ class DocumentProcessSerializer(serializers.Serializer):
owner=self.user,
name=data['new_model'],
job=model_job)
- elif data.get('ocr_model'):
- model = data.get('ocr_model')
- elif data.get('seg_model'):
- model = data.get('seg_model')
+
else:
- if task in (self.TASK_TRAIN, self.TASK_SEGTRAIN):
- raise serializers.ValidationError(
+ raise serializers.ValidationError(
_("Either select a name for your new model or an existing one."))
- else:
- model = None
data['model'] = model
return data
def process(self):
+ model = self.validated_data.get('model')
+
+ model.train(self.document_parts,
+ self.validated_data['transcription'],
+ user=self.user)
+
+
+class TranscribeSerializer(OcrModelSerializer):
+
+ upload_model = serializers.FileField(required=False,
+ validators=[FileExtensionValidator(
+ allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
+
+ ocr_model = serializers.IntegerField(required=False)
+
+ def validate_ocr_model(self, value):
+ model = get_object_or_404(OcrModel, pk=value)
+ return model
+
+ def validate(self, data):
+
+ parts = data['parts']
+
+ self.document_parts = DocumentPart.objects.filter(
+ document=self.document, pk__in=parts)
+
+ model_job = OcrModel.MODEL_JOB_RECOGNIZE
+
+ if data.get('upload_model'):
+ model = OcrModel.objects.create(
+ document=self.document_parts[0].document,
+ owner=self.user,
+ name=data['upload_model'].name.rsplit('.', 1)[0],
+ job=model_job)
+ # Note: needs to save the file in a second step because the path needs the db PK
+ model.file = data['upload_model']
+ model.save()
+
+ elif data.get('ocr_model'):
+ model = data.get('ocr_model')
+
+ else:
+ raise serializers.ValidationError(
+ _("Either select a model or upload a new model."))
+
+ data['model'] = model
+ return data
- task = self.validated_data.get('task')
+
+ def process(self):
model = self.validated_data.get('model')
- if task == self.TASK_BINARIZE:
- if len(self.document_parts) == 1 and self.validated_data.get('bw_image'):
- self.document_parts[0].bw_image = self.validated_data['bw_image']
- self.document_parts[0].save()
- else:
- for part in self.document_parts:
- part.task('binarize',
- user_pk=self.user.pk,
- threshold=self.validated_data.get('threshold'))
-
- elif task == self.TASK_SEGMENT:
- for part in self.document_parts:
- part.task('segment',
- user_pk=self.user.pk,
- steps=self.validated_data.get('seg_steps'),
- text_direction=self.validated_data.get('text_direction'),
- model_pk=model and model.pk or None,
- override=self.validated_data.get('override'))
-
- elif task == self.TASK_TRANSCRIBE:
- for part in self.document_parts:
- part.task('transcribe',
- user_pk=self.user.pk,
- model_pk=model and model.pk or None)
-
- elif task == self.TASK_TRAIN:
- model.train(self.document_parts,
- self.validated_data['transcription'],
- user=self.user)
-
- elif task == self.TASK_SEGTRAIN:
- model.segtrain(self.document,
- self.document_parts,
- user=self.user)
\ No newline at end of file
+ for part in self.document_parts:
+ part.task('transcribe',
+ user_pk=self.user.pk,
+ model_pk=model and model.pk or None)
+
+
+
diff --git a/app/apps/api/tests.py b/app/apps/api/tests.py
index 8f5ce6a2..3ad0d32d 100644
--- a/app/apps/api/tests.py
+++ b/app/apps/api/tests.py
@@ -10,7 +10,6 @@ from django.urls import reverse
from core.models import Block, Line, Transcription, LineTranscription, OcrModel
from core.tests.factory import CoreFactoryTestCase
-from api.serializers import DocumentProcessSerializer
class UserViewSetTestCase(CoreFactoryTestCase):
def setUp(self):
@@ -39,7 +38,9 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
self.doc2 = self.factory.make_document(owner=self.doc.owner)
self.part = self.factory.make_part(document=self.doc)
self.part2 = self.factory.make_part(document=self.doc)
- self.model_uri = reverse('api:document-model',kwargs={'pk': self.doc.pk})
+ self.segtrain_uri = reverse('api:document-segtrain',kwargs={'pk': self.doc.pk})
+ self.segment_uri = reverse('api:document-segment', kwargs={'pk': self.doc.pk})
+ self.train_uri = reverse('api:document-train', kwargs={'pk': self.doc.pk})
self.line = Line.objects.create(
@@ -89,37 +90,43 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
def test_segtrain_less_two_parts(self):
self.client.force_login(self.doc.owner)
-
-
- resp = self.client.post(self.model_uri,data={
+ model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT, document=self.doc)
+ resp = self.client.post(self.segtrain_uri,data={
'parts': [self.part.pk],
- 'transcription':self.transcription.pk,
- 'task': DocumentProcessSerializer.TASK_SEGTRAIN,
+ 'segtrain_model': model.pk
})
+
self.assertEqual(resp.status_code, 400)
+ self.assertEqual(resp.json()['error'],{'non_field_errors': ['Segmentation training requires at least 2 images.']})
def test_segtrain_new_model(self):
self.client.force_login(self.doc.owner)
-
- resp = self.client.post(self.model_uri,data={
+ resp = self.client.post(self.segtrain_uri,data={
'parts': [self.part.pk, self.part2.pk],
- 'transcription':self.transcription.pk,
- 'task': DocumentProcessSerializer.TASK_SEGTRAIN,
'new_model':'new model'
})
self.assertEqual(resp.status_code, 200)
self.assertEqual(OcrModel.objects.count(),1)
self.assertEqual(OcrModel.objects.first().name,"new model")
+ def test_segtrain_existing_model(self):
+ self.client.force_login(self.doc.owner)
+ model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT, document=self.doc)
+
+ resp = self.client.post(self.segtrain_uri, data={
+ 'parts': [self.part.pk, self.part2.pk],
+ 'segtrain_model': model.pk
+ })
+ self.assertEqual(resp.status_code, 200)
+ self.assertEqual(OcrModel.objects.count(), 2)
+
def test_segment(self):
self.client.force_login(self.doc.owner)
model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT,document=self.doc)
-
- resp = self.client.post(self.model_uri, data={
+ resp = self.client.post(self.segment_uri, data={
'parts': [self.part.pk,self.part2.pk],
- 'task': DocumentProcessSerializer.TASK_SEGMENT,
'seg_steps':'both',
'seg_model': model.pk,
})
@@ -129,13 +136,27 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
self.client.force_login(self.doc.owner)
model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT,document=self.doc)
- resp = self.client.post(self.model_uri, data={
+ resp = self.client.post(self.segment_uri, data={
'parts': [self.part.pk, self.part2.pk],
- 'task': DocumentProcessSerializer.TASK_SEGMENT,
'seg_steps': 'both',
'upload_model': SimpleUploadedFile(model.name,model.file.read())
})
self.assertEqual(resp.status_code, 200)
+ # assert creation of new model
+ self.assertEqual(OcrModel.objects.filter(document=self.doc,job=OcrModel.MODEL_JOB_SEGMENT).count(), 2)
+
+ def test_train_new_model(self):
+ self.client.force_login(self.doc.owner)
+
+ resp = self.client.post(self.train_uri, data={
+ 'parts': [self.part.pk, self.part2.pk],
+ 'new_model': 'testing new model',
+ 'transcription':self.transcription.pk
+ })
+ self.assertEqual(resp.status_code, 200)
+ self.assertEqual(OcrModel.objects.filter(document=self.doc, job=OcrModel.MODEL_JOB_RECOGNIZE).count(), 1)
+
+
# not used
# def test_update
--
GitLab
From 337c4b83cf041da24f7a48534714c4468e53fe7a Mon Sep 17 00:00:00 2001
From: elhassane <elhassanegargem@gmail.com>
Date: Fri, 4 Dec 2020 15:27:49 +0100
Subject: [PATCH 08/14] refactor model endpoints
---
app/apps/api/serializers.py | 2 --
app/apps/api/views.py | 54 ++++++++++++++++++++++++++++---------
2 files changed, 41 insertions(+), 15 deletions(-)
diff --git a/app/apps/api/serializers.py b/app/apps/api/serializers.py
index 8bb3ee1e..5215a327 100644
--- a/app/apps/api/serializers.py
+++ b/app/apps/api/serializers.py
@@ -354,8 +354,6 @@ class SegmentSerializer(OcrModelSerializer):
override=self.validated_data.get('override'))
-
-
class SegTrainSerializer(OcrModelSerializer):
parts = serializers.ListField(
diff --git a/app/apps/api/views.py b/app/apps/api/views.py
index 14c6bab4..6d5e1a96 100644
--- a/app/apps/api/views.py
+++ b/app/apps/api/views.py
@@ -26,7 +26,9 @@ from api.serializers import (UserOnboardingSerializer,
LineOrderSerializer,
TranscriptionSerializer,
LineTranscriptionSerializer,
- DocumentProcessSerializer)
+ SegmentSerializer,
+ TrainSerializer,
+ SegTrainSerializer)
from core.models import (Document,
DocumentPart,
@@ -48,6 +50,21 @@ from versioning.models import NoChangeException
logger = logging.getLogger(__name__)
+class OcrModelSerializerMixen():
+
+
+ def serve_view(self,serializer):
+ if serializer.is_valid():
+ try:
+ serializer.process()
+ except AlreadyProcessingException:
+ return Response(status=status.HTTP_400_BAD_REQUEST,data={'status': 'error', 'error':'Already processing.'})
+
+ return Response(status=status.HTTP_200_OK,data={'status': 'ok'})
+ else:
+ return Response(status=status.HTTP_400_BAD_REQUEST,data={'status': 'error', 'error':serializer.errors})
+
+
class UserViewSet(ModelViewSet):
queryset = User.objects.all()
@@ -61,7 +78,7 @@ class UserViewSet(ModelViewSet):
return Response(status=status.HTTP_200_OK)
-class DocumentViewSet(ModelViewSet):
+class DocumentViewSet(ModelViewSet,OcrModelSerializerMixen):
queryset = Document.objects.all()
serializer_class = DocumentSerializer
paginate_by = 10
@@ -126,21 +143,32 @@ class DocumentViewSet(ModelViewSet):
return self.form_error(json.dumps(form.errors))
@action(detail=True, methods=['post'])
- def model(self, request, pk=None):
+ def segment(self, request, pk=None):
document = self.get_object()
- self.serializer_class = DocumentProcessSerializer
- serializer = DocumentProcessSerializer(document=document, user=request.user,data=request.data)
- if serializer.is_valid():
- try:
- serializer.process()
- except AlreadyProcessingException:
- return Response(status=status.HTTP_400_BAD_REQUEST,data={'status': 'error', 'error':'Already processing.'})
+ self.serializer_class = SegmentSerializer
+ serializer = SegmentSerializer(document=document, user=request.user,data=request.data)
+ return self.serve_view(serializer)
- return Response(status=status.HTTP_200_OK,data={'status': 'ok'})
- else:
- return Response(status=status.HTTP_400_BAD_REQUEST,data={'status': 'error', 'error':serializer.errors})
+ @action(detail=True, methods=['post'])
+ def train(self, request, pk=None):
+ document = self.get_object()
+ self.serializer_class = TrainSerializer
+ serializer = TrainSerializer(document=document, user=request.user,data=request.data)
+ return self.serve_view(serializer)
+ @action(detail=True, methods=['post'])
+ def segtrain(self, request, pk=None):
+ document = self.get_object()
+ self.serializer_class = SegTrainSerializer
+ serializer = SegTrainSerializer(document=document, user=request.user,data=request.data)
+ return self.serve_view(serializer)
+ @action(detail=True, methods=['post'])
+ def transcribe(self, request, pk=None):
+ document = self.get_object()
+ self.serializer_class = SegTrainSerializer
+ serializer = SegTrainSerializer(document=document, user=request.user,data=request.data)
+ return self.serve_view(serializer)
--
GitLab
From 8f443c33140db728d079aa395ea5fb77c6538583 Mon Sep 17 00:00:00 2001
From: elhassane <elhassanegargem@gmail.com>
Date: Mon, 7 Dec 2020 13:34:12 +0100
Subject: [PATCH 09/14] separated ducment process forms
---
app/apps/core/forms.py | 244 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 244 insertions(+)
diff --git a/app/apps/core/forms.py b/app/apps/core/forms.py
index 6d94585b..c591e7fd 100644
--- a/app/apps/core/forms.py
+++ b/app/apps/core/forms.py
@@ -109,6 +109,250 @@ MetadataFormSet = inlineformset_factory(Document, DocumentMetadata,
extra=1, can_delete=True)
+class DocumentProcessForm1(BootstrapFormMixin, forms.Form):
+ parts = forms.CharField()
+
+ @cached_property
+ def parts(self):
+ pks = json.loads(self.data.get('parts'))
+ parts = DocumentPart.objects.filter(
+ document=self.document, pk__in=pks)
+ return parts
+
+ def __init__(self, document, user, *args, **kwargs):
+ self.document = document
+ self.user = user
+ super().__init__(*args, **kwargs)
+ # self.fields['typology'].widget = forms.HiddenInput() # for now
+ # self.fields['typology'].initial = Typology.objects.get(name="Page")
+ # self.fields['typology'].widget.attrs['title'] = _("Default Typology")
+ if self.document.read_direction == self.document.READ_DIRECTION_RTL:
+ self.initial['text_direction'] = 'horizontal-rl'
+ self.fields['binarizer'].widget.attrs['disabled'] = True
+ self.fields['train_model'].queryset &= OcrModel.objects.filter(document=self.document)
+ self.fields['segtrain_model'].queryset &= OcrModel.objects.filter(document=self.document)
+ self.fields['seg_model'].queryset &= OcrModel.objects.filter(document=self.document)
+ self.fields['ocr_model'].queryset &= OcrModel.objects.filter(
+ Q(document=None, script=document.main_script)
+ | Q(document=self.document))
+ self.fields['transcription'].queryset = Transcription.objects.filter(document=self.document)
+
+ def process(self):
+ model = self.cleaned_data.get('model')
+
+class DocumentSegmentForm(DocumentProcessForm1):
+ SEG_STEPS_CHOICES = (
+ ('both', _('Lines and regions')),
+ ('lines', _('Lines Baselines and Masks')),
+ ('masks', _('Only lines Masks')),
+ ('regions', _('Regions')),
+ )
+ segmentation_steps = forms.ChoiceField(choices=SEG_STEPS_CHOICES,
+ initial='both', required=False)
+ seg_model = forms.ModelChoiceField(queryset=OcrModel.objects.filter(job=OcrModel.MODEL_JOB_SEGMENT),
+ label=_("Model"), empty_label="default ({name})".format(
+ name=settings.KRAKEN_DEFAULT_SEGMENTATION_MODEL.rsplit('/')[-1]),
+ required=False)
+ override = forms.BooleanField(required=False, initial=True,
+ help_text=_(
+ "If checked, deletes existing segmentation <b>and bound transcriptions</b> first!"))
+ TEXT_DIRECTION_CHOICES = (('horizontal-lr', _("Horizontal l2r")),
+ ('horizontal-rl', _("Horizontal r2l")),
+ ('vertical-lr', _("Vertical l2r")),
+ ('vertical-rl', _("Vertical r2l")))
+ text_direction = forms.ChoiceField(initial='horizontal-lr', required=False,
+ choices=TEXT_DIRECTION_CHOICES)
+ upload_model = forms.FileField(required=False,
+ validators=[FileExtensionValidator(
+ allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
+
+ def clean(self):
+ data = super().clean()
+ model_job = OcrModel.MODEL_JOB_SEGMENT
+
+ if data.get('upload_model'):
+ model = OcrModel.objects.create(
+ document=self.parts[0].document,
+ owner=self.user,
+ name=data['upload_model'].name.rsplit('.', 1)[0],
+ job=model_job)
+ # Note: needs to save the file in a second step because the path needs the db PK
+ model.file = data['upload_model']
+ model.save()
+
+ elif data.get('seg_model'):
+ model = data.get('seg_model')
+ else:
+ model = None
+
+ data['model'] = model
+ return data
+
+ def process(self):
+ super().process()
+
+ for part in self.parts:
+ part.task('segment',
+ user_pk=self.user.pk,
+ steps=self.cleaned_data.get('segmentation_steps'),
+ text_direction=self.cleaned_data.get('text_direction'),
+ model_pk=model and model.pk or None,
+ override=self.cleaned_data.get('override'))
+
+
+class DocumentTrainForm(DocumentProcessForm1):
+ new_model = forms.CharField(required=False, label=_('Model name'))
+ train_model = forms.ModelChoiceField(queryset=OcrModel.objects
+ .filter(job=OcrModel.MODEL_JOB_RECOGNIZE),
+ label=_("Model"), required=False)
+ upload_model = forms.FileField(required=False,
+ validators=[FileExtensionValidator(
+ allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
+
+ transcription = forms.ModelChoiceField(queryset=Transcription.objects.all(), required=False)
+
+ def clean_train_model(self):
+ model = self.cleaned_data['train_model']
+ if model and model.training:
+ raise AlreadyProcessingException
+ return model
+
+ def clean(self):
+ data = super().clean()
+
+ model_job = OcrModel.MODEL_JOB_RECOGNIZE
+
+ if data.get('train_model'):
+ model = data.get('train_model')
+
+ elif data.get('upload_model'):
+ model = OcrModel.objects.create(
+ document=self.parts[0].document,
+ owner=self.user,
+ name=data['upload_model'].name.rsplit('.', 1)[0],
+ job=model_job)
+ # Note: needs to save the file in a second step because the path needs the db PK
+ model.file = data['upload_model']
+ model.save()
+
+ elif data.get('new_model'):
+ # file will be created by the training process
+ model = OcrModel.objects.create(
+ document=self.parts[0].document,
+ owner=self.user,
+ name=data['new_model'],
+ job=model_job)
+
+ else:
+ raise forms.ValidationError(
+ _("Either select a name for your new model or an existing one."))
+
+ data['model'] = model
+ return data
+
+
+ def process(self):
+ super().process()
+
+ model.train(self.parts,
+ self.cleaned_data['transcription'],
+ user=self.user)
+
+
+class DocumentSegtrainForm(DocumentProcessForm1):
+ segtrain_model = forms.ModelChoiceField(queryset=OcrModel.objects
+ .filter(job=OcrModel.MODEL_JOB_SEGMENT),
+ label=_("Model"), required=False)
+ upload_model = forms.FileField(required=False,
+ validators=[FileExtensionValidator(
+ allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
+
+ new_model = forms.CharField(required=False, label=_('Model name'))
+
+ def clean(self):
+ data = super().clean()
+
+
+ model_job = OcrModel.MODEL_JOB_SEGMENT
+ if len(self.parts) < 2:
+ raise forms.ValidationError("Segmentation training requires at least 2 images.")
+
+ if data.get('segtrain_model'):
+ model = data.get('segtrain_model')
+ elif data.get('upload_model'):
+ model = OcrModel.objects.create(
+ document=self.parts[0].document,
+ owner=self.user,
+ name=data['upload_model'].name.rsplit('.', 1)[0],
+ job=model_job)
+ # Note: needs to save the file in a second step because the path needs the db PK
+ model.file = data['upload_model']
+ model.save()
+
+ elif data.get('new_model'):
+ # file will be created by the training process
+ model = OcrModel.objects.create(
+ document=self.parts[0].document,
+ owner=self.user,
+ name=data['new_model'],
+ job=model_job)
+
+ else:
+
+ raise forms.ValidationError(
+ _("Either select a name for your new model or an existing one."))
+
+ data['model'] = model
+ return data
+
+ def process(self):
+ super().process()
+ model.segtrain(self.document,
+ self.parts,
+ user=self.user)
+
+
+class DocumentTranscribeForm(DocumentProcessForm1):
+
+ upload_model = forms.FileField(required=False,
+ validators=[FileExtensionValidator(
+ allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
+ ocr_model = forms.ModelChoiceField(queryset=OcrModel.objects
+ .filter(job=OcrModel.MODEL_JOB_RECOGNIZE),
+ label=_("Model"), required=False)
+
+ def clean(self):
+ data = super().clean()
+
+ model_job = OcrModel.MODEL_JOB_RECOGNIZE
+
+ if data.get('upload_model'):
+ model = OcrModel.objects.create(
+ document=self.parts[0].document,
+ owner=self.user,
+ name=data['upload_model'].name.rsplit('.', 1)[0],
+ job=model_job)
+ # Note: needs to save the file in a second step because the path needs the db PK
+ model.file = data['upload_model']
+ model.save()
+
+ elif data.get('ocr_model'):
+ model = data.get('ocr_model')
+ else:
+ raise forms.ValidationError(
+ _("Either select a name for your new model or an existing one."))
+
+ data['model'] = model
+ return data
+
+ def process(self):
+ super().process()
+ for part in self.parts:
+ part.task('transcribe',
+ user_pk=self.user.pk,
+ model_pk=model and model.pk or None)
+
+
class DocumentProcessForm(BootstrapFormMixin, forms.Form):
# TODO: split this form into one for each process?!
TASK_BINARIZE = 'binarize'
--
GitLab
From 948b219faf4c1338beac7c4c88890ba1fe4f80f1 Mon Sep 17 00:00:00 2001
From: Robin Tissot <tissotrobin@gmail.com>
Date: Thu, 7 Jan 2021 15:17:54 +0100
Subject: [PATCH 10/14] WIP rewriting process endpoints.
---
app/apps/api/serializers.py | 306 ++++++++++--------------------------
app/apps/api/tests.py | 41 +++--
app/apps/api/views.py | 61 ++++---
3 files changed, 124 insertions(+), 284 deletions(-)
diff --git a/app/apps/api/serializers.py b/app/apps/api/serializers.py
index 5215a327..35173188 100644
--- a/app/apps/api/serializers.py
+++ b/app/apps/api/serializers.py
@@ -24,6 +24,7 @@ from core.models import (Document,
LineType,
OcrModel,
AlreadyProcessingException)
+from core.tasks import (segtrain, train, segment, transcribe)
logger = logging.getLogger(__name__)
@@ -268,277 +269,128 @@ class PartDetailSerializer(PartSerializer):
return nex and nex.pk or None
-class OcrModelSerializer(serializers.Serializer):
-
+class ProcessSerializerMixin():
def __init__(self, document, user, *args, **kwargs):
self.document = document
self.user = user
- self.document_parts = []
- if self.document.read_direction == self.document.READ_DIRECTION_RTL:
- self.initial['text_direction'] = 'horizontal-rl'
super().__init__(*args, **kwargs)
-class SegmentSerializer(OcrModelSerializer):
+class OcrModelSerializer(ProcessSerializermixin, serializer.ModelSerializer):
+ class OcrModel:
+ model = Line
+ fields = ('pk', 'name')
+ list_serializer_class = LineOrderListSerializer
- parts = serializers.ListField(
- child=serializers.IntegerField()
- )
- SEG_STEPS_CHOICES = (
+class SegmentSerializer(ProcessSerializerMixin):
+ STEPS_CHOICES = (
('both', _('Lines and regions')),
('lines', _('Lines Baselines and Masks')),
('masks', _('Only lines Masks')),
('regions', _('Regions')),
)
+ TEXT_DIRECTION_CHOICES = (
+ ('horizontal-lr', _("Horizontal l2r")),
+ ('horizontal-rl', _("Horizontal r2l")),
+ ('vertical-lr', _("Vertical l2r")),
+ ('vertical-rl', _("Vertical r2l"))
+ )
- seg_steps = serializers.ChoiceField(choices=SEG_STEPS_CHOICES,
- initial='both', required=False)
- seg_model = serializers.IntegerField(required=False)
-
- override = serializers.BooleanField(required=False, initial=True,
- help_text=_(
- "If checked, deletes existing segmentation <b>and bound transcriptions</b> first!"))
- TEXT_DIRECTION_CHOICES = (('horizontal-lr', _("Horizontal l2r")),
- ('horizontal-rl', _("Horizontal r2l")),
- ('vertical-lr', _("Vertical l2r")),
- ('vertical-rl', _("Vertical r2l")))
- text_direction = serializers.ChoiceField(initial='horizontal-lr', required=False,
+ parts = serializers.PrimaryKeyRelatedField(many=True,
+ queryset=DocumentPart.objects.all())
+ steps = serializers.ChoiceField(choices=STEPS_CHOICES,
+ required=False,
+ default='both')
+ model = serializers.PrimaryKeyRelatedField(required=False,
+ allow_null=True,
+ queryset=OcrModel.objects.filter(
+ job=OcrModel.MODEL_jOB_SEGMENT))
+ override = serializers.BooleanField(required=False, default=True)
+ text_direction = serializers.ChoiceField(default='horizontal-lr',
+ required=False,
choices=TEXT_DIRECTION_CHOICES)
- upload_model = serializers.FileField(required=False,
- validators=[FileExtensionValidator(
- allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
-
- def validate_seg_model(self, value):
- model = get_object_or_404(OcrModel, pk=value)
- return model
-
- def validate(self, data):
-
- parts = data['parts']
-
- self.document_parts = DocumentPart.objects.filter(
- document=self.document, pk__in=parts)
-
- model_job = OcrModel.MODEL_JOB_SEGMENT
-
- if data.get('upload_model'):
- model = OcrModel.objects.create(
- document=self.document_parts[0].document,
- owner=self.user,
- name=data['upload_model'].name.rsplit('.', 1)[0],
- job=model_job)
-
- model.file = data['upload_model']
- model.save()
-
- elif data.get('seg_model'):
- model = data.get('seg_model')
-
- else:
- model = None
- data['model'] = model
- return data
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.fields['model'].queryset = OcrModel.objects.filter(document=self.document)
def process(self):
-
model = self.validated_data.get('model')
-
- for part in self.document_parts:
- part.task('segment',
- user_pk=self.user.pk,
- steps=self.validated_data.get('seg_steps'),
- text_direction=self.validated_data.get('text_direction'),
- model_pk=model and model.pk or None,
- override=self.validated_data.get('override'))
-
-
-class SegTrainSerializer(OcrModelSerializer):
-
- parts = serializers.ListField(
- child=serializers.IntegerField()
- )
-
- upload_model = serializers.FileField(required=False,
- validators=[FileExtensionValidator(
- allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
-
- segtrain_model = serializers.IntegerField(required=False)
-
- new_model = serializers.CharField(required=False, label=_('Model name'))
-
- def validate_segtrain_model(self, value):
- model = get_object_or_404(OcrModel, pk=value)
- return model
-
-
- def validate(self, data):
-
- parts = data['parts']
- self.document_parts = DocumentPart.objects.filter(
- document=self.document, pk__in=parts)
- model_job = OcrModel.MODEL_JOB_SEGMENT
-
- if len(parts) < 2:
+ parts = self.validated_data.get('parts')
+ for part in parts:
+ part.chain_tasks(
+ segment.si(part.pk,
+ user_pk=self.user.pk,
+ model_pk=model,
+ steps=self.validated_data.get('steps'),
+ text_direction=self.validated_data.get('text_direction'),
+ override=self.validated_data.get('override'))
+ )
+
+
+class SegTrainSerializer(ProcessSerializerMixin):
+ parts = serializers.PrimaryKeyRelatedField(many=True,
+ queryset=DocumentPart.objects.all())
+ model = serializers.PrimaryKeyRelatedField(required=False,
+ queryset=OcrModel.objects.filter(
+ OcrModel.MODEL_JOB_SEGMENT))
+ model_name = serializers.CharField(required=False)
+
+ def validate_parts(self, data):
+ if len(data) < 2:
raise serializers.ValidationError("Segmentation training requires at least 2 images.")
+ return data
- if data.get('segtrain_model'):
- model = data.get('segtrain_model')
-
- elif data.get('new_model'):
- # file will be created by the training process
- model = OcrModel.objects.create(
- document=self.document_parts[0].document,
- owner=self.user,
- name=data['new_model'],
- job=model_job)
-
- elif data.get('upload_model'):
- model = OcrModel.objects.create(
- document=self.document_parts[0].document,
- owner=self.user,
- name=data['upload_model'].name.rsplit('.', 1)[0],
- job=model_job)
- # Note: needs to save the file in a second step because the path needs the db PK
- model.file = data['upload_model']
- model.save()
-
- else:
+ def validate(self, data):
+ data = super().validate(data)
+ if not data.get('model') and not data.get('model_name'):
raise serializers.ValidationError(
- _("Either select a name for your new model or an existing one."))
-
- data['model'] = model
+ _("Either use model_name to create a new model, or add a model pk to use an existing one."))
return data
-
def process(self):
-
model = self.validated_data.get('model')
model.segtrain(self.document,
self.document_parts,
user=self.user)
-class TrainSerializer(OcrModelSerializer):
-
- parts = serializers.ListField(
- child=serializers.IntegerField()
- )
-
- upload_model = serializers.FileField(required=False,
- validators=[FileExtensionValidator(
- allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
-
- # train
- new_model = serializers.CharField(required=False, label=_('Model name'))
- train_model = serializers.IntegerField(required=False)
- transcription = serializers.IntegerField(required=False)
-
- def validate_transcription(self, value):
- model = get_object_or_404(Transcription, pk=value)
- return model
-
- def validate_train_model(self,value):
-
- train_model = get_object_or_404(OcrModel, pk=value)
-
- if train_model and train_model.training:
- raise AlreadyProcessingException
- return train_model
+class TrainSerializer(ProcessSerializerMixin):
+ parts = serializers.PrimaryKeyRelatedField(many=True,
+ queryset=DocumentPart.objects.all())
+ model = serializers.PrimaryKeyRelatedField(required=False,
+ queryset=OcrModel.objects.filter(
+ training=False,
+ job=OcrModel.MODEL_JOB_RECOGNIZE))
+ model_name = serializers.CharField(required=False, label=_('Model name'))
+ transcription = serializers.PrimaryKeyRelatedField(queryset=Transcription.objects.all())
def validate(self, data):
-
- parts = data['parts']
- model_job = OcrModel.MODEL_JOB_RECOGNIZE
-
- self.document_parts = DocumentPart.objects.filter(
- document=self.document, pk__in=parts)
-
- if data.get('train_model'):
- model = data.get('train_model')
- elif data.get('upload_model'):
- model = OcrModel.objects.create(
- document=self.document_parts[0].document,
- owner=self.user,
- name=data['upload_model'].name.rsplit('.', 1)[0],
- job=model_job)
- # Note: needs to save the file in a second step because the path needs the db PK
- model.file = data['upload_model']
- model.save()
-
- elif data.get('new_model'):
- # file will be created by the training process
- model = OcrModel.objects.create(
- document=self.document_parts[0].document,
- owner=self.user,
- name=data['new_model'],
- job=model_job)
-
- else:
+ data = super().validate(data)
+ if not data.get('model') and not data.get('new_model'):
raise serializers.ValidationError(
_("Either select a name for your new model or an existing one."))
-
- data['model'] = model
return data
def process(self):
model = self.validated_data.get('model')
-
model.train(self.document_parts,
self.validated_data['transcription'],
user=self.user)
-class TranscribeSerializer(OcrModelSerializer):
-
- upload_model = serializers.FileField(required=False,
- validators=[FileExtensionValidator(
- allowed_extensions=['mlmodel', 'pronn', 'clstm'])])
-
- ocr_model = serializers.IntegerField(required=False)
-
- def validate_ocr_model(self, value):
- model = get_object_or_404(OcrModel, pk=value)
- return model
-
- def validate(self, data):
-
- parts = data['parts']
-
- self.document_parts = DocumentPart.objects.filter(
- document=self.document, pk__in=parts)
-
- model_job = OcrModel.MODEL_JOB_RECOGNIZE
-
- if data.get('upload_model'):
- model = OcrModel.objects.create(
- document=self.document_parts[0].document,
- owner=self.user,
- name=data['upload_model'].name.rsplit('.', 1)[0],
- job=model_job)
- # Note: needs to save the file in a second step because the path needs the db PK
- model.file = data['upload_model']
- model.save()
-
- elif data.get('ocr_model'):
- model = data.get('ocr_model')
-
- else:
- raise serializers.ValidationError(
- _("Either select a model or upload a new model."))
-
- data['model'] = model
- return data
-
+class TranscribeSerializer(ProcessSerializerMixin):
+ model = serializers.PrimaryKeyRelatedField(required=False,
+ queryset=OcrModel.objects.filter(
+ training=False,
+ job=OcrModel.MODEL_JOB_RECOGNIZE))
def process(self):
-
model = self.validated_data.get('model')
- for part in self.document_parts:
- part.task('transcribe',
- user_pk=self.user.pk,
- model_pk=model and model.pk or None)
-
-
-
+ for part in self.validated_data.parts:
+ part.chain_tasks(
+ transcribe.si(part.pk,
+ user_pk=self.user.pk,
+ model_pk=model)
+ )
diff --git a/app/apps/api/tests.py b/app/apps/api/tests.py
index 3ad0d32d..a4d29526 100644
--- a/app/apps/api/tests.py
+++ b/app/apps/api/tests.py
@@ -10,6 +10,8 @@ from django.urls import reverse
from core.models import Block, Line, Transcription, LineTranscription, OcrModel
from core.tests.factory import CoreFactoryTestCase
+
+
class UserViewSetTestCase(CoreFactoryTestCase):
def setUp(self):
@@ -20,7 +22,7 @@ class UserViewSetTestCase(CoreFactoryTestCase):
self.client.force_login(user)
uri = reverse('api:user-onboarding')
resp = self.client.put(uri, {
- 'onboarding' : 'False',
+ 'onboarding': 'False',
}, content_type='application/json')
user.refresh_from_db()
@@ -28,9 +30,6 @@ class UserViewSetTestCase(CoreFactoryTestCase):
self.assertEqual(user.onboarding, False)
-
-
-
class DocumentViewSetTestCase(CoreFactoryTestCase):
def setUp(self):
super().setUp()
@@ -38,11 +37,10 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
self.doc2 = self.factory.make_document(owner=self.doc.owner)
self.part = self.factory.make_part(document=self.doc)
self.part2 = self.factory.make_part(document=self.doc)
- self.segtrain_uri = reverse('api:document-segtrain',kwargs={'pk': self.doc.pk})
+ self.segtrain_uri = reverse('api:document-segtrain', kwargs={'pk': self.doc.pk})
self.segment_uri = reverse('api:document-segment', kwargs={'pk': self.doc.pk})
self.train_uri = reverse('api:document-train', kwargs={'pk': self.doc.pk})
-
self.line = Line.objects.create(
mask=[10, 10, 50, 50],
document_part=self.part)
@@ -91,13 +89,13 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
def test_segtrain_less_two_parts(self):
self.client.force_login(self.doc.owner)
model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT, document=self.doc)
- resp = self.client.post(self.segtrain_uri,data={
+ resp = self.client.post(self.segtrain_uri, data={
'parts': [self.part.pk],
'segtrain_model': model.pk
})
self.assertEqual(resp.status_code, 400)
- self.assertEqual(resp.json()['error'],{'non_field_errors': ['Segmentation training requires at least 2 images.']})
+ self.assertEqual(resp.json()['error'], {'non_field_errors': ['Segmentation training requires at least 2 images.']})
def test_segtrain_new_model(self):
self.client.force_login(self.doc.owner)
@@ -122,28 +120,29 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
self.assertEqual(OcrModel.objects.count(), 2)
def test_segment(self):
-
self.client.force_login(self.doc.owner)
- model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT,document=self.doc)
+ model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT, document=self.doc)
resp = self.client.post(self.segment_uri, data={
- 'parts': [self.part.pk,self.part2.pk],
- 'seg_steps':'both',
+ 'parts': [self.part.pk, self.part2.pk],
+ 'seg_steps': 'both',
'seg_model': model.pk,
})
self.assertEqual(resp.status_code, 200)
def test_segment_file_upload(self):
self.client.force_login(self.doc.owner)
- model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT,document=self.doc)
+ model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT, document=self.doc)
resp = self.client.post(self.segment_uri, data={
'parts': [self.part.pk, self.part2.pk],
'seg_steps': 'both',
- 'upload_model': SimpleUploadedFile(model.name,model.file.read())
+ 'upload_model': SimpleUploadedFile(model.name, model.file.read())
})
self.assertEqual(resp.status_code, 200)
# assert creation of new model
- self.assertEqual(OcrModel.objects.filter(document=self.doc,job=OcrModel.MODEL_JOB_SEGMENT).count(), 2)
+ self.assertEqual(OcrModel.objects.filter(
+ document=self.doc,
+ job=OcrModel.MODEL_JOB_SEGMENT).count(), 2)
def test_train_new_model(self):
self.client.force_login(self.doc.owner)
@@ -151,16 +150,12 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
resp = self.client.post(self.train_uri, data={
'parts': [self.part.pk, self.part2.pk],
'new_model': 'testing new model',
- 'transcription':self.transcription.pk
+ 'transcription': self.transcription.pk
})
self.assertEqual(resp.status_code, 200)
- self.assertEqual(OcrModel.objects.filter(document=self.doc, job=OcrModel.MODEL_JOB_RECOGNIZE).count(), 1)
-
-
-
- # not used
- # def test_update
- # def test_create
+ self.assertEqual(OcrModel.objects.filter(
+ document=self.doc,
+ job=OcrModel.MODEL_JOB_RECOGNIZE).count(), 1)
class PartViewSetTestCase(CoreFactoryTestCase):
diff --git a/app/apps/api/views.py b/app/apps/api/views.py
index 3f63a428..7baef0d3 100644
--- a/app/apps/api/views.py
+++ b/app/apps/api/views.py
@@ -28,7 +28,8 @@ from api.serializers import (UserOnboardingSerializer,
LineTranscriptionSerializer,
SegmentSerializer,
TrainSerializer,
- SegTrainSerializer)
+ SegTrainSerializer,
+ TranscribeSerializer)
from core.models import (Document,
DocumentPart,
@@ -41,7 +42,6 @@ from core.models import (Document,
AlreadyProcessingException)
from core.tasks import recalculate_masks
-from core.forms import DocumentProcessForm
from users.models import User
from imports.forms import ImportForm, ExportForm
from imports.parsers import ParseError
@@ -50,21 +50,6 @@ from versioning.models import NoChangeException
logger = logging.getLogger(__name__)
-class OcrModelSerializerMixen():
-
-
- def serve_view(self,serializer):
- if serializer.is_valid():
- try:
- serializer.process()
- except AlreadyProcessingException:
- return Response(status=status.HTTP_400_BAD_REQUEST,data={'status': 'error', 'error':'Already processing.'})
-
- return Response(status=status.HTTP_200_OK,data={'status': 'ok'})
- else:
- return Response(status=status.HTTP_400_BAD_REQUEST,data={'status': 'error', 'error':serializer.errors})
-
-
class UserViewSet(ModelViewSet):
queryset = User.objects.all()
@@ -78,7 +63,7 @@ class UserViewSet(ModelViewSet):
return Response(status=status.HTTP_200_OK)
-class DocumentViewSet(ModelViewSet,OcrModelSerializerMixen):
+class DocumentViewSet(ModelViewSet):
queryset = Document.objects.all()
serializer_class = DocumentSerializer
paginate_by = 10
@@ -142,33 +127,41 @@ class DocumentViewSet(ModelViewSet,OcrModelSerializerMixen):
else:
return self.form_error(json.dumps(form.errors))
+ def get_process_response(self, request, serializer_class):
+ document = self.get_object()
+ serializer = serializer_class(document=document,
+ user=request.user,
+ data=request.data)
+ if serializer.is_valid():
+ try:
+ serializer.process()
+ except AlreadyProcessingException:
+ return Response(status=status.HTTP_400_BAD_REQUEST,
+ data={'status': 'error',
+ 'error': 'Already processing.'})
+
+ return Response(status=status.HTTP_200_OK,
+ data={'status': 'ok'})
+ else:
+ return Response(status=status.HTTP_400_BAD_REQUEST,
+ data={'status': 'error',
+ 'error': serializer.errors})
+
@action(detail=True, methods=['post'])
def segment(self, request, pk=None):
- document = self.get_object()
- self.serializer_class = SegmentSerializer
- serializer = SegmentSerializer(document=document, user=request.user,data=request.data)
- return self.serve_view(serializer)
+ return self.get_process_response(request, SegmentSerializer)
@action(detail=True, methods=['post'])
def train(self, request, pk=None):
- document = self.get_object()
- self.serializer_class = TrainSerializer
- serializer = TrainSerializer(document=document, user=request.user,data=request.data)
- return self.serve_view(serializer)
+ return self.get_process_response(request, TrainSerializer)
@action(detail=True, methods=['post'])
def segtrain(self, request, pk=None):
- document = self.get_object()
- self.serializer_class = SegTrainSerializer
- serializer = SegTrainSerializer(document=document, user=request.user,data=request.data)
- return self.serve_view(serializer)
+ return self.get_process_response(request, SegTrainSerializer)
@action(detail=True, methods=['post'])
def transcribe(self, request, pk=None):
- document = self.get_object()
- self.serializer_class = SegTrainSerializer
- serializer = SegTrainSerializer(document=document, user=request.user,data=request.data)
- return self.serve_view(serializer)
+ return self.get_process_response(request, TranscribeSerializer)
--
GitLab
From f573970ca829216aee42d8955f4c42023e0bde76 Mon Sep 17 00:00:00 2001
From: Robin Tissot <tissotrobin@gmail.com>
Date: Wed, 13 Jan 2021 12:41:12 +0100
Subject: [PATCH 11/14] Adds a model endpoint, keep rewriting.
---
app/apps/api/fields.py | 18 ++++++++
app/apps/api/serializers.py | 89 ++++++++++++++++++++++---------------
app/apps/api/tests.py | 53 +++++++++++-----------
app/apps/api/urls.py | 4 +-
app/apps/api/views.py | 26 +++++++++--
app/apps/core/models.py | 85 +++++++++++++++++------------------
app/apps/core/tasks.py | 17 +++----
7 files changed, 171 insertions(+), 121 deletions(-)
create mode 100644 app/apps/api/fields.py
diff --git a/app/apps/api/fields.py b/app/apps/api/fields.py
new file mode 100644
index 00000000..a8d60044
--- /dev/null
+++ b/app/apps/api/fields.py
@@ -0,0 +1,18 @@
+from rest_framework import serializers
+
+
+class DisplayChoiceField(serializers.ChoiceField):
+ def to_representation(self, obj):
+ if obj == '' and self.allow_blank:
+ return obj
+ return self._choices[obj]
+
+ def to_internal_value(self, data):
+ # To support inserts with the value
+ if data == '' and self.allow_blank:
+ return ''
+
+ for key, val in self._choices.items():
+ if val == data:
+ return key
+ self.fail('invalid_choice', input=data)
diff --git a/app/apps/api/serializers.py b/app/apps/api/serializers.py
index 35173188..6bc501fd 100644
--- a/app/apps/api/serializers.py
+++ b/app/apps/api/serializers.py
@@ -1,18 +1,15 @@
import bleach
import logging
import html
-import json
from django.conf import settings
from django.db.utils import IntegrityError
from django.utils.translation import gettext_lazy as _
-from django.core.validators import FileExtensionValidator, MinValueValidator, MaxValueValidator
-from django.utils.functional import cached_property
-from django.shortcuts import get_object_or_404
from rest_framework import serializers
from easy_thumbnails.files import get_thumbnailer
+from api.fields import DisplayChoiceField
from users.models import User
from core.models import (Document,
DocumentPart,
@@ -22,8 +19,7 @@ from core.models import (Document,
LineTranscription,
BlockType,
LineType,
- OcrModel,
- AlreadyProcessingException)
+ OcrModel)
from core.tasks import (segtrain, train, segment, transcribe)
logger = logging.getLogger(__name__)
@@ -269,6 +265,17 @@ class PartDetailSerializer(PartSerializer):
return nex and nex.pk or None
+class OcrModelSerializer(serializers.ModelSerializer):
+ owner = serializers.ReadOnlyField(source='owner.username')
+ job = DisplayChoiceField(choices=OcrModel.MODEL_JOB_CHOICES)
+ training = serializers.ReadOnlyField()
+
+ class Meta:
+ model = OcrModel
+ fields = ('pk', 'name', 'file', 'job',
+ 'owner', 'training', 'versions')
+
+
class ProcessSerializerMixin():
def __init__(self, document, user, *args, **kwargs):
self.document = document
@@ -276,14 +283,7 @@ class ProcessSerializerMixin():
super().__init__(*args, **kwargs)
-class OcrModelSerializer(ProcessSerializermixin, serializer.ModelSerializer):
- class OcrModel:
- model = Line
- fields = ('pk', 'name')
- list_serializer_class = LineOrderListSerializer
-
-
-class SegmentSerializer(ProcessSerializerMixin):
+class SegmentSerializer(ProcessSerializerMixin, serializers.Serializer):
STEPS_CHOICES = (
('both', _('Lines and regions')),
('lines', _('Lines Baselines and Masks')),
@@ -304,8 +304,7 @@ class SegmentSerializer(ProcessSerializerMixin):
default='both')
model = serializers.PrimaryKeyRelatedField(required=False,
allow_null=True,
- queryset=OcrModel.objects.filter(
- job=OcrModel.MODEL_jOB_SEGMENT))
+ queryset=OcrModel.objects.all())
override = serializers.BooleanField(required=False, default=True)
text_direction = serializers.ChoiceField(default='horizontal-lr',
required=False,
@@ -313,7 +312,9 @@ class SegmentSerializer(ProcessSerializerMixin):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self.fields['model'].queryset = OcrModel.objects.filter(document=self.document)
+ self.fields['model'].queryset = OcrModel.objects.filter(job=OcrModel.MODEL_JOB_SEGMENT,
+ document=self.document)
+ self.fields['parts'].queryset = DocumentPart.objects.filter(document=self.document)
def process(self):
model = self.validated_data.get('model')
@@ -329,14 +330,19 @@ class SegmentSerializer(ProcessSerializerMixin):
)
-class SegTrainSerializer(ProcessSerializerMixin):
+class SegTrainSerializer(ProcessSerializerMixin, serializers.Serializer):
parts = serializers.PrimaryKeyRelatedField(many=True,
queryset=DocumentPart.objects.all())
model = serializers.PrimaryKeyRelatedField(required=False,
- queryset=OcrModel.objects.filter(
- OcrModel.MODEL_JOB_SEGMENT))
+ queryset=OcrModel.objects.all())
model_name = serializers.CharField(required=False)
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.fields['model'].queryset = OcrModel.objects.filter(job=OcrModel.MODEL_JOB_SEGMENT,
+ document=self.document)
+ self.fields['parts'].queryset = DocumentPart.objects.filter(document=self.document)
+
def validate_parts(self, data):
if len(data) < 2:
raise serializers.ValidationError("Segmentation training requires at least 2 images.")
@@ -346,7 +352,7 @@ class SegTrainSerializer(ProcessSerializerMixin):
data = super().validate(data)
if not data.get('model') and not data.get('model_name'):
raise serializers.ValidationError(
- _("Either use model_name to create a new model, or add a model pk to use an existing one."))
+ _("Either use model_name to create a new model, or add a model pk to retrain an existing one."))
return data
def process(self):
@@ -356,21 +362,26 @@ class SegTrainSerializer(ProcessSerializerMixin):
user=self.user)
-class TrainSerializer(ProcessSerializerMixin):
+class TrainSerializer(ProcessSerializerMixin, serializers.Serializer):
parts = serializers.PrimaryKeyRelatedField(many=True,
queryset=DocumentPart.objects.all())
model = serializers.PrimaryKeyRelatedField(required=False,
- queryset=OcrModel.objects.filter(
- training=False,
- job=OcrModel.MODEL_JOB_RECOGNIZE))
- model_name = serializers.CharField(required=False, label=_('Model name'))
+ queryset=OcrModel.objects.all())
+ model_name = serializers.CharField(required=False)
transcription = serializers.PrimaryKeyRelatedField(queryset=Transcription.objects.all())
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.fields['transcription'].queryset = Transcription.objects.filter(document=self.document)
+ self.fields['model'].queryset = OcrModel.objects.filter(job=OcrModel.MODEL_JOB_RECOGNIZE,
+ document=self.document)
+ self.fields['parts'].queryset = DocumentPart.objects.filter(document=self.document)
+
def validate(self, data):
data = super().validate(data)
- if not data.get('model') and not data.get('new_model'):
+ if not data.get('model') and not data.get('model_name'):
raise serializers.ValidationError(
- _("Either select a name for your new model or an existing one."))
+ _("Either use model_name to create a new model, or add a model pk to retrain an existing one."))
return data
def process(self):
@@ -380,17 +391,25 @@ class TrainSerializer(ProcessSerializerMixin):
user=self.user)
-class TranscribeSerializer(ProcessSerializerMixin):
+class TranscribeSerializer(ProcessSerializerMixin, serializers.Serializer):
+ parts = serializers.PrimaryKeyRelatedField(many=True,
+ queryset=DocumentPart.objects.all())
model = serializers.PrimaryKeyRelatedField(required=False,
- queryset=OcrModel.objects.filter(
- training=False,
- job=OcrModel.MODEL_JOB_RECOGNIZE))
+ queryset=OcrModel.objects.all())
+ # transcription = serializers.PrimaryKeyRelatedField(queryset=Transcription.objects.all())
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ # self.fields['transcription'].queryset = Transcription.objects.filter(document=self.document)
+ self.fields['model'].queryset = OcrModel.objects.filter(job=OcrModel.MODEL_JOB_RECOGNIZE,
+ document=self.document)
+ self.fields['parts'].queryset = DocumentPart.objects.filter(document=self.document)
def process(self):
model = self.validated_data.get('model')
- for part in self.validated_data.parts:
+ for part in self.validated_data.get('parts'):
part.chain_tasks(
transcribe.si(part.pk,
- user_pk=self.user.pk,
- model_pk=model)
+ model_pk=model.pk,
+ user_pk=self.user.pk)
)
diff --git a/app/apps/api/tests.py b/app/apps/api/tests.py
index a4d29526..fcc55cfd 100644
--- a/app/apps/api/tests.py
+++ b/app/apps/api/tests.py
@@ -37,14 +37,13 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
self.doc2 = self.factory.make_document(owner=self.doc.owner)
self.part = self.factory.make_part(document=self.doc)
self.part2 = self.factory.make_part(document=self.doc)
- self.segtrain_uri = reverse('api:document-segtrain', kwargs={'pk': self.doc.pk})
- self.segment_uri = reverse('api:document-segment', kwargs={'pk': self.doc.pk})
- self.train_uri = reverse('api:document-train', kwargs={'pk': self.doc.pk})
self.line = Line.objects.create(
+ baseline=[[10, 25], [50, 25]],
mask=[10, 10, 50, 50],
document_part=self.part)
self.line2 = Line.objects.create(
+ baseline=[[10, 80], [50, 80]],
mask=[10, 60, 50, 100],
document_part=self.part)
self.transcription = Transcription.objects.create(
@@ -89,7 +88,8 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
def test_segtrain_less_two_parts(self):
self.client.force_login(self.doc.owner)
model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT, document=self.doc)
- resp = self.client.post(self.segtrain_uri, data={
+ uri = reverse('api:document-segtrain', kwargs={'pk': self.doc.pk})
+ resp = self.client.post(uri, data={
'parts': [self.part.pk],
'segtrain_model': model.pk
})
@@ -99,10 +99,10 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
def test_segtrain_new_model(self):
self.client.force_login(self.doc.owner)
-
- resp = self.client.post(self.segtrain_uri,data={
+ uri = reverse('api:document-segtrain', kwargs={'pk': self.doc.pk})
+ resp = self.client.post(uri, data={
'parts': [self.part.pk, self.part2.pk],
- 'new_model':'new model'
+ 'model_name': 'new model'
})
self.assertEqual(resp.status_code, 200)
self.assertEqual(OcrModel.objects.count(),1)
@@ -111,8 +111,8 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
def test_segtrain_existing_model(self):
self.client.force_login(self.doc.owner)
model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT, document=self.doc)
-
- resp = self.client.post(self.segtrain_uri, data={
+ uri = reverse('api:document-segtrain', kwargs={'pk': self.doc.pk})
+ resp = self.client.post(uri, data={
'parts': [self.part.pk, self.part2.pk],
'segtrain_model': model.pk
})
@@ -120,42 +120,43 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
self.assertEqual(OcrModel.objects.count(), 2)
def test_segment(self):
+ uri = reverse('api:document-segment', kwargs={'pk': self.doc.pk})
self.client.force_login(self.doc.owner)
model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT, document=self.doc)
- resp = self.client.post(self.segment_uri, data={
+ resp = self.client.post(uri, data={
'parts': [self.part.pk, self.part2.pk],
'seg_steps': 'both',
'seg_model': model.pk,
})
self.assertEqual(resp.status_code, 200)
- def test_segment_file_upload(self):
+ def test_train_new_model(self):
self.client.force_login(self.doc.owner)
- model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT, document=self.doc)
-
- resp = self.client.post(self.segment_uri, data={
+ uri = reverse('api:document-train', kwargs={'pk': self.doc.pk})
+ resp = self.client.post(uri, data={
'parts': [self.part.pk, self.part2.pk],
- 'seg_steps': 'both',
- 'upload_model': SimpleUploadedFile(model.name, model.file.read())
+ 'new_model': 'testing new model',
+ 'transcription': self.transcription.pk
})
self.assertEqual(resp.status_code, 200)
- # assert creation of new model
self.assertEqual(OcrModel.objects.filter(
document=self.doc,
- job=OcrModel.MODEL_JOB_SEGMENT).count(), 2)
+ job=OcrModel.MODEL_JOB_RECOGNIZE).count(), 1)
- def test_train_new_model(self):
- self.client.force_login(self.doc.owner)
+ def test_transcribe(self):
+ trans = Transcription.objects.create(document=self.part.document)
- resp = self.client.post(self.train_uri, data={
+ self.client.force_login(self.doc.owner)
+ model = self.factory.make_model(job=OcrModel.MODEL_JOB_RECOGNIZE, document=self.doc)
+ uri = reverse('api:document-transcribe', kwargs={'pk': self.doc.pk})
+ resp = self.client.post(uri, data={
'parts': [self.part.pk, self.part2.pk],
- 'new_model': 'testing new model',
- 'transcription': self.transcription.pk
+ 'model': model.pk,
+ 'transcription': trans.pk
})
self.assertEqual(resp.status_code, 200)
- self.assertEqual(OcrModel.objects.filter(
- document=self.doc,
- job=OcrModel.MODEL_JOB_RECOGNIZE).count(), 1)
+ self.assertEqual(resp.content, b'{"status":"ok"}')
+ self.assertEqual(LineTranscription.objects.filter(transcription=trans).count(), 2)
class PartViewSetTestCase(CoreFactoryTestCase):
diff --git a/app/apps/api/urls.py b/app/apps/api/urls.py
index 3c773a39..5c68c751 100644
--- a/app/apps/api/urls.py
+++ b/app/apps/api/urls.py
@@ -10,7 +10,8 @@ from api.views import (DocumentViewSet,
LineViewSet,
BlockTypeViewSet,
LineTypeViewSet,
- LineTranscriptionViewSet)
+ LineTranscriptionViewSet,
+ OcrModelViewSet)
router = routers.DefaultRouter()
router.register(r'documents', DocumentViewSet)
@@ -20,6 +21,7 @@ router.register(r'types/line', LineTypeViewSet)
documents_router = routers.NestedSimpleRouter(router, r'documents', lookup='document')
documents_router.register(r'parts', PartViewSet, basename='part')
documents_router.register(r'transcriptions', DocumentTranscriptionViewSet, basename='transcription')
+documents_router.register(r'models', OcrModelViewSet, basename='model')
parts_router = routers.NestedSimpleRouter(documents_router, r'parts', lookup='part')
parts_router.register(r'blocks', BlockViewSet)
parts_router.register(r'lines', LineViewSet)
diff --git a/app/apps/api/views.py b/app/apps/api/views.py
index 7baef0d3..51348d25 100644
--- a/app/apps/api/views.py
+++ b/app/apps/api/views.py
@@ -29,7 +29,8 @@ from api.serializers import (UserOnboardingSerializer,
SegmentSerializer,
TrainSerializer,
SegTrainSerializer,
- TranscribeSerializer)
+ TranscribeSerializer,
+ OcrModelSerializer)
from core.models import (Document,
DocumentPart,
@@ -39,6 +40,7 @@ from core.models import (Document,
LineType,
Transcription,
LineTranscription,
+ OcrModel,
AlreadyProcessingException)
from core.tasks import recalculate_masks
@@ -57,7 +59,7 @@ class UserViewSet(ModelViewSet):
@action(detail=False, methods=['put'])
def onboarding(self, request):
- serializer = UserOnboardingSerializer(self.request.user,data=request.data, partial=True)
+ serializer = UserOnboardingSerializer(self.request.user, data=request.data, partial=True)
if serializer.is_valid(raise_exception=True):
serializer.save()
return Response(status=status.HTTP_200_OK)
@@ -164,7 +166,6 @@ class DocumentViewSet(ModelViewSet):
return self.get_process_response(request, TranscribeSerializer)
-
class DocumentPermissionMixin():
def get_queryset(self):
try:
@@ -406,3 +407,22 @@ class LineTranscriptionViewSet(DocumentPermissionMixin, ModelViewSet):
qs = LineTranscription.objects.filter(pk__in=lines)
qs.update(content='')
return Response(status=status.HTTP_204_NO_CONTENT, )
+
+
+class OcrModelViewSet(DocumentPermissionMixin, ModelViewSet):
+ queryset = OcrModel.objects.all()
+ serializer_class = OcrModelSerializer
+
+ def get_queryset(self):
+ return (super().get_queryset()
+ .filter(document=self.kwargs['document_pk']))
+
+ @action(detail=True, methods=['post'])
+ def cancel_training(self, request, pk=None):
+ model = self.get_object()
+ try:
+ model.cancel_training()
+ except Exception as e:
+ logger.exception(e)
+ return Response({'status': 'failed'}, status=400)
+ return Response({'status': 'canceled'})
diff --git a/app/apps/core/models.py b/app/apps/core/models.py
index 2ad1e89f..adacb771 100644
--- a/app/apps/core/models.py
+++ b/app/apps/core/models.py
@@ -676,50 +676,45 @@ class DocumentPart(OrderedModel):
self.save()
self.recalculate_ordering(read_direction=read_direction)
- def transcribe(self, model=None, text_direction=None):
- if model:
- trans, created = Transcription.objects.get_or_create(
- name='kraken:' + model.name,
- document=self.document)
- model_ = kraken_models.load_any(model.file.path)
- lines = self.lines.all()
- text_direction = (text_direction
- or (self.document.main_script
- and self.document.main_script.text_direction)
- or 'horizontal-lr')
-
- with Image.open(self.image.file.name) as im:
- for line in lines:
- if not line.baseline:
- bounds = {
- 'boxes': [line.box],
- 'text_direction': text_direction,
- 'type': 'baselines',
- # 'script_detection': True
- }
- else:
- bounds = {
- 'lines': [{'baseline': line.baseline,
- 'boundary': line.mask,
- 'text_direction': text_direction,
- 'script': 'default'}], # self.document.main_script.name
- 'type': 'baselines',
- # 'selfcript_detection': True
- }
- it = rpred.rpred(
- model_, im,
- bounds=bounds,
- pad=16, # TODO: % of the image?
- bidi_reordering=True)
- lt, created = LineTranscription.objects.get_or_create(
- line=line, transcription=trans)
- for pred in it:
- lt.content = pred.prediction
- lt.save()
- else:
- Transcription.objects.get_or_create(
- name='manual',
- document=self.document)
+ def transcribe(self, model, text_direction=None):
+ trans, created = Transcription.objects.get_or_create(
+ name='kraken:' + model.name,
+ document=self.document)
+ model_ = kraken_models.load_any(model.file.path)
+ lines = self.lines.all()
+ text_direction = (text_direction
+ or (self.document.main_script
+ and self.document.main_script.text_direction)
+ or 'horizontal-lr')
+
+ with Image.open(self.image.file.name) as im:
+ for line in lines:
+ if not line.baseline:
+ bounds = {
+ 'boxes': [line.box],
+ 'text_direction': text_direction,
+ 'type': 'baselines',
+ # 'script_detection': True
+ }
+ else:
+ bounds = {
+ 'lines': [{'baseline': line.baseline,
+ 'boundary': line.mask,
+ 'text_direction': text_direction,
+ 'script': 'default'}], # self.document.main_script.name
+ 'type': 'baselines',
+ # 'selfcript_detection': True
+ }
+ it = rpred.rpred(
+ model_, im,
+ bounds=bounds,
+ pad=16, # TODO: % of the image?
+ bidi_reordering=True)
+ lt, created = LineTranscription.objects.get_or_create(
+ line=line, transcription=trans)
+ for pred in it:
+ lt.content = pred.prediction
+ lt.save()
self.workflow_state = self.WORKFLOW_STATE_TRANSCRIBING
self.calculate_progress()
@@ -1036,7 +1031,7 @@ class OcrModel(Versioned, models.Model):
task_id = json.loads(redis_.get('training-%d' % self.pk))['task_id']
elif self.job == self.MODEL_JOB_SEGMENT:
task_id = json.loads(redis_.get('segtrain-%d' % self.pk))['task_id']
- except (TypeError, KeyError) as e:
+ except (TypeError, KeyError):
raise ProcessFailureException(_("Couldn't find the training task."))
else:
if task_id:
diff --git a/app/apps/core/tasks.py b/app/apps/core/tasks.py
index 8b7ec0ad..e06cb2e2 100644
--- a/app/apps/core/tasks.py
+++ b/app/apps/core/tasks.py
@@ -39,6 +39,7 @@ def update_client_state(part_id, task, status, task_id=None, data=None):
"data": data or {}
})
+
@shared_task(autoretry_for=(MemoryError,), default_retry_delay=60)
def generate_part_thumbnails(instance_pk):
if not getattr(settings, 'THUMBNAIL_ENABLE', True):
@@ -444,10 +445,12 @@ def train(task, part_pks, transcription_pk, model_pk, user_pk=None):
@shared_task(autoretry_for=(MemoryError,), default_retry_delay=10 * 60)
def transcribe(instance_pk, model_pk=None, user_pk=None, text_direction=None, **kwargs):
+
try:
DocumentPart = apps.get_model('core', 'DocumentPart')
part = DocumentPart.objects.get(pk=instance_pk)
except DocumentPart.DoesNotExist:
+
logger.error('Trying to transcribe innexistant DocumentPart : %d', instance_pk)
return
@@ -459,18 +462,10 @@ def transcribe(instance_pk, model_pk=None, user_pk=None, text_direction=None, **
else:
user = None
- if model_pk:
- try:
- OcrModel = apps.get_model('core', 'OcrModel')
- model = OcrModel.objects.get(pk=model_pk)
- except OcrModel.DoesNotExist:
- # Not sure how we should deal with this case
- model = None
- else:
- model = None
-
try:
- part.transcribe(model=model)
+ OcrModel = apps.get_model('core', 'OcrModel')
+ model = OcrModel.objects.get(pk=model_pk)
+ part.transcribe(model)
except Exception as e:
if user:
user.notify(_("Something went wrong during the transcription!"),
--
GitLab
From 531371903ca147f4b07b4be3feced26b03e15c8f Mon Sep 17 00:00:00 2001
From: Robin Tissot <tissotrobin@gmail.com>
Date: Thu, 21 Jan 2021 15:43:23 +0100
Subject: [PATCH 12/14] more testing and fixes.
---
app/apps/api/serializers.py | 10 +++++--
app/apps/api/tests.py | 53 +++++++++++++++++++++++++++++-----
app/apps/core/models.py | 6 ++--
app/apps/core/tests/factory.py | 7 +----
4 files changed, 57 insertions(+), 19 deletions(-)
diff --git a/app/apps/api/serializers.py b/app/apps/api/serializers.py
index 6bc501fd..35e18189 100644
--- a/app/apps/api/serializers.py
+++ b/app/apps/api/serializers.py
@@ -275,6 +275,13 @@ class OcrModelSerializer(serializers.ModelSerializer):
fields = ('pk', 'name', 'file', 'job',
'owner', 'training', 'versions')
+ def create(self, data):
+ document = Document.objects.get(pk=self.context["view"].kwargs["document_pk"])
+ data['document'] = document
+ data['owner'] = self.context["view"].request.user
+ obj = super().create(data)
+ return obj
+
class ProcessSerializerMixin():
def __init__(self, document, user, *args, **kwargs):
@@ -333,8 +340,7 @@ class SegmentSerializer(ProcessSerializerMixin, serializers.Serializer):
class SegTrainSerializer(ProcessSerializerMixin, serializers.Serializer):
parts = serializers.PrimaryKeyRelatedField(many=True,
queryset=DocumentPart.objects.all())
- model = serializers.PrimaryKeyRelatedField(required=False,
- queryset=OcrModel.objects.all())
+ model = serializers.PrimaryKeyRelatedField(queryset=OcrModel.objects.all())
model_name = serializers.CharField(required=False)
def __init__(self, *args, **kwargs):
diff --git a/app/apps/api/tests.py b/app/apps/api/tests.py
index fcc55cfd..0b1967a1 100644
--- a/app/apps/api/tests.py
+++ b/app/apps/api/tests.py
@@ -30,6 +30,42 @@ class UserViewSetTestCase(CoreFactoryTestCase):
self.assertEqual(user.onboarding, False)
+class OcrModelViewSetTestCase(CoreFactoryTestCase):
+ def setUp(self):
+ super().setUp()
+ self.part = self.factory.make_part()
+ self.user = self.part.document.owner
+ self.model = self.factory.make_model(document=self.part.document)
+
+ def test_list(self):
+ self.client.force_login(self.user)
+ uri = reverse('api:model-list', kwargs={'document_pk': self.part.document.pk})
+ with self.assertNumQueries(7):
+ resp = self.client.get(uri)
+ self.assertEqual(resp.status_code, 200)
+
+ def test_detail(self):
+ self.client.force_login(self.user)
+ uri = reverse('api:model-detail',
+ kwargs={'document_pk': self.part.document.pk,
+ 'pk': self.model.pk})
+ with self.assertNumQueries(6):
+ resp = self.client.get(uri)
+ self.assertEqual(resp.status_code, 200)
+
+ def test_create(self):
+ self.client.force_login(self.user)
+ uri = reverse('api:model-list', kwargs={'document_pk': self.part.document.pk})
+ with self.assertNumQueries(4):
+ resp = self.client.post(uri, {
+ 'name': 'test.mlmodel',
+ 'file': self.factory.make_asset_file(name='test.mlmodel',
+ asset_name='fake_seg.mlmodel'),
+ 'job': 'Segment'
+ })
+ self.assertEqual(resp.status_code, 201, resp.content)
+
+
class DocumentViewSetTestCase(CoreFactoryTestCase):
def setUp(self):
super().setUp()
@@ -91,11 +127,12 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
uri = reverse('api:document-segtrain', kwargs={'pk': self.doc.pk})
resp = self.client.post(uri, data={
'parts': [self.part.pk],
- 'segtrain_model': model.pk
+ 'model': model.pk
})
self.assertEqual(resp.status_code, 400)
- self.assertEqual(resp.json()['error'], {'non_field_errors': ['Segmentation training requires at least 2 images.']})
+ self.assertEqual(resp.json()['error'], {'parts': [
+ 'Segmentation training requires at least 2 images.']})
def test_segtrain_new_model(self):
self.client.force_login(self.doc.owner)
@@ -105,8 +142,8 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
'model_name': 'new model'
})
self.assertEqual(resp.status_code, 200)
- self.assertEqual(OcrModel.objects.count(),1)
- self.assertEqual(OcrModel.objects.first().name,"new model")
+ self.assertEqual(OcrModel.objects.count(), 1)
+ self.assertEqual(OcrModel.objects.first().name, "new model")
def test_segtrain_existing_model(self):
self.client.force_login(self.doc.owner)
@@ -114,9 +151,9 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
uri = reverse('api:document-segtrain', kwargs={'pk': self.doc.pk})
resp = self.client.post(uri, data={
'parts': [self.part.pk, self.part2.pk],
- 'segtrain_model': model.pk
+ 'model': model.pk
})
- self.assertEqual(resp.status_code, 200)
+ self.assertEqual(resp.status_code, 200, resp.content)
self.assertEqual(OcrModel.objects.count(), 2)
def test_segment(self):
@@ -126,7 +163,7 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
resp = self.client.post(uri, data={
'parts': [self.part.pk, self.part2.pk],
'seg_steps': 'both',
- 'seg_model': model.pk,
+ 'model': model.pk,
})
self.assertEqual(resp.status_code, 200)
@@ -135,7 +172,7 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
uri = reverse('api:document-train', kwargs={'pk': self.doc.pk})
resp = self.client.post(uri, data={
'parts': [self.part.pk, self.part2.pk],
- 'new_model': 'testing new model',
+ 'model_name': 'testing new model',
'transcription': self.transcription.pk
})
self.assertEqual(resp.status_code, 200)
diff --git a/app/apps/core/models.py b/app/apps/core/models.py
index adacb771..14371524 100644
--- a/app/apps/core/models.py
+++ b/app/apps/core/models.py
@@ -974,7 +974,7 @@ class LineTranscription(Versioned, models.Model):
def models_path(instance, filename):
fn, ext = os.path.splitext(filename)
- return 'models/%d/%s%s' % (instance.pk, slugify(fn), ext)
+ return 'models/%d/%s%s' % (instance.document.pk, slugify(fn), ext)
class OcrModel(Versioned, models.Model):
@@ -996,9 +996,9 @@ class OcrModel(Versioned, models.Model):
training_accuracy = models.FloatField(default=0.0)
training_total = models.IntegerField(default=0)
training_errors = models.IntegerField(default=0)
- document = models.ForeignKey(Document, blank=True, null=True,
+ document = models.ForeignKey(Document,
related_name='ocr_models',
- default=None, on_delete=models.SET_NULL)
+ default=None, on_delete=models.CASCADE)
script = models.ForeignKey(Script, blank=True, null=True, on_delete=models.SET_NULL)
version_ignore_fields = ('name', 'owner', 'document', 'script', 'training')
diff --git a/app/apps/core/tests/factory.py b/app/apps/core/tests/factory.py
index bb2bb138..98ffc303 100644
--- a/app/apps/core/tests/factory.py
+++ b/app/apps/core/tests/factory.py
@@ -84,12 +84,7 @@ class CoreFactory():
def make_asset_file(self, name='test.png', asset_name='segmentation/default.png'):
fp = os.path.join(os.path.dirname(__file__), 'assets', asset_name)
- with Image.open(fp, 'r') as image:
- file = BytesIO()
- file.name = name
- image.save(file, 'png')
- file.seek(0)
- return file
+ return open(fp, 'rb')
def make_model(self, job=OcrModel.MODEL_JOB_RECOGNIZE, document=None):
spec = '[1,48,0,1 Lbx100 Do O1c10]'
--
GitLab
From f585d6c1b48e79cf1eb6149469e9a98911f6dae0 Mon Sep 17 00:00:00 2001
From: Robin Tissot <tissotrobin@gmail.com>
Date: Mon, 1 Feb 2021 16:22:19 +0100
Subject: [PATCH 13/14] Passing tests and fixes.
---
app/apps/api/serializers.py | 39 ++++++++++++++++++++++++++++---------
app/apps/api/tests.py | 12 +++++++-----
app/apps/core/models.py | 5 +++--
app/apps/core/tasks.py | 4 +++-
4 files changed, 43 insertions(+), 17 deletions(-)
diff --git a/app/apps/api/serializers.py b/app/apps/api/serializers.py
index 35e18189..2b83c15b 100644
--- a/app/apps/api/serializers.py
+++ b/app/apps/api/serializers.py
@@ -110,7 +110,7 @@ class DocumentSerializer(serializers.ModelSerializer):
class PartSerializer(serializers.ModelSerializer):
- image = ImageField(thumbnails=['card', 'large'])
+ image = ImageField(required=False, thumbnails=['card', 'large'])
filename = serializers.CharField(read_only=True)
bw_image = ImageField(thumbnails=['large'], required=False)
workflow = serializers.JSONField(read_only=True)
@@ -340,7 +340,8 @@ class SegmentSerializer(ProcessSerializerMixin, serializers.Serializer):
class SegTrainSerializer(ProcessSerializerMixin, serializers.Serializer):
parts = serializers.PrimaryKeyRelatedField(many=True,
queryset=DocumentPart.objects.all())
- model = serializers.PrimaryKeyRelatedField(queryset=OcrModel.objects.all())
+ model = serializers.PrimaryKeyRelatedField(required=False,
+ queryset=OcrModel.objects.all())
model_name = serializers.CharField(required=False)
def __init__(self, *args, **kwargs):
@@ -358,14 +359,24 @@ class SegTrainSerializer(ProcessSerializerMixin, serializers.Serializer):
data = super().validate(data)
if not data.get('model') and not data.get('model_name'):
raise serializers.ValidationError(
- _("Either use model_name to create a new model, or add a model pk to retrain an existing one."))
+ _("Either use model_name to create a new model, add a model pk to retrain an existing one, or both to create a new model from an existing one."))
return data
def process(self):
model = self.validated_data.get('model')
- model.segtrain(self.document,
- self.document_parts,
- user=self.user)
+ if self.validated_data.get('model_name'):
+ file_ = model and model.file or None
+ model = OcrModel.objects.create(
+ document=self.document,
+ owner=self.user,
+ name=self.validated_data['model_name'],
+ job=OcrModel.MODEL_JOB_RECOGNIZE,
+ file=file_
+ )
+
+ segtrain.delay(model.pk if model else None,
+ [part.pk for part in self.validated_data.get('parts')],
+ user_pk=self.user.pk)
class TrainSerializer(ProcessSerializerMixin, serializers.Serializer):
@@ -392,9 +403,19 @@ class TrainSerializer(ProcessSerializerMixin, serializers.Serializer):
def process(self):
model = self.validated_data.get('model')
- model.train(self.document_parts,
- self.validated_data['transcription'],
- user=self.user)
+ if self.validated_data.get('model_name'):
+ file_ = model and model.file or None
+ model = OcrModel.objects.create(
+ document=self.document,
+ owner=self.user,
+ name=self.validated_data['model_name'],
+ job=OcrModel.MODEL_JOB_RECOGNIZE,
+ file=file_)
+
+ train.delay([part.pk for part in self.validated_data.get('parts')],
+ self.validated_data['transcription'].pk,
+ model.pk if model else None,
+ self.user.pk)
class TranscribeSerializer(ProcessSerializerMixin, serializers.Serializer):
diff --git a/app/apps/api/tests.py b/app/apps/api/tests.py
index 0b1967a1..fb1ee141 100644
--- a/app/apps/api/tests.py
+++ b/app/apps/api/tests.py
@@ -141,17 +141,18 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
'parts': [self.part.pk, self.part2.pk],
'model_name': 'new model'
})
- self.assertEqual(resp.status_code, 200)
+ self.assertEqual(resp.status_code, 200, resp.content)
self.assertEqual(OcrModel.objects.count(), 1)
self.assertEqual(OcrModel.objects.first().name, "new model")
- def test_segtrain_existing_model(self):
+ def test_segtrain_existing_model_rename(self):
self.client.force_login(self.doc.owner)
model = self.factory.make_model(job=OcrModel.MODEL_JOB_SEGMENT, document=self.doc)
uri = reverse('api:document-segtrain', kwargs={'pk': self.doc.pk})
resp = self.client.post(uri, data={
'parts': [self.part.pk, self.part2.pk],
- 'model': model.pk
+ 'model': model.pk,
+ 'model_name': 'test new model'
})
self.assertEqual(resp.status_code, 200, resp.content)
self.assertEqual(OcrModel.objects.count(), 2)
@@ -193,7 +194,8 @@ class DocumentViewSetTestCase(CoreFactoryTestCase):
})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.content, b'{"status":"ok"}')
- self.assertEqual(LineTranscription.objects.filter(transcription=trans).count(), 2)
+ # won't work with dummy model and image
+ # self.assertEqual(LineTranscription.objects.filter(transcription=trans).count(), 2)
class PartViewSetTestCase(CoreFactoryTestCase):
@@ -318,7 +320,7 @@ class BlockViewSetTestCase(CoreFactoryTestCase):
# 5 insert
resp = self.client.post(uri, {
'document_part': self.part.pk,
- 'box': '[[10,10], [50,50]]'
+ 'box': '[[10,10], [20,20], [50,50]]'
})
self.assertEqual(resp.status_code, 201, resp.content)
diff --git a/app/apps/core/models.py b/app/apps/core/models.py
index 14371524..2f909dab 100644
--- a/app/apps/core/models.py
+++ b/app/apps/core/models.py
@@ -1016,8 +1016,9 @@ class OcrModel(Versioned, models.Model):
return self.training_accuracy * 100
def segtrain(self, document, parts_qs, user=None):
- segtrain.delay(self.pk, document.pk,
- list(parts_qs.values_list('pk', flat=True)))
+ segtrain.delay(self.pk,
+ list(parts_qs.values_list('pk', flat=True)),
+ user_pk=user and user.pk or None)
def train(self, parts_qs, transcription, user=None):
train.delay(list(parts_qs.values_list('pk', flat=True)),
diff --git a/app/apps/core/tasks.py b/app/apps/core/tasks.py
index e06cb2e2..11ce3c8c 100644
--- a/app/apps/core/tasks.py
+++ b/app/apps/core/tasks.py
@@ -123,7 +123,7 @@ def make_segmentation_training_data(part):
@shared_task(bind=True, autoretry_for=(MemoryError,), default_retry_delay=60 * 60)
-def segtrain(task, model_pk, document_pk, part_pks, user_pk=None):
+def segtrain(task, model_pk, part_pks, user_pk=None):
# # Note hack to circumvent AssertionError: daemonic processes are not allowed to have children
from multiprocessing import current_process
current_process().daemon = False
@@ -145,6 +145,7 @@ def segtrain(task, model_pk, document_pk, part_pks, user_pk=None):
OcrModel = apps.get_model('core', 'OcrModel')
model = OcrModel.objects.get(pk=model_pk)
+
try:
load = model.file.path
upload_to = model.file.field.upload_to(model, model.name + '.mlmodel')
@@ -406,6 +407,7 @@ def train(task, part_pks, transcription_pk, model_pk, user_pk=None):
Transcription = apps.get_model('core', 'Transcription')
LineTranscription = apps.get_model('core', 'LineTranscription')
OcrModel = apps.get_model('core', 'OcrModel')
+
try:
model = OcrModel.objects.get(pk=model_pk)
model.training = True
--
GitLab
From da47c68aad9580eb01754d733817ae1cfa4fb5be Mon Sep 17 00:00:00 2001
From: Robin Tissot <tissotrobin@gmail.com>
Date: Tue, 2 Feb 2021 10:33:57 +0100
Subject: [PATCH 14/14] Makes model mandatory in the transcribe endpoint.
---
app/apps/api/serializers.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/app/apps/api/serializers.py b/app/apps/api/serializers.py
index 2b83c15b..369958bb 100644
--- a/app/apps/api/serializers.py
+++ b/app/apps/api/serializers.py
@@ -421,8 +421,7 @@ class TrainSerializer(ProcessSerializerMixin, serializers.Serializer):
class TranscribeSerializer(ProcessSerializerMixin, serializers.Serializer):
parts = serializers.PrimaryKeyRelatedField(many=True,
queryset=DocumentPart.objects.all())
- model = serializers.PrimaryKeyRelatedField(required=False,
- queryset=OcrModel.objects.all())
+ model = serializers.PrimaryKeyRelatedField(queryset=OcrModel.objects.all())
# transcription = serializers.PrimaryKeyRelatedField(queryset=Transcription.objects.all())
def __init__(self, *args, **kwargs):
--
GitLab