From 57ce6281ca0e1d798ef5abb4c33de43512659675 Mon Sep 17 00:00:00 2001
From: Robin Tissot <tissotrobin@gmail.com>
Date: Wed, 21 Apr 2021 11:03:54 +0200
Subject: [PATCH] Add a new worker/queue to avoid waiting for mask
recalculation. And fix celery acks late setting typo.
---
app/escriptorium/settings.py | 6 ++++--
docker-compose.yml | 4 ++++
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/app/escriptorium/settings.py b/app/escriptorium/settings.py
index 4f6471c7..4c1ebec8 100644
--- a/app/escriptorium/settings.py
+++ b/app/escriptorium/settings.py
@@ -181,19 +181,21 @@ CELERY_RESULT_BACKEND = 'redis://%s:%d' % (REDIS_HOST, REDIS_PORT)
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
-CELERYD_ACKS_LATE = True
+CELERY_ACKS_LATE = True
# time in seconds a user has to wait after a task is started before being able to recover
TASK_RECOVER_DELAY = 60 * 60 * 24 # 1 day
CELERY_TASK_QUEUES = (
Queue('default', routing_key='default'),
+ Queue('live', routing_key='live'), # for everything that needs to be done on the spot to update the ui
Queue('low-priority', routing_key='low-priority'),
- Queue('gpu', routing_key='gpu'),
+ Queue('gpu', routing_key='gpu'), # for everything that could use a GPU
)
CELERY_TASK_DEFAULT_QUEUE = 'default'
CELERY_TASK_ROUTES = {
# 'core.tasks.*': {'queue': 'default'},
+ 'core.tasks.recalculate_masks': {'queue': 'live'},
'core.tasks.generate_part_thumbnails': {'queue': 'low-priority'},
'core.tasks.train': {'queue': 'gpu'},
'core.tasks.segtrain': {'queue': 'gpu'},
diff --git a/docker-compose.yml b/docker-compose.yml
index 6a3de0b9..f8f155c8 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -73,6 +73,10 @@ services:
- OMP_NUM_THREADS=1
command: "celery worker -l INFO -E -A escriptorium -Ofair --prefetch-multiplier 1 -Q default -c ${CELERY_MAIN_CONC:-10} --max-tasks-per-child=10"
+ celery-live:
+ <<: *app
+ command: "celery worker -l INFO -E -A escriptorium -Ofair --prefetch-multiplier 1 -Q live -c ${CELERY_LIVE_CONC:-10} --max-tasks-per-child=10"
+
celery-low-priority:
<<: *app
command: "celery worker -l INFO -E -A escriptorium -Ofair --prefetch-multiplier 1 -Q low-priority -c ${CELERY_LOW_CONC:-10} --max-tasks-per-child=10"
--
GitLab