diff --git a/app/escriptorium/settings.py b/app/escriptorium/settings.py index 4f6471c79ba21d39cea427be34c4c4a6f3a7c5f9..4c1ebec8415d05b8725acb96e4f05eb0fbbfd42a 100644 --- a/app/escriptorium/settings.py +++ b/app/escriptorium/settings.py @@ -181,19 +181,21 @@ CELERY_RESULT_BACKEND = 'redis://%s:%d' % (REDIS_HOST, REDIS_PORT) CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' -CELERYD_ACKS_LATE = True +CELERY_ACKS_LATE = True # time in seconds a user has to wait after a task is started before being able to recover TASK_RECOVER_DELAY = 60 * 60 * 24 # 1 day CELERY_TASK_QUEUES = ( Queue('default', routing_key='default'), + Queue('live', routing_key='live'), # for everything that needs to be done on the spot to update the ui Queue('low-priority', routing_key='low-priority'), - Queue('gpu', routing_key='gpu'), + Queue('gpu', routing_key='gpu'), # for everything that could use a GPU ) CELERY_TASK_DEFAULT_QUEUE = 'default' CELERY_TASK_ROUTES = { # 'core.tasks.*': {'queue': 'default'}, + 'core.tasks.recalculate_masks': {'queue': 'live'}, 'core.tasks.generate_part_thumbnails': {'queue': 'low-priority'}, 'core.tasks.train': {'queue': 'gpu'}, 'core.tasks.segtrain': {'queue': 'gpu'}, diff --git a/docker-compose.yml b/docker-compose.yml index 6a3de0b94ee7816f552907e350b272416e485b68..f8f155c8789579ec25ded7040881e93d8e578c79 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,6 +73,10 @@ services: - OMP_NUM_THREADS=1 command: "celery worker -l INFO -E -A escriptorium -Ofair --prefetch-multiplier 1 -Q default -c ${CELERY_MAIN_CONC:-10} --max-tasks-per-child=10" + celery-live: + <<: *app + command: "celery worker -l INFO -E -A escriptorium -Ofair --prefetch-multiplier 1 -Q live -c ${CELERY_LIVE_CONC:-10} --max-tasks-per-child=10" + celery-low-priority: <<: *app command: "celery worker -l INFO -E -A escriptorium -Ofair --prefetch-multiplier 1 -Q low-priority -c ${CELERY_LOW_CONC:-10} --max-tasks-per-child=10"