UnisKB/apps/ops/celery/__init__.py

45 lines
1.7 KiB
Python

# -*- coding: utf-8 -*-
import os
from celery import Celery
from celery.schedules import crontab
from kombu import Exchange, Queue
from maxkb import settings
from .heartbeat import *
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
app = Celery('MaxKB')
configs = {k: v for k, v in settings.__dict__.items() if k.startswith('CELERY')}
configs['worker_concurrency'] = 5
# Using a string here means the worker will not have to
# pickle the object when using Windows.
# app.config_from_object('django.conf:settings', namespace='CELERY')
configs["task_queues"] = [
Queue("celery", Exchange("celery"), routing_key="celery"),
Queue("model", Exchange("model"), routing_key="model"),
Queue("uniskb_knowledge", Exchange("uniskb_knowledge"), routing_key="uniskb_knowledge")
]
app.namespace = 'CELERY'
app.conf.update(
{key.replace('CELERY_', '') if key.replace('CELERY_', '').lower() == key.replace('CELERY_',
'') else key: configs.get(
key) for
key
in configs.keys()})
# 新知识库任务与共享 Redis 中的旧 celery 队列隔离。
knowledge_routes = {
"celery:embedding_*": {"queue": "uniskb_knowledge"},
"celery:rebuild_knowledge_index": {"queue": "uniskb_knowledge"},
"celery:finalize_rebuild_task": {"queue": "uniskb_knowledge"},
}
existing_routes = app.conf.task_routes or ()
if isinstance(existing_routes, dict):
existing_routes = (existing_routes,)
app.conf.task_routes = (knowledge_routes, *existing_routes)
app.autodiscover_tasks(lambda: [app_config.split('.')[0] for app_config in settings.INSTALLED_APPS])