fix(tasks): register missing Celery tasks in celery_worker.py
Add imports for convert_to_pdfa, finalize_document_storage, process_with_ocr, upload_with_rclone, and webhook_tasks which were missing from celery_worker.py, causing "unregistered task" errors at runtime. Add dynamic test that discovers all task modules in app/tasks/ and verifies each is imported in celery_worker.py. Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
@@ -11,14 +11,17 @@ from app.config import settings
|
|||||||
from app.tasks.check_credentials import check_credentials
|
from app.tasks.check_credentials import check_credentials
|
||||||
from app.tasks.compute_embedding import backfill_missing_embeddings, compute_document_embedding # noqa: F401
|
from app.tasks.compute_embedding import backfill_missing_embeddings, compute_document_embedding # noqa: F401
|
||||||
from app.tasks.convert_to_pdf import convert_to_pdf # noqa: F401
|
from app.tasks.convert_to_pdf import convert_to_pdf # noqa: F401
|
||||||
|
from app.tasks.convert_to_pdfa import convert_to_pdfa # noqa: F401
|
||||||
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf # noqa: F401
|
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf # noqa: F401
|
||||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt # noqa: F401
|
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt # noqa: F401
|
||||||
|
from app.tasks.finalize_document_storage import finalize_document_storage # noqa: F401
|
||||||
from app.tasks.imap_tasks import pull_all_inboxes # noqa: F401
|
from app.tasks.imap_tasks import pull_all_inboxes # noqa: F401
|
||||||
from app.tasks.monitor_stalled_steps import monitor_stalled_steps # noqa: F401
|
from app.tasks.monitor_stalled_steps import monitor_stalled_steps # noqa: F401
|
||||||
|
|
||||||
# **Ensure all tasks are imported before Celery starts**
|
# **Ensure all tasks are imported before Celery starts**
|
||||||
from app.tasks.process_document import process_document # noqa: F401
|
from app.tasks.process_document import process_document # noqa: F401
|
||||||
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence # noqa: F401
|
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence # noqa: F401
|
||||||
|
from app.tasks.process_with_ocr import process_with_ocr # noqa: F401
|
||||||
from app.tasks.refine_text_with_gpt import refine_text_with_gpt # noqa: F401
|
from app.tasks.refine_text_with_gpt import refine_text_with_gpt # noqa: F401
|
||||||
from app.tasks.rotate_pdf_pages import rotate_pdf_pages # noqa: F401
|
from app.tasks.rotate_pdf_pages import rotate_pdf_pages # noqa: F401
|
||||||
from app.tasks.send_to_all import send_to_all_destinations # noqa: F401
|
from app.tasks.send_to_all import send_to_all_destinations # noqa: F401
|
||||||
@@ -34,7 +37,9 @@ from app.tasks.upload_to_paperless import upload_to_paperless # noqa: F401
|
|||||||
from app.tasks.upload_to_s3 import upload_to_s3 # noqa: F401
|
from app.tasks.upload_to_s3 import upload_to_s3 # noqa: F401
|
||||||
from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401
|
from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401
|
||||||
from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401
|
from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401
|
||||||
|
from app.tasks.upload_with_rclone import send_to_all_rclone_destinations, upload_with_rclone # noqa: F401
|
||||||
from app.tasks.uptime_kuma_tasks import ping_uptime_kuma # noqa: F401
|
from app.tasks.uptime_kuma_tasks import ping_uptime_kuma # noqa: F401
|
||||||
|
from app.tasks.webhook_tasks import deliver_webhook_task # noqa: F401
|
||||||
|
|
||||||
# Register the settings reload signal handler so workers pick up config changes
|
# Register the settings reload signal handler so workers pick up config changes
|
||||||
from app.utils.settings_sync import register_settings_reload_signal
|
from app.utils.settings_sync import register_settings_reload_signal
|
||||||
|
|||||||
@@ -55,6 +55,51 @@ class TestCeleryWorkerConfig:
|
|||||||
mod = _reload_celery_worker()
|
mod = _reload_celery_worker()
|
||||||
assert mod is not None
|
assert mod is not None
|
||||||
|
|
||||||
|
def test_all_task_modules_registered(self):
|
||||||
|
"""Test that every task module in app/tasks/ is imported in celery_worker.
|
||||||
|
|
||||||
|
Dynamically discovers all task files in app/tasks/ and verifies that
|
||||||
|
celery_worker imports from each one. This prevents new task modules
|
||||||
|
from being accidentally left unregistered.
|
||||||
|
"""
|
||||||
|
import ast
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
tasks_dir = Path("app/tasks")
|
||||||
|
# Discover all task modules (excluding __init__.py and non-task helpers)
|
||||||
|
task_modules = set()
|
||||||
|
for py_file in sorted(tasks_dir.glob("*.py")):
|
||||||
|
if py_file.name.startswith("_"):
|
||||||
|
continue
|
||||||
|
# Only include modules that define Celery tasks (use @celery.task decorator)
|
||||||
|
source = py_file.read_text()
|
||||||
|
tree = ast.parse(source)
|
||||||
|
for node in ast.walk(tree):
|
||||||
|
if isinstance(node, ast.FunctionDef):
|
||||||
|
for decorator in node.decorator_list:
|
||||||
|
decorator_src = ast.dump(decorator)
|
||||||
|
if "celery" in decorator_src and "task" in decorator_src:
|
||||||
|
task_modules.add(py_file.stem)
|
||||||
|
break
|
||||||
|
|
||||||
|
assert task_modules, "Should discover at least one task module"
|
||||||
|
|
||||||
|
# Read celery_worker.py source and collect all 'from app.tasks.X import ...'
|
||||||
|
worker_source = Path("app/celery_worker.py").read_text()
|
||||||
|
worker_tree = ast.parse(worker_source)
|
||||||
|
imported_modules = set()
|
||||||
|
for node in ast.walk(worker_tree):
|
||||||
|
if isinstance(node, ast.ImportFrom) and node.module and node.module.startswith("app.tasks."):
|
||||||
|
# e.g. "app.tasks.convert_to_pdfa" -> "convert_to_pdfa"
|
||||||
|
mod_name = node.module.split(".")[-1]
|
||||||
|
imported_modules.add(mod_name)
|
||||||
|
|
||||||
|
missing = task_modules - imported_modules
|
||||||
|
assert not missing, (
|
||||||
|
f"Task modules not imported in celery_worker.py: {sorted(missing)}. "
|
||||||
|
f"Add 'from app.tasks.<module> import <task> # noqa: F401' for each."
|
||||||
|
)
|
||||||
|
|
||||||
def test_register_settings_reload_signal_called(self):
|
def test_register_settings_reload_signal_called(self):
|
||||||
"""Test that register_settings_reload_signal is called at module load."""
|
"""Test that register_settings_reload_signal is called at module load."""
|
||||||
with (
|
with (
|
||||||
|
|||||||
Reference in New Issue
Block a user