feat(tasks): add retry logic with exponential backoff and jitter
- Rewrite app/tasks/retry_config.py with compute_countdown() function implementing per-retry delays with ±20% jitter (default: 60s, 300s, 900s) - Add BaseTaskWithRetry.retry() override to inject proper countdown - Add OcrTaskWithRetry (120s, 600s, 1800s) for OCR/AI tasks - Add UploadTaskWithRetry for cloud-storage upload tasks - Add config settings: TASK_RETRY_MAX_RETRIES, TASK_RETRY_DELAYS, TASK_RETRY_JITTER - Update process_with_ocr and process_with_azure tasks to use OcrTaskWithRetry - Update all 11 upload tasks to use UploadTaskWithRetry - Add 38 unit tests in tests/test_retry_config.py - Update docs/ConfigurationGuide.md and .env.demo Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
@@ -339,6 +339,32 @@ class Settings(BaseSettings):
|
||||
),
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task retry settings (see app/tasks/retry_config.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
task_retry_max_retries: int = Field(
|
||||
default=3,
|
||||
description=("Maximum number of automatic retry attempts for failed Celery tasks. Default: 3."),
|
||||
)
|
||||
task_retry_delays: Union[List[int], str] = Field(
|
||||
default_factory=lambda: [60, 300, 900],
|
||||
description=(
|
||||
"Comma-separated list of retry countdown values in seconds. "
|
||||
"Each value is the delay before the corresponding retry attempt. "
|
||||
"If a task fails more times than entries in this list, the last delay "
|
||||
"is doubled for each additional attempt. "
|
||||
"Default: 60,300,900 (1 min, 5 min, 15 min)."
|
||||
),
|
||||
)
|
||||
task_retry_jitter: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"Apply ±20 % random jitter to retry countdowns to prevent "
|
||||
"thundering-herd problems when many tasks fail simultaneously. "
|
||||
"Default: True (enabled)."
|
||||
),
|
||||
)
|
||||
|
||||
# Processing step timeout - prevents files from getting stuck in "in_progress" state
|
||||
step_timeout: int = Field(
|
||||
default=600,
|
||||
@@ -527,6 +553,15 @@ class Settings(BaseSettings):
|
||||
return []
|
||||
return v
|
||||
|
||||
@field_validator("task_retry_delays", mode="before")
|
||||
@classmethod
|
||||
def parse_task_retry_delays(cls, v: str | list) -> list[int]:
|
||||
"""Parse task retry delays from comma-separated string or list of ints."""
|
||||
if isinstance(v, str):
|
||||
parts = [p.strip() for p in v.split(",") if p.strip()]
|
||||
return [int(p) for p in parts]
|
||||
return [int(item) for item in v]
|
||||
|
||||
@field_validator("session_secret")
|
||||
@classmethod
|
||||
def validate_session_secret(cls, v: str | None, info: object) -> str | None:
|
||||
|
||||
Reference in New Issue
Block a user