feat(tasks): add retry logic with exponential backoff and jitter

- Rewrite app/tasks/retry_config.py with compute_countdown() function
  implementing per-retry delays with ±20% jitter (default: 60s, 300s, 900s)
- Add BaseTaskWithRetry.retry() override to inject proper countdown
- Add OcrTaskWithRetry (120s, 600s, 1800s) for OCR/AI tasks
- Add UploadTaskWithRetry for cloud-storage upload tasks
- Add config settings: TASK_RETRY_MAX_RETRIES, TASK_RETRY_DELAYS, TASK_RETRY_JITTER
- Update process_with_ocr and process_with_azure tasks to use OcrTaskWithRetry
- Update all 11 upload tasks to use UploadTaskWithRetry
- Add 38 unit tests in tests/test_retry_config.py
- Update docs/ConfigurationGuide.md and .env.demo

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-01 17:37:41 +00:00
parent af3ff0b581
commit 5ff7b72a80
18 changed files with 642 additions and 29 deletions
+35
View File
@@ -339,6 +339,32 @@ class Settings(BaseSettings):
),
)
# ---------------------------------------------------------------------------
# Task retry settings (see app/tasks/retry_config.py)
# ---------------------------------------------------------------------------
task_retry_max_retries: int = Field(
default=3,
description=("Maximum number of automatic retry attempts for failed Celery tasks. Default: 3."),
)
task_retry_delays: Union[List[int], str] = Field(
default_factory=lambda: [60, 300, 900],
description=(
"Comma-separated list of retry countdown values in seconds. "
"Each value is the delay before the corresponding retry attempt. "
"If a task fails more times than entries in this list, the last delay "
"is doubled for each additional attempt. "
"Default: 60,300,900 (1 min, 5 min, 15 min)."
),
)
task_retry_jitter: bool = Field(
default=True,
description=(
"Apply ±20 % random jitter to retry countdowns to prevent "
"thundering-herd problems when many tasks fail simultaneously. "
"Default: True (enabled)."
),
)
# Processing step timeout - prevents files from getting stuck in "in_progress" state
step_timeout: int = Field(
default=600,
@@ -527,6 +553,15 @@ class Settings(BaseSettings):
return []
return v
@field_validator("task_retry_delays", mode="before")
@classmethod
def parse_task_retry_delays(cls, v: str | list) -> list[int]:
"""Parse task retry delays from comma-separated string or list of ints."""
if isinstance(v, str):
parts = [p.strip() for p in v.split(",") if p.strip()]
return [int(p) for p in parts]
return [int(item) for item in v]
@field_validator("session_secret")
@classmethod
def validate_session_secret(cls, v: str | None, info: object) -> str | None: