feat(tasks): add retry logic with exponential backoff and jitter

- Rewrite app/tasks/retry_config.py with compute_countdown() function
  implementing per-retry delays with ±20% jitter (default: 60s, 300s, 900s)
- Add BaseTaskWithRetry.retry() override to inject proper countdown
- Add OcrTaskWithRetry (120s, 600s, 1800s) for OCR/AI tasks
- Add UploadTaskWithRetry for cloud-storage upload tasks
- Add config settings: TASK_RETRY_MAX_RETRIES, TASK_RETRY_DELAYS, TASK_RETRY_JITTER
- Update process_with_ocr and process_with_azure tasks to use OcrTaskWithRetry
- Update all 11 upload tasks to use UploadTaskWithRetry
- Add 38 unit tests in tests/test_retry_config.py
- Update docs/ConfigurationGuide.md and .env.demo

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/config.py

@@ -339,6 +339,32 @@ class Settings(BaseSettings):
         ),
     )
 
+    # ---------------------------------------------------------------------------
+    # Task retry settings (see app/tasks/retry_config.py)
+    # ---------------------------------------------------------------------------
+    task_retry_max_retries: int = Field(
+        default=3,
+        description=("Maximum number of automatic retry attempts for failed Celery tasks. Default: 3."),
+    )
+    task_retry_delays: Union[List[int], str] = Field(
+        default_factory=lambda: [60, 300, 900],
+        description=(
+            "Comma-separated list of retry countdown values in seconds. "
+            "Each value is the delay before the corresponding retry attempt. "
+            "If a task fails more times than entries in this list, the last delay "
+            "is doubled for each additional attempt. "
+            "Default: 60,300,900 (1 min, 5 min, 15 min)."
+        ),
+    )
+    task_retry_jitter: bool = Field(
+        default=True,
+        description=(
+            "Apply ±20 % random jitter to retry countdowns to prevent "
+            "thundering-herd problems when many tasks fail simultaneously. "
+            "Default: True (enabled)."
+        ),
+    )
+
     # Processing step timeout - prevents files from getting stuck in "in_progress" state
     step_timeout: int = Field(
         default=600,
@@ -527,6 +553,15 @@ class Settings(BaseSettings):
             return []
         return v
 
+    @field_validator("task_retry_delays", mode="before")
+    @classmethod
+    def parse_task_retry_delays(cls, v: str | list) -> list[int]:
+        """Parse task retry delays from comma-separated string or list of ints."""
+        if isinstance(v, str):
+            parts = [p.strip() for p in v.split(",") if p.strip()]
+            return [int(p) for p in parts]
+        return [int(item) for item in v]
+
     @field_validator("session_secret")
     @classmethod
     def validate_session_secret(cls, v: str | None, info: object) -> str | None:

app/tasks/process_with_azure_document_intelligence.py

@@ -9,7 +9,7 @@ from azure.core.credentials import AzureKeyCredential
 
 from app.celery_app import celery
 from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.retry_config import OcrTaskWithRetry
 from app.tasks.rotate_pdf_pages import rotate_pdf_pages
 from app.utils import log_task_progress
 
@@ -81,7 +81,7 @@ def check_page_rotation(result, filename, task_id=None):
     return rotation_data
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=OcrTaskWithRetry, bind=True)
 def process_with_azure_document_intelligence(self, filename: str, file_id: int = None):
     """
     Processes a PDF document using Azure Document Intelligence and overlays OCR text onto

app/tasks/process_with_ocr.py

@@ -23,7 +23,7 @@ from app.celery_app import celery
 from app.config import settings
 from app.database import SessionLocal
 from app.models import FileRecord
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.retry_config import OcrTaskWithRetry
 from app.tasks.rotate_pdf_pages import rotate_pdf_pages
 from app.utils import log_task_progress
 from app.utils.ocr_provider import OCRResult, embed_text_layer, get_ocr_providers, merge_ocr_results
@@ -32,7 +32,7 @@ from app.utils.text_quality import TextSource, check_text_quality, compare_text_
 logger = logging.getLogger(__name__)
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=OcrTaskWithRetry, bind=True)
 def process_with_ocr(self, filename: str, file_id: Optional[int] = None, original_text: Optional[str] = None):
     """Run the configured OCR providers on *filename* and continue the pipeline.
 

app/tasks/retry_config.py

@@ -1,9 +1,232 @@
 #!/usr/bin/env python3
+"""Retry configuration for Celery tasks with exponential backoff and jitter.
+
+Provides a :class:`BaseTaskWithRetry` Celery task base class that implements
+configurable retry logic with exponential backoff and optional ±20 % random
+jitter.  Pre-defined subclasses offer task-type-specific retry policies:
+
+* :class:`BaseTaskWithRetry` – general default (3 retries: 60 s, 300 s, 900 s)
+* :class:`OcrTaskWithRetry` – longer waits for OCR / AI API calls
+* :class:`UploadTaskWithRetry` – standard waits for cloud-storage uploads
+
+Usage::
+
+    from app.tasks.retry_config import BaseTaskWithRetry, OcrTaskWithRetry
+
+    @celery.task(base=OcrTaskWithRetry, bind=True)
+    def my_ocr_task(self, ...):
+        ...
+"""
+
+import logging
+import random
+from typing import Any
 
 from celery import Task
 
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Defaults
+# ---------------------------------------------------------------------------
+
+#: Default per-retry countdowns in seconds (1 min, 5 min, 15 min).
+DEFAULT_RETRY_DELAYS: list[int] = [60, 300, 900]
+
+
+def _parse_delay_string(value: str) -> list[int]:
+    """Parse a comma-separated string of integers into a list.
+
+    Args:
+        value: Comma-separated integer string, e.g. ``"60,300,900"``.
+
+    Returns:
+        Parsed list of integers, e.g. ``[60, 300, 900]``.
+    """
+    return [int(v.strip()) for v in value.split(",") if v.strip()]
+
+
+def compute_countdown(
+    retries: int,
+    base_delays: list[int] | None = None,
+    jitter: bool = True,
+) -> int:
+    """Compute the countdown in seconds for the next retry attempt.
+
+    Selects the appropriate base delay for the given retry number.  When all
+    defined delays are exhausted the last delay is doubled for each additional
+    attempt.  An optional ±20 % jitter is then applied to spread retry storms.
+
+    Args:
+        retries: Current retry count (0-based; 0 = first retry attempt).
+        base_delays: Ordered list of base countdown values (in seconds) for
+            each retry attempt.  ``None`` uses :data:`DEFAULT_RETRY_DELAYS`.
+        jitter: When ``True``, apply ±20 % random jitter to the countdown.
+
+    Returns:
+        Countdown in seconds (minimum 1 s).
+
+    Examples::
+
+        >>> compute_countdown(0, [60, 300, 900], jitter=False)
+        60
+        >>> compute_countdown(1, [60, 300, 900], jitter=False)
+        300
+        >>> compute_countdown(3, [60, 300, 900], jitter=False)  # beyond list
+        1800
+    """
+    delays = base_delays if base_delays is not None else DEFAULT_RETRY_DELAYS
+
+    if not delays:
+        base = 60
+    elif retries < len(delays):
+        base = delays[retries]
+    else:
+        # Exhausted defined delays – double the last value for each extra attempt.
+        extra = retries - len(delays) + 1
+        base = delays[-1] * (2**extra)
+
+    if jitter:
+        # ±20 % uniform jitter – not cryptographic, S311 is intentional.
+        jitter_factor = 1.0 + random.uniform(-0.2, 0.2)  # noqa: S311
+        base = int(base * jitter_factor)
+
+    return max(base, 1)
+
 
 class BaseTaskWithRetry(Task):
+    """Celery task base class with exponential backoff and optional jitter.
+
+    Automatically retries on any :class:`Exception` using delays derived from
+    :attr:`retry_delays`.  When :attr:`retry_delays` is ``None`` the value is
+    read from ``TASK_RETRY_DELAYS`` (env-var / settings); if that is also
+    unset :data:`DEFAULT_RETRY_DELAYS` (``[60, 300, 900]`` seconds) is used.
+
+    Override class attributes in subclasses to customise per-task-type policy:
+
+    * ``max_retries`` (``int``) – maximum retry attempts; default ``3``.
+    * ``retry_delays`` (``list[int] | None``) – per-retry countdowns in
+      seconds; ``None`` falls back to settings / :data:`DEFAULT_RETRY_DELAYS`.
+    * ``retry_jitter`` (``bool``) – add ±20 % jitter; default ``True``.
+    """
+
+    #: Retry on any exception raised inside the task body.
     autoretry_for = (Exception,)
-    retry_kwargs = {"max_retries": 3, "countdown": 10}  # 3 retries, 10s delay
-    retry_backoff = True  # Exponential backoff
+
+    #: Maximum number of retry attempts.
+    max_retries: int = 3
+
+    #: Pass max_retries through autoretry_for; no countdown override here
+    #: (our retry() method injects the countdown instead).
+    retry_kwargs: dict = {"max_retries": 3}
+
+    #: Per-retry countdown values (seconds).  ``None`` → settings / DEFAULT.
+    retry_delays: list[int] | None = None
+
+    #: Apply ±20 % random jitter to prevent thundering-herd problems.
+    retry_jitter: bool = True
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def retry(
+        self,
+        args: Any = None,
+        kwargs: Any = None,
+        exc: BaseException | None = None,
+        throw: bool = True,
+        eta: Any = None,
+        countdown: int | None = None,
+        max_retries: int | None = None,
+        **options: Any,
+    ) -> Any:
+        """Retry the task, injecting the backoff countdown when not supplied.
+
+        If *countdown* is not explicitly provided (and *eta* is not set) the
+        countdown is computed via :func:`compute_countdown` using this task's
+        :attr:`retry_delays` and :attr:`retry_jitter` settings.
+        """
+        if countdown is None and eta is None:
+            countdown = compute_countdown(
+                retries=self.request.retries,
+                base_delays=self._effective_retry_delays(),
+                jitter=self.retry_jitter,
+            )
+            logger.debug(
+                "Retry %d/%d for task %s in %d s",
+                self.request.retries + 1,
+                max_retries if max_retries is not None else self.max_retries,
+                self.name,
+                countdown,
+            )
+
+        return super().retry(
+            args=args,
+            kwargs=kwargs,
+            exc=exc,
+            throw=throw,
+            eta=eta,
+            countdown=countdown,
+            max_retries=max_retries,
+            **options,
+        )
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _effective_retry_delays(self) -> list[int]:
+        """Return the retry delays to use, with settings-level override support.
+
+        Priority (highest first):
+
+        1. Explicit class-level ``retry_delays`` attribute (not ``None``).
+        2. ``TASK_RETRY_DELAYS`` environment variable / setting.
+        3. :data:`DEFAULT_RETRY_DELAYS` module-level constant.
+        """
+        if self.retry_delays is not None:
+            return self.retry_delays
+
+        # Lazily read from settings to avoid circular imports at module load.
+        try:
+            from app.config import settings  # noqa: PLC0415
+
+            raw = getattr(settings, "task_retry_delays", None)
+            if raw:
+                if isinstance(raw, list):
+                    return [int(v) for v in raw]
+                if isinstance(raw, str):
+                    return _parse_delay_string(raw)
+        except Exception as exc:  # pragma: no cover
+            logger.debug("Could not read task_retry_delays from settings: %s", exc)
+
+        return DEFAULT_RETRY_DELAYS
+
+
+# ---------------------------------------------------------------------------
+# Task-type-specific retry policies
+# ---------------------------------------------------------------------------
+
+
+class OcrTaskWithRetry(BaseTaskWithRetry):
+    """Retry policy for OCR and document-intelligence API tasks.
+
+    Uses longer initial delays to allow transient API rate-limit windows to
+    clear before the next attempt.
+
+    Default: 3 retries at 120 s, 600 s, 1800 s.
+    """
+
+    retry_delays: list[int] = [120, 600, 1800]
+
+
+class UploadTaskWithRetry(BaseTaskWithRetry):
+    """Retry policy for cloud-storage upload tasks.
+
+    Uses the standard default delays (60 s, 300 s, 900 s) which are
+    appropriate for most transient upload failures (network blips, rate
+    limits, temporary service outages).
+    """
+
+    # Inherits DEFAULT_RETRY_DELAYS via retry_delays = None.

app/tasks/upload_to_dropbox.py

@@ -9,7 +9,7 @@ from dropbox.exceptions import ApiError, AuthError
 
 from app.celery_app import celery
 from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.retry_config import UploadTaskWithRetry
 from app.utils import log_task_progress
 from app.utils.filename_utils import extract_remote_path, get_unique_filename
 
@@ -102,7 +102,7 @@ def get_dropbox_client():
         raise
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=UploadTaskWithRetry, bind=True)
 def upload_to_dropbox(self, file_path: str, file_id: int = None):
     """
     Upload a file to Dropbox.

app/tasks/upload_to_email.py

@@ -15,7 +15,7 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape
 
 from app.celery_app import celery
 from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.retry_config import UploadTaskWithRetry
 from app.utils import log_task_progress
 
 logger = logging.getLogger(__name__)
@@ -166,7 +166,7 @@ def _send_email_with_smtp(msg, filename, recipients):
         return {"status": "Failed", "reason": error_msg, "error": str(e)}
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=UploadTaskWithRetry, bind=True)
 def upload_to_email(
     self,
     file_path: str,

app/tasks/upload_to_ftp.py

@@ -8,13 +8,13 @@ import os
 
 from app.celery_app import celery
 from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.retry_config import UploadTaskWithRetry
 from app.utils import log_task_progress
 
 logger = logging.getLogger(__name__)
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=UploadTaskWithRetry, bind=True)
 def upload_to_ftp(self, file_path: str, file_id: int = None):
     """
     Uploads a file to an FTP server in the configured folder.

app/tasks/upload_to_google_drive.py

@@ -15,7 +15,7 @@ from googleapiclient.http import MediaFileUpload
 
 from app.celery_app import celery
 from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.retry_config import UploadTaskWithRetry
 from app.utils import log_task_progress
 
 logger = logging.getLogger(__name__)
@@ -152,7 +152,7 @@ def truncate_property_value(key, value, max_bytes=100):
     return str_value
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=UploadTaskWithRetry, bind=True)
 def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id: int = None):
     """
     Uploads a file to Google Drive in the configured folder with optional metadata.

app/tasks/upload_to_nextcloud.py

@@ -8,14 +8,14 @@ from requests.auth import HTTPBasicAuth
 
 from app.celery_app import celery
 from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.retry_config import UploadTaskWithRetry
 from app.utils import log_task_progress
 from app.utils.filename_utils import extract_remote_path, get_unique_filename
 
 logger = logging.getLogger(__name__)
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=UploadTaskWithRetry, bind=True)
 def upload_to_nextcloud(self, file_path: str, file_id: int = None):
     """
     Upload a file to Nextcloud WebDAV.

app/tasks/upload_to_onedrive.py

@@ -10,7 +10,7 @@ import requests
 
 from app.celery_app import celery
 from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.retry_config import UploadTaskWithRetry
 from app.utils import log_task_progress
 
 logger = logging.getLogger(__name__)
@@ -209,7 +209,7 @@ def upload_large_file(file_path, upload_url):
     return response.json()
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=UploadTaskWithRetry, bind=True)
 def upload_to_onedrive(self, file_path: str, file_id: int = None):
     """
     Uploads a file to OneDrive in the configured folder.

app/tasks/upload_to_paperless.py

@@ -10,7 +10,7 @@ import requests
 
 from app.celery_app import celery
 from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.retry_config import UploadTaskWithRetry
 from app.utils import log_task_progress
 
 logger = logging.getLogger(__name__)
@@ -199,7 +199,7 @@ def set_document_custom_fields(doc_id: int, custom_fields: dict, task_id: str) -
         logger.error(f"[{task_id}] Response: {getattr(exc.response, 'text', '<no response>')}")
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=UploadTaskWithRetry, bind=True)
 def upload_to_paperless(self, file_path: str, file_id: int = None):
     """
     Uploads a file to Paperless-ngx and sets custom fields from metadata.

app/tasks/upload_to_s3.py

@@ -8,13 +8,13 @@ from botocore.exceptions import ClientError
 
 from app.celery_app import celery
 from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.retry_config import UploadTaskWithRetry
 from app.utils import log_task_progress
 
 logger = logging.getLogger(__name__)
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=UploadTaskWithRetry, bind=True)
 def upload_to_s3(self, file_path: str, file_id: int = None):
     """
     Uploads a file to Amazon S3 in the configured bucket and folder.

app/tasks/upload_to_sftp.py

@@ -7,14 +7,14 @@ import paramiko
 
 from app.celery_app import celery
 from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.retry_config import UploadTaskWithRetry
 from app.utils import log_task_progress
 from app.utils.filename_utils import extract_remote_path, get_unique_filename, sanitize_filename
 
 logger = logging.getLogger(__name__)
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=UploadTaskWithRetry, bind=True)
 def upload_to_sftp(self, file_path: str, file_id: int = None):
     """
     Upload a file to an SFTP server.

app/tasks/upload_to_webdav.py

@@ -8,13 +8,13 @@ import requests
 
 from app.celery_app import celery
 from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.retry_config import UploadTaskWithRetry
 from app.utils import log_task_progress
 
 logger = logging.getLogger(__name__)
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=UploadTaskWithRetry, bind=True)
 def upload_to_webdav(self, file_path: str, file_id: int = None):
     """
     Uploads a file to a WebDAV server in the configured folder.

app/tasks/upload_with_rclone.py

@@ -6,13 +6,13 @@ import subprocess
 
 from app.celery_app import celery
 from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
+from app.tasks.retry_config import UploadTaskWithRetry
 from app.utils import log_task_progress
 
 logger = logging.getLogger(__name__)
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=UploadTaskWithRetry, bind=True)
 def upload_with_rclone(self, file_path: str, destination: str):
     """
     Uploads a file using rclone to the specified destination.
@@ -107,7 +107,7 @@ def upload_with_rclone(self, file_path: str, destination: str):
         raise RuntimeError(error_msg) from e
 
 
-@celery.task(base=BaseTaskWithRetry, bind=True)
+@celery.task(base=UploadTaskWithRetry, bind=True)
 def send_to_all_rclone_destinations(self, file_path: str):
     """
     Uploads a file to all configured rclone destinations.