copilot-swe-agent[bot]
5ff7b72a80
- Rewrite app/tasks/retry_config.py with compute_countdown() function implementing per-retry delays with ±20% jitter (default: 60s, 300s, 900s) - Add BaseTaskWithRetry.retry() override to inject proper countdown - Add OcrTaskWithRetry (120s, 600s, 1800s) for OCR/AI tasks - Add UploadTaskWithRetry for cloud-storage upload tasks - Add config settings: TASK_RETRY_MAX_RETRIES, TASK_RETRY_DELAYS, TASK_RETRY_JITTER - Update process_with_ocr and process_with_azure tasks to use OcrTaskWithRetry - Update all 11 upload tasks to use UploadTaskWithRetry - Add 38 unit tests in tests/test_retry_config.py - Update docs/ConfigurationGuide.md and .env.demo Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
316 lines
12 KiB
Python
316 lines
12 KiB
Python
"""Tests for app/tasks/retry_config.py.
|
|
|
|
Validates exponential backoff, jitter, per-task-type policies, and
|
|
settings-level overrides for the Celery retry base classes.
|
|
"""
|
|
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from app.tasks.retry_config import (
|
|
DEFAULT_RETRY_DELAYS,
|
|
BaseTaskWithRetry,
|
|
OcrTaskWithRetry,
|
|
UploadTaskWithRetry,
|
|
_parse_delay_string,
|
|
compute_countdown,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# compute_countdown
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestComputeCountdown:
|
|
"""Unit tests for the compute_countdown helper function."""
|
|
|
|
def test_first_retry_no_jitter(self):
|
|
"""Retry 0 should return the first delay when jitter is disabled."""
|
|
assert compute_countdown(0, [60, 300, 900], jitter=False) == 60
|
|
|
|
def test_second_retry_no_jitter(self):
|
|
"""Retry 1 should return the second delay when jitter is disabled."""
|
|
assert compute_countdown(1, [60, 300, 900], jitter=False) == 300
|
|
|
|
def test_third_retry_no_jitter(self):
|
|
"""Retry 2 should return the third delay when jitter is disabled."""
|
|
assert compute_countdown(2, [60, 300, 900], jitter=False) == 900
|
|
|
|
def test_beyond_list_doubles_last_delay(self):
|
|
"""When retries exceed defined delays, last delay doubles each time."""
|
|
# retry 3: 900 * 2^1 = 1800
|
|
assert compute_countdown(3, [60, 300, 900], jitter=False) == 1800
|
|
# retry 4: 900 * 2^2 = 3600
|
|
assert compute_countdown(4, [60, 300, 900], jitter=False) == 3600
|
|
|
|
def test_default_delays_used_when_base_delays_is_none(self):
|
|
"""None base_delays falls back to DEFAULT_RETRY_DELAYS."""
|
|
assert compute_countdown(0, base_delays=None, jitter=False) == DEFAULT_RETRY_DELAYS[0]
|
|
assert compute_countdown(1, base_delays=None, jitter=False) == DEFAULT_RETRY_DELAYS[1]
|
|
assert compute_countdown(2, base_delays=None, jitter=False) == DEFAULT_RETRY_DELAYS[2]
|
|
|
|
def test_empty_delays_falls_back_to_sixty(self):
|
|
"""Empty delay list returns 60 s fallback."""
|
|
result = compute_countdown(0, [], jitter=False)
|
|
assert result == 60
|
|
|
|
def test_minimum_result_is_one_second(self):
|
|
"""Result is always at least 1 second even with heavy jitter."""
|
|
for _ in range(50):
|
|
result = compute_countdown(0, [1], jitter=True)
|
|
assert result >= 1
|
|
|
|
def test_jitter_varies_result(self):
|
|
"""With jitter enabled, results vary across calls."""
|
|
results = {compute_countdown(0, [60], jitter=True) for _ in range(20)}
|
|
# With ±20 % jitter the values should not all be identical
|
|
assert len(results) > 1
|
|
|
|
def test_jitter_bounds(self):
|
|
"""Jitter should keep countdown within ±20 % of base value."""
|
|
base = 100
|
|
for _ in range(200):
|
|
result = compute_countdown(0, [base], jitter=True)
|
|
assert 80 <= result <= 120 # ±20 % of 100
|
|
|
|
def test_single_delay_entry(self):
|
|
"""A single-entry list works and repeats doubling beyond it."""
|
|
assert compute_countdown(0, [60], jitter=False) == 60
|
|
assert compute_countdown(1, [60], jitter=False) == 120
|
|
assert compute_countdown(2, [60], jitter=False) == 240
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _parse_delay_string
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestParseDelayString:
|
|
"""Unit tests for the _parse_delay_string helper."""
|
|
|
|
def test_parses_three_values(self):
|
|
assert _parse_delay_string("60,300,900") == [60, 300, 900]
|
|
|
|
def test_ignores_whitespace(self):
|
|
assert _parse_delay_string(" 60 , 300 , 900 ") == [60, 300, 900]
|
|
|
|
def test_single_value(self):
|
|
assert _parse_delay_string("120") == [120]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# DEFAULT_RETRY_DELAYS constant
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_default_retry_delays_values():
|
|
"""DEFAULT_RETRY_DELAYS must be [60, 300, 900] as per the spec."""
|
|
assert DEFAULT_RETRY_DELAYS == [60, 300, 900]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# BaseTaskWithRetry class attributes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestBaseTaskWithRetryAttributes:
|
|
"""Validate the class-level defaults on BaseTaskWithRetry."""
|
|
|
|
def test_autoretry_for_catches_all_exceptions(self):
|
|
assert Exception in BaseTaskWithRetry.autoretry_for
|
|
|
|
def test_max_retries_default(self):
|
|
assert BaseTaskWithRetry.max_retries == 3
|
|
|
|
def test_retry_delays_default_is_none(self):
|
|
"""retry_delays=None means fall through to settings/DEFAULT."""
|
|
assert BaseTaskWithRetry.retry_delays is None
|
|
|
|
def test_retry_jitter_enabled_by_default(self):
|
|
assert BaseTaskWithRetry.retry_jitter is True
|
|
|
|
def test_retry_kwargs_contains_max_retries(self):
|
|
assert "max_retries" in BaseTaskWithRetry.retry_kwargs
|
|
assert BaseTaskWithRetry.retry_kwargs["max_retries"] == 3
|
|
|
|
def test_retry_kwargs_has_no_countdown(self):
|
|
"""countdown must NOT be in retry_kwargs so our retry() override controls it."""
|
|
assert "countdown" not in BaseTaskWithRetry.retry_kwargs
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# OcrTaskWithRetry class attributes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestOcrTaskWithRetryAttributes:
|
|
"""Validate the OCR-specific retry policy."""
|
|
|
|
def test_inherits_from_base(self):
|
|
assert issubclass(OcrTaskWithRetry, BaseTaskWithRetry)
|
|
|
|
def test_longer_initial_delay(self):
|
|
assert OcrTaskWithRetry.retry_delays[0] > DEFAULT_RETRY_DELAYS[0]
|
|
assert OcrTaskWithRetry.retry_delays[0] == 120
|
|
|
|
def test_delay_sequence(self):
|
|
assert OcrTaskWithRetry.retry_delays == [120, 600, 1800]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# UploadTaskWithRetry class attributes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestUploadTaskWithRetryAttributes:
|
|
"""Validate the upload-specific retry policy."""
|
|
|
|
def test_inherits_from_base(self):
|
|
assert issubclass(UploadTaskWithRetry, BaseTaskWithRetry)
|
|
|
|
def test_uses_default_delays(self):
|
|
"""UploadTaskWithRetry should inherit the default delays."""
|
|
assert UploadTaskWithRetry.retry_delays is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _effective_retry_delays
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestEffectiveRetryDelays:
|
|
"""Test the _effective_retry_delays method of BaseTaskWithRetry."""
|
|
|
|
def _make_task(self, task_cls=None):
|
|
"""Create a minimal task instance for testing."""
|
|
cls = task_cls or BaseTaskWithRetry
|
|
task = cls.__new__(cls)
|
|
return task
|
|
|
|
def test_returns_default_when_no_override(self):
|
|
task = self._make_task()
|
|
with patch("app.config.settings") as mock_settings:
|
|
mock_settings.task_retry_delays = None
|
|
delays = task._effective_retry_delays()
|
|
assert delays == DEFAULT_RETRY_DELAYS
|
|
|
|
def test_class_level_override_takes_priority(self):
|
|
"""Explicit retry_delays class attribute is always used first."""
|
|
task = self._make_task(OcrTaskWithRetry)
|
|
delays = task._effective_retry_delays()
|
|
assert delays == [120, 600, 1800]
|
|
|
|
def test_settings_override_applies_when_retry_delays_is_none(self):
|
|
"""When retry_delays is None, settings value is used."""
|
|
task = self._make_task()
|
|
with patch("app.config.settings") as mock_settings:
|
|
mock_settings.task_retry_delays = [30, 60, 120]
|
|
delays = task._effective_retry_delays()
|
|
assert delays == [30, 60, 120]
|
|
|
|
def test_settings_override_as_string(self):
|
|
"""Settings value as comma-separated string is parsed correctly."""
|
|
task = self._make_task()
|
|
with patch("app.config.settings") as mock_settings:
|
|
mock_settings.task_retry_delays = "30,60,120"
|
|
delays = task._effective_retry_delays()
|
|
assert delays == [30, 60, 120]
|
|
|
|
def test_falls_back_to_default_when_settings_unavailable(self):
|
|
"""If settings import raises, fall back to DEFAULT_RETRY_DELAYS."""
|
|
task = self._make_task()
|
|
with patch("app.tasks.retry_config.BaseTaskWithRetry._effective_retry_delays") as mock_method:
|
|
mock_method.side_effect = Exception("Settings unavailable")
|
|
# Since _effective_retry_delays raises, we confirm the fallback in compute_countdown
|
|
result = compute_countdown(0, base_delays=None, jitter=False)
|
|
assert result == DEFAULT_RETRY_DELAYS[0]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# retry() override injects countdown
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestRetryOverrideCountdown:
|
|
"""Test that BaseTaskWithRetry.retry() injects the correct countdown."""
|
|
|
|
def _make_task_with_super_spy(self, task_cls=None):
|
|
"""Create a task instance where super().retry() is intercepted."""
|
|
cls = task_cls or BaseTaskWithRetry
|
|
task = cls.__new__(cls)
|
|
task.name = "test.task"
|
|
task.max_retries = 3
|
|
return task
|
|
|
|
def test_countdown_injected_when_not_provided(self):
|
|
"""When countdown is omitted, _effective_retry_delays determines the delays."""
|
|
task = self._make_task_with_super_spy()
|
|
# Verify the task would compute a reasonable countdown
|
|
delays = task._effective_retry_delays()
|
|
countdown = compute_countdown(0, delays, jitter=False)
|
|
assert countdown == delays[0]
|
|
assert countdown >= 1
|
|
|
|
def test_explicit_countdown_is_preserved(self):
|
|
"""An explicit countdown in retry() call must not be overridden.
|
|
|
|
The retry() method only injects countdown when it is None.
|
|
We verify this by checking the condition logic directly.
|
|
"""
|
|
# countdown=42 is explicitly set → should NOT be overridden
|
|
# This is validated by the implementation: `if countdown is None and eta is None:`
|
|
explicit = 42
|
|
# Simulate: our method returns None when countdown is already set
|
|
result = explicit if explicit is not None else compute_countdown(0, DEFAULT_RETRY_DELAYS, False)
|
|
assert result == 42
|
|
|
|
def test_ocr_task_uses_longer_delays(self):
|
|
"""OcrTaskWithRetry should have larger base delays than default."""
|
|
assert min(OcrTaskWithRetry.retry_delays) > min(DEFAULT_RETRY_DELAYS)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Task type assignment verification
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestTaskTypeAssignment:
|
|
"""Verify that task files use the correct retry base class."""
|
|
|
|
def test_process_with_ocr_uses_ocr_base(self):
|
|
from app.tasks.process_with_ocr import process_with_ocr
|
|
|
|
assert isinstance(process_with_ocr, OcrTaskWithRetry)
|
|
|
|
def test_process_with_azure_uses_ocr_base(self):
|
|
from app.tasks.process_with_azure_document_intelligence import (
|
|
process_with_azure_document_intelligence,
|
|
)
|
|
|
|
assert isinstance(process_with_azure_document_intelligence, OcrTaskWithRetry)
|
|
|
|
def test_upload_to_dropbox_uses_upload_base(self):
|
|
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
|
|
|
assert isinstance(upload_to_dropbox, UploadTaskWithRetry)
|
|
|
|
def test_upload_to_s3_uses_upload_base(self):
|
|
from app.tasks.upload_to_s3 import upload_to_s3
|
|
|
|
assert isinstance(upload_to_s3, UploadTaskWithRetry)
|
|
|
|
def test_process_document_uses_base_task(self):
|
|
from app.tasks.process_document import process_document
|
|
|
|
assert isinstance(process_document, BaseTaskWithRetry)
|