copilot-swe-agent[bot]
4897cb6655
- Replace datetime.utcnow() with datetime.now(timezone.utc) in 4 files - Extract duplicate literals to constants in 5 files - models.py: "files.id" → _FILES_ID_FK - upload_to_email.py: "logo.png" → _LOGO_FILENAME - general.py: "%B %d, %Y" → _DATE_DISPLAY_FORMAT - files.py: "File not found" → _FILE_NOT_FOUND - upload_to_google_drive.py: Google token URL → _GOOGLE_TOKEN_URL Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
52 lines
1.7 KiB
Python
52 lines
1.7 KiB
Python
"""
|
|
Periodic task to detect and recover from stalled processing steps.
|
|
|
|
This task runs periodically (every minute by default) to find any processing steps
|
|
that have been stuck in "in_progress" state for too long and mark them as failed.
|
|
"""
|
|
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
|
|
from app.celery_app import celery
|
|
from app.database import SessionLocal
|
|
from app.utils.step_timeout import mark_stalled_steps_as_failed
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@celery.task(name="app.tasks.monitor_stalled_steps.monitor_stalled_steps")
|
|
def monitor_stalled_steps():
|
|
"""
|
|
Periodic task to detect and mark stalled processing steps as failed.
|
|
|
|
This task:
|
|
1. Connects to the database
|
|
2. Finds any in-progress steps that exceeded the timeout
|
|
3. Marks them as failed with a timeout error message
|
|
4. Logs the recovery action
|
|
|
|
This helps prevent files from getting stuck in "pending" state when
|
|
processing crashes or hangs without proper error handling.
|
|
|
|
Scheduled to run every minute via Celery Beat.
|
|
"""
|
|
try:
|
|
with SessionLocal() as db:
|
|
stalled_count = mark_stalled_steps_as_failed(db)
|
|
|
|
if stalled_count > 0:
|
|
logger.warning(
|
|
f"[{datetime.now(timezone.utc).isoformat()}] "
|
|
f"Recovered {stalled_count} stalled step(s). "
|
|
f"Marked as failed due to timeout."
|
|
)
|
|
else:
|
|
logger.debug(f"[{datetime.now(timezone.utc).isoformat()}] No stalled steps found.")
|
|
|
|
return {"recovered": stalled_count}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error in monitor_stalled_steps task: {e}", exc_info=True)
|
|
return {"error": str(e), "recovered": 0}
|