feat(tasks): add scheduled batch processing infrastructure (model, migration, tasks, API, view, template)
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -0,0 +1,258 @@
|
||||
"""
|
||||
Admin API endpoints for managing scheduled batch processing jobs.
|
||||
|
||||
All endpoints require admin privileges (checked via session ``is_admin`` flag).
|
||||
|
||||
Available routes:
|
||||
GET /api/admin/scheduled-jobs – list all scheduled jobs
|
||||
PATCH /api/admin/scheduled-jobs/{id} – update schedule / enable-disable
|
||||
POST /api/admin/scheduled-jobs/{id}/run-now – trigger a job immediately
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import ScheduledJob
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/admin/scheduled-jobs", tags=["admin-scheduled-jobs"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Authorisation helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _require_admin(request: Request) -> dict:
|
||||
"""Ensure the caller is an admin; raises HTTP 403 otherwise."""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
AdminUser = Annotated[dict, Depends(_require_admin)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ScheduledJobResponse(BaseModel):
|
||||
"""Read model for a scheduled job."""
|
||||
|
||||
id: int
|
||||
name: str
|
||||
display_name: str
|
||||
description: str | None
|
||||
task_name: str
|
||||
enabled: bool
|
||||
schedule_type: str
|
||||
cron_minute: str
|
||||
cron_hour: str
|
||||
cron_day_of_week: str
|
||||
cron_day_of_month: str
|
||||
cron_month_of_year: str
|
||||
interval_seconds: int | None
|
||||
last_run_at: datetime | None
|
||||
last_run_status: str | None
|
||||
last_run_detail: str | None
|
||||
created_at: datetime | None
|
||||
updated_at: datetime | None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class ScheduledJobUpdate(BaseModel):
|
||||
"""Writable fields for a scheduled job update (all optional)."""
|
||||
|
||||
enabled: bool | None = Field(None, description="Whether the job is active")
|
||||
schedule_type: str | None = Field(None, pattern="^(cron|interval)$", description="'cron' or 'interval'")
|
||||
cron_minute: str | None = Field(None, max_length=50)
|
||||
cron_hour: str | None = Field(None, max_length=50)
|
||||
cron_day_of_week: str | None = Field(None, max_length=50)
|
||||
cron_day_of_month: str | None = Field(None, max_length=50)
|
||||
cron_month_of_year: str | None = Field(None, max_length=50)
|
||||
interval_seconds: int | None = Field(None, ge=60, description="Interval in seconds (min 60)")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default job definitions – seeded into the DB on first startup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DEFAULT_JOBS: list[dict[str, Any]] = [
|
||||
{
|
||||
"name": "process-new-documents",
|
||||
"display_name": "Process New Documents",
|
||||
"description": (
|
||||
"Scans for documents that have been uploaded but never processed "
|
||||
"and queues them through the full processing pipeline. "
|
||||
"Runs hourly by default."
|
||||
),
|
||||
"task_name": "app.tasks.batch_tasks.process_new_documents",
|
||||
"enabled": True,
|
||||
"schedule_type": "cron",
|
||||
"cron_minute": "0",
|
||||
"cron_hour": "*/1",
|
||||
"cron_day_of_week": "*",
|
||||
"cron_day_of_month": "*",
|
||||
"cron_month_of_year": "*",
|
||||
"interval_seconds": None,
|
||||
},
|
||||
{
|
||||
"name": "reprocess-failed-documents",
|
||||
"display_name": "Reprocess Failed Documents",
|
||||
"description": (
|
||||
"Finds documents whose last processing attempt failed and re-queues "
|
||||
"them for reprocessing. Only picks up files that are not currently "
|
||||
"being processed. Runs every 6 hours by default."
|
||||
),
|
||||
"task_name": "app.tasks.batch_tasks.reprocess_failed_documents",
|
||||
"enabled": True,
|
||||
"schedule_type": "cron",
|
||||
"cron_minute": "30",
|
||||
"cron_hour": "*/6",
|
||||
"cron_day_of_week": "*",
|
||||
"cron_day_of_month": "*",
|
||||
"cron_month_of_year": "*",
|
||||
"interval_seconds": None,
|
||||
},
|
||||
{
|
||||
"name": "cleanup-temp-files",
|
||||
"display_name": "Clean Up Temporary Files",
|
||||
"description": (
|
||||
"Removes stale files from the workdir/tmp directory. "
|
||||
"Only files older than 24 hours that are not referenced by any active "
|
||||
"processing job are deleted. Runs daily at 03:30 UTC by default."
|
||||
),
|
||||
"task_name": "app.tasks.batch_tasks.cleanup_temp_files",
|
||||
"enabled": True,
|
||||
"schedule_type": "cron",
|
||||
"cron_minute": "30",
|
||||
"cron_hour": "3",
|
||||
"cron_day_of_week": "*",
|
||||
"cron_day_of_month": "*",
|
||||
"cron_month_of_year": "*",
|
||||
"interval_seconds": None,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def seed_default_scheduled_jobs(db: Session) -> None:
|
||||
"""
|
||||
Insert the built-in scheduled jobs if they do not already exist.
|
||||
|
||||
Called from the FastAPI lifespan handler so the records are available
|
||||
immediately after the first startup.
|
||||
"""
|
||||
for job_data in DEFAULT_JOBS:
|
||||
existing = db.query(ScheduledJob).filter(ScheduledJob.name == job_data["name"]).first()
|
||||
if existing is None:
|
||||
db.add(ScheduledJob(**job_data))
|
||||
try:
|
||||
db.commit()
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.error("Failed to seed default scheduled jobs: %s", exc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("", response_model=list[ScheduledJobResponse])
|
||||
def list_scheduled_jobs(request: Request, db: DbSession, _admin: AdminUser) -> list[ScheduledJobResponse]:
|
||||
"""
|
||||
Return all scheduled jobs ordered by display name.
|
||||
|
||||
Requires admin privileges.
|
||||
"""
|
||||
jobs = db.query(ScheduledJob).order_by(ScheduledJob.display_name).all()
|
||||
return jobs # type: ignore[return-value]
|
||||
|
||||
|
||||
@router.patch("/{job_id}", response_model=ScheduledJobResponse)
|
||||
def update_scheduled_job(
|
||||
job_id: int,
|
||||
payload: ScheduledJobUpdate,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
_admin: AdminUser,
|
||||
) -> ScheduledJobResponse:
|
||||
"""
|
||||
Update schedule configuration or enabled state for a job.
|
||||
|
||||
Only the fields included in the request body are modified.
|
||||
Changes to the Celery Beat schedule take effect after the worker restarts.
|
||||
|
||||
Requires admin privileges.
|
||||
"""
|
||||
job = db.query(ScheduledJob).filter(ScheduledJob.id == job_id).first()
|
||||
if job is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Scheduled job not found")
|
||||
|
||||
update_data = payload.model_dump(exclude_none=True)
|
||||
if not update_data:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No fields to update")
|
||||
|
||||
for field, value in update_data.items():
|
||||
setattr(job, field, value)
|
||||
|
||||
job.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.error("Failed to update scheduled job %s: %s", job_id, exc)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to update scheduled job",
|
||||
) from exc
|
||||
|
||||
logger.info("Admin updated scheduled job %s (id=%s): %s", job.name, job_id, update_data)
|
||||
return job # type: ignore[return-value]
|
||||
|
||||
|
||||
@router.post("/{job_id}/run-now")
|
||||
def run_scheduled_job_now(
|
||||
job_id: int,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
_admin: AdminUser,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Immediately dispatch the Celery task for the given scheduled job.
|
||||
|
||||
The task is sent to the default queue; its result is tracked asynchronously
|
||||
via the ``last_run_at`` / ``last_run_status`` fields updated by the task
|
||||
itself.
|
||||
|
||||
Requires admin privileges.
|
||||
"""
|
||||
job = db.query(ScheduledJob).filter(ScheduledJob.id == job_id).first()
|
||||
if job is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Scheduled job not found")
|
||||
|
||||
from app.celery_app import celery as celery_app
|
||||
|
||||
task = celery_app.send_task(job.task_name)
|
||||
logger.info("Admin triggered scheduled job %s (id=%s) manually, task_id=%s", job.name, job_id, task.id)
|
||||
|
||||
return {
|
||||
"status": "dispatched",
|
||||
"job_id": job_id,
|
||||
"job_name": job.name,
|
||||
"task_id": task.id,
|
||||
}
|
||||
Reference in New Issue
Block a user