perf(db): add indexes on query-hot columns and Redis caching layer

- Add database indexes on FileRecord.created_at, FileRecord.mime_type,
  ProcessingLog.file_id, ProcessingLog.timestamp, and
  FileProcessingStep.status for faster filtering, sorting, and joins.
- Add _ensure_indexes() migration for existing databases.
- Create app/utils/cache.py with fail-open Redis GET/SET/DELETE helpers.
- Cache MIME types dropdown query in files view (120s TTL).
- Optimize batch status query to load only needed columns.

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-01 14:38:28 +00:00
parent a4fafe5c56
commit b79ba81e05
7 changed files with 467 additions and 10 deletions
+7 -3
View File
@@ -8,6 +8,7 @@ from typing import Optional
from fastapi import Depends, HTTPException, Query, Request
from sqlalchemy.orm import Session
from app.utils.cache import cache_get, cache_set
from app.utils.file_queries import apply_status_filter
from app.utils.file_status import get_files_processing_status
from app.views.base import APIRouter, get_db, logger, require_login, templates
@@ -149,9 +150,12 @@ def files_page(
# Calculate pagination info
total_pages = (total_items + per_page - 1) // per_page
# Get unique MIME types for filter dropdown
mime_types = db.query(FileRecord.mime_type).distinct().filter(FileRecord.mime_type.isnot(None)).all()
mime_types = [mt[0] for mt in mime_types if mt[0]]
# Get unique MIME types for filter dropdown (cached)
mime_types = cache_get("mime_types")
if mime_types is None:
raw = db.query(FileRecord.mime_type).distinct().filter(FileRecord.mime_type.isnot(None)).all()
mime_types = [mt[0] for mt in raw if mt[0]]
cache_set("mime_types", mime_types, ttl=120)
# Debug output
logger.info(f"Retrieved {len(files_with_status)} files from database (page {page}/{total_pages})")