perf(db): add indexes on query-hot columns and Redis caching layer

- Add database indexes on FileRecord.created_at, FileRecord.mime_type,
  ProcessingLog.file_id, ProcessingLog.timestamp, and
  FileProcessingStep.status for faster filtering, sorting, and joins.
- Add _ensure_indexes() migration for existing databases.
- Create app/utils/cache.py with fail-open Redis GET/SET/DELETE helpers.
- Cache MIME types dropdown query in files view (120s TTL).
- Optimize batch status query to load only needed columns.

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/database.py

@@ -165,6 +165,36 @@ def _run_schema_migrations(engine: Any) -> None:
             conn.execute(text("CREATE INDEX IF NOT EXISTS ix_saved_searches_user_id ON saved_searches (user_id)"))
         logger.info("Migration complete: 'saved_searches' table created")
 
+    # Migration: Add performance indexes for common query patterns
+    _ensure_indexes(engine, inspector)
+
+
+def _ensure_indexes(engine: Any, inspector: Any) -> None:
+    """
+    Create performance indexes for common query patterns.
+
+    Each ``CREATE INDEX IF NOT EXISTS`` is idempotent and safe to run
+    on every startup.  The indexes target the columns most frequently
+    used in file listing/filtering, status computation and log retrieval.
+    """
+    from sqlalchemy import text
+
+    _PERF_INDEXES = [
+        ("ix_files_created_at", "files", "created_at"),
+        ("ix_files_mime_type", "files", "mime_type"),
+        ("ix_processing_logs_file_id", "processing_logs", "file_id"),
+        ("ix_processing_logs_timestamp", "processing_logs", "timestamp"),
+        ("ix_file_processing_steps_status", "file_processing_steps", "status"),
+    ]
+
+    table_names = inspector.get_table_names()
+    with engine.begin() as conn:
+        for idx_name, table, column in _PERF_INDEXES:
+            if table in table_names:
+                conn.execute(text(f"CREATE INDEX IF NOT EXISTS {idx_name} ON {table} ({column})"))
+
+    logger.info("Performance indexes ensured")
+
 
 def get_db() -> Generator[Session, None, None]:
     """

app/models.py

@@ -46,7 +46,7 @@ class FileRecord(Base):
     file_size = Column(Integer, nullable=False)
 
     # MIME type or extension (optional)
-    mime_type = Column(String)
+    mime_type = Column(String, index=True)
 
     # Deduplication tracking: True if this file is a duplicate of another file
     # When a duplicate is detected, this file record is created but marked as duplicate
@@ -68,7 +68,7 @@ class FileRecord(Base):
     document_title = Column(String, nullable=True)
 
     # Timestamp when we inserted this record
-    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    created_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
 
 
 class FileProcessingStep(Base):
@@ -82,7 +82,7 @@ class FileProcessingStep(Base):
     id = Column(Integer, primary_key=True, index=True)
     file_id = Column(Integer, ForeignKey(_FILES_ID_FK), nullable=False, index=True)
     step_name = Column(String, nullable=False, index=True)  # e.g., "hash_file", "upload_to_dropbox"
-    status = Column(String, nullable=False)  # "pending", "in_progress", "success", "failure", "skipped"
+    status = Column(String, nullable=False, index=True)  # "pending", "in_progress", "success", "failure", "skipped"
     started_at = Column(DateTime(timezone=True), nullable=True)  # When step started
     completed_at = Column(DateTime(timezone=True), nullable=True)  # When step finished (success/failure)
     error_message = Column(Text, nullable=True)  # Error message if status is "failure"
@@ -95,13 +95,13 @@ class FileProcessingStep(Base):
 class ProcessingLog(Base):
     __tablename__ = "processing_logs"
     id = Column(Integer, primary_key=True, index=True)
-    file_id = Column(Integer, ForeignKey(_FILES_ID_FK), nullable=True)  # Optional file association
+    file_id = Column(Integer, ForeignKey(_FILES_ID_FK), nullable=True, index=True)  # Optional file association
     task_id = Column(String, index=True)  # Celery task ID
     step_name = Column(String)  # e.g., "OCR", "convert_to_pdf", "upload_s3"
     status = Column(String)  # "pending", "in_progress", "success", "failure"
     message = Column(String, nullable=True)  # Error text or success note
     detail = Column(Text, nullable=True)  # Verbose worker log output for diagnostics
-    timestamp = Column(DateTime(timezone=True), server_default=func.now())
+    timestamp = Column(DateTime(timezone=True), server_default=func.now(), index=True)
 
 
 class ApplicationSettings(Base):

app/utils/cache.py

@@ -0,0 +1,122 @@
+"""
+Lightweight Redis caching layer for frequently accessed data.
+
+Provides a thin wrapper around Redis GET/SET with JSON serialisation and
+configurable TTLs.  All operations are **fail-open**: if Redis is
+unavailable the caller simply gets a cache miss and falls back to the
+database or other source of truth.
+
+Usage::
+
+    from app.utils.cache import cache_get, cache_set, cache_delete
+
+    # Try cache first
+    value = cache_get("my_key")
+    if value is None:
+        value = expensive_query()
+        cache_set("my_key", value, ttl=300)
+"""
+
+import json
+import logging
+from typing import Any
+
+import redis
+
+logger = logging.getLogger(__name__)
+
+#: Prefix applied to all cache keys to avoid collisions with other Redis users.
+_KEY_PREFIX = "docuelevate:cache:"
+
+#: Module-level Redis client – lazily initialised on first use.
+_redis_client: redis.Redis | None = None
+
+
+def _get_redis() -> redis.Redis | None:
+    """Return a shared Redis client, or *None* if Redis is unreachable."""
+    global _redis_client
+    if _redis_client is not None:
+        return _redis_client
+    try:
+        from app.config import settings
+
+        _redis_client = redis.from_url(settings.redis_url, socket_connect_timeout=2, decode_responses=True)
+        # Quick connectivity check
+        _redis_client.ping()
+        return _redis_client
+    except Exception as exc:
+        logger.debug(f"Redis cache unavailable: {exc}")
+        _redis_client = None
+        return None
+
+
+def cache_get(key: str) -> Any | None:
+    """
+    Retrieve a cached value by *key*.
+
+    Returns the deserialised Python object, or ``None`` on cache miss or
+    Redis error.
+    """
+    client = _get_redis()
+    if client is None:
+        return None
+    try:
+        raw = client.get(f"{_KEY_PREFIX}{key}")
+        if raw is None:
+            return None
+        return json.loads(raw)
+    except Exception as exc:
+        logger.debug(f"Cache get failed for {key}: {exc}")
+        return None
+
+
+def cache_set(key: str, value: Any, ttl: int = 300) -> None:
+    """
+    Store *value* under *key* with a time-to-live of *ttl* seconds.
+
+    Silently ignores errors so callers are never blocked by cache issues.
+    """
+    client = _get_redis()
+    if client is None:
+        return
+    try:
+        client.setex(f"{_KEY_PREFIX}{key}", ttl, json.dumps(value))
+    except Exception as exc:
+        logger.debug(f"Cache set failed for {key}: {exc}")
+
+
+def cache_delete(key: str) -> None:
+    """
+    Remove *key* from the cache.
+
+    Silently ignores errors.
+    """
+    client = _get_redis()
+    if client is None:
+        return
+    try:
+        client.delete(f"{_KEY_PREFIX}{key}")
+    except Exception as exc:
+        logger.debug(f"Cache delete failed for {key}: {exc}")
+
+
+def cache_delete_pattern(pattern: str) -> None:
+    """
+    Remove all keys matching *pattern* (glob-style) from the cache.
+
+    Silently ignores errors.
+    """
+    client = _get_redis()
+    if client is None:
+        return
+    try:
+        full_pattern = f"{_KEY_PREFIX}{pattern}"
+        cursor = 0
+        while True:
+            cursor, keys = client.scan(cursor, match=full_pattern, count=100)
+            if keys:
+                client.delete(*keys)
+            if cursor == 0:
+                break
+    except Exception as exc:
+        logger.debug(f"Cache delete pattern failed for {pattern}: {exc}")

app/utils/file_status.py

@@ -106,9 +106,15 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
     if settings.enable_deduplication:
         REAL_STEPS.add("check_for_duplicates")
 
-    # Get all REAL steps for these files in one query
+    # Get all REAL steps for these files in one query (load only needed columns)
     steps = (
-        db.query(FileProcessingStep)
+        db.query(
+            FileProcessingStep.file_id,
+            FileProcessingStep.step_name,
+            FileProcessingStep.status,
+            FileProcessingStep.updated_at,
+            FileProcessingStep.created_at,
+        )
         .filter(FileProcessingStep.file_id.in_(file_ids), FileProcessingStep.step_name.in_(REAL_STEPS))
         .all()
     )

app/views/files.py

@@ -8,6 +8,7 @@ from typing import Optional
 from fastapi import Depends, HTTPException, Query, Request
 from sqlalchemy.orm import Session
 
+from app.utils.cache import cache_get, cache_set
 from app.utils.file_queries import apply_status_filter
 from app.utils.file_status import get_files_processing_status
 from app.views.base import APIRouter, get_db, logger, require_login, templates
@@ -149,9 +150,12 @@ def files_page(
         # Calculate pagination info
         total_pages = (total_items + per_page - 1) // per_page
 
-        # Get unique MIME types for filter dropdown
-        mime_types = db.query(FileRecord.mime_type).distinct().filter(FileRecord.mime_type.isnot(None)).all()
-        mime_types = [mt[0] for mt in mime_types if mt[0]]
+        # Get unique MIME types for filter dropdown (cached)
+        mime_types = cache_get("mime_types")
+        if mime_types is None:
+            raw = db.query(FileRecord.mime_type).distinct().filter(FileRecord.mime_type.isnot(None)).all()
+            mime_types = [mt[0] for mt in raw if mt[0]]
+            cache_set("mime_types", mime_types, ttl=120)
 
         # Debug output
         logger.info(f"Retrieved {len(files_with_status)} files from database (page {page}/{total_pages})")