refactor(database): enforce Alembic-only database migrations, deprecate manual schema migrations

- Create alembic.ini and migrations/env.py for full Alembic CLI + programmatic support
- Add Alembic migration 006: detail column on processing_logs
- Add Alembic migration 007: ocr_quality_score column + drop unique filehash index
- Add Alembic migration 008: performance indexes
- Replace _run_schema_migrations() call in init_db() with _run_alembic_upgrade()
- Deprecate _run_schema_migrations() with DeprecationWarning
- Update tests for new Alembic-based approach and deprecation
- Update DatabaseConfiguration.md documentation

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

migrations/env.py

@@ -0,0 +1,107 @@
+"""Alembic environment configuration for DocuElevate.
+
+This module configures Alembic to use the application's database URL
+from ``app.config.settings`` and the SQLAlchemy ``Base.metadata`` so
+that autogenerate can detect model changes.
+
+It also supports receiving an existing connection via
+``config.attributes["connection"]`` for programmatic invocation from
+``app.database.init_db()``, which is essential for in-memory SQLite
+databases used in testing.
+"""
+
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+# Import application Base and models so target_metadata reflects the full schema.
+from app.database import Base
+
+# Ensure all models are imported so Base.metadata is populated.
+from app.models import (  # noqa: F401
+    ApplicationSettings,
+    DocumentMetadata,
+    FileProcessingStep,
+    FileRecord,
+    ProcessingLog,
+    SavedSearch,
+    SettingsAuditLog,
+)
+
+# Alembic Config object – provides access to values in alembic.ini.
+config = context.config
+
+# Set up Python logging from the config file (if present).
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# MetaData object for autogenerate support.
+target_metadata = Base.metadata
+
+
+def _get_url() -> str:
+    """Return the database URL, preferring the application config."""
+    url = config.get_main_option("sqlalchemy.url")
+    if url:
+        return url
+    # Fall back to application settings
+    from app.config import settings
+
+    return settings.database_url
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    Configures the context with just a URL and not an Engine.
+    Calls to ``context.execute()`` emit the given string to the script output.
+    """
+    url = _get_url()
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    Creates an Engine (or reuses a connection passed via
+    ``config.attributes["connection"]``) and associates it with the context.
+    """
+    # If a connection was passed programmatically, reuse it.
+    connectable = config.attributes.get("connection", None)
+
+    if connectable is not None:
+        # Already have a connection — run migrations directly.
+        context.configure(connection=connectable, target_metadata=target_metadata)
+        with context.begin_transaction():
+            context.run_migrations()
+    else:
+        # Create a new engine from configuration.
+        configuration = config.get_section(config.config_ini_section, {})
+        url = _get_url()
+        if url:
+            configuration["sqlalchemy.url"] = url
+        connectable = engine_from_config(
+            configuration,
+            prefix="sqlalchemy.",
+            poolclass=pool.NullPool,
+        )
+
+        with connectable.connect() as connection:
+            context.configure(connection=connection, target_metadata=target_metadata)
+            with context.begin_transaction():
+                context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

migrations/versions/006_add_detail_column.py

@@ -0,0 +1,27 @@
+"""Add detail column to processing_logs table
+
+Revision ID: 006_add_detail_column
+Revises: 005_add_saved_searches
+Create Date: 2026-03-01
+
+"""
+
+from typing import Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "006_add_detail_column"
+down_revision: Union[str, None] = "005_add_saved_searches"
+depends_on: Union[str, None] = None
+
+
+def upgrade() -> None:
+    """Add detail column to processing_logs table for verbose worker log output."""
+    op.add_column("processing_logs", sa.Column("detail", sa.Text(), nullable=True))
+
+
+def downgrade() -> None:
+    """Remove detail column from processing_logs table."""
+    op.drop_column("processing_logs", "detail")

migrations/versions/007_add_ocr_quality_drop_filehash_unique.py

@@ -0,0 +1,42 @@
+"""Add ocr_quality_score column and drop unique filehash index
+
+Revision ID: 007_add_ocr_quality_drop_filehash_unique
+Revises: 006_add_detail_column
+Create Date: 2026-03-01
+
+"""
+
+import logging
+from typing import Union
+
+import sqlalchemy as sa
+from alembic import op
+
+logger = logging.getLogger(__name__)
+
+# revision identifiers, used by Alembic.
+revision: str = "007_add_ocr_quality_drop_filehash_unique"
+down_revision: Union[str, None] = "006_add_detail_column"
+depends_on: Union[str, None] = None
+
+
+def upgrade() -> None:
+    """Add ocr_quality_score column and drop unique constraint on filehash."""
+    # Add ocr_quality_score column for AI-assessed text quality (0-100)
+    op.add_column("files", sa.Column("ocr_quality_score", sa.Integer(), nullable=True))
+
+    # Drop unique index on filehash to allow duplicate file records.
+    # The filehash column retains a non-unique index for lookups.
+    try:
+        op.drop_index("ix_files_filehash", table_name="files")
+    except Exception as exc:
+        logger.debug("Could not drop ix_files_filehash (may not exist): %s", exc)
+    try:
+        op.create_index("ix_files_filehash", "files", ["filehash"], unique=False)
+    except Exception as exc:
+        logger.debug("Could not create ix_files_filehash (may already exist): %s", exc)
+
+
+def downgrade() -> None:
+    """Remove ocr_quality_score column and restore unique filehash index."""
+    op.drop_column("files", "ocr_quality_score")

migrations/versions/008_add_performance_indexes.py

@@ -0,0 +1,43 @@
+"""Add performance indexes for common query patterns
+
+Revision ID: 008_add_performance_indexes
+Revises: 007_add_ocr_quality_drop_filehash_unique
+Create Date: 2026-03-01
+
+"""
+
+from typing import Union
+
+from alembic import op
+from sqlalchemy import text
+
+# revision identifiers, used by Alembic.
+revision: str = "008_add_performance_indexes"
+down_revision: Union[str, None] = "007_add_ocr_quality_drop_filehash_unique"
+depends_on: Union[str, None] = None
+
+
+def upgrade() -> None:
+    """Create performance indexes for file listing, filtering, and log retrieval.
+
+    Uses ``CREATE INDEX IF NOT EXISTS`` to remain idempotent — safe to run
+    even if the indexes were previously created by ``Base.metadata.create_all()``
+    or an earlier manual migration.
+    """
+    conn = op.get_bind()
+    conn.execute(text("CREATE INDEX IF NOT EXISTS ix_files_created_at ON files (created_at)"))
+    conn.execute(text("CREATE INDEX IF NOT EXISTS ix_files_mime_type ON files (mime_type)"))
+    conn.execute(text("CREATE INDEX IF NOT EXISTS ix_processing_logs_file_id ON processing_logs (file_id)"))
+    conn.execute(text("CREATE INDEX IF NOT EXISTS ix_processing_logs_timestamp ON processing_logs (timestamp)"))
+    conn.execute(
+        text("CREATE INDEX IF NOT EXISTS ix_file_processing_steps_status ON file_processing_steps (status)")
+    )
+
+
+def downgrade() -> None:
+    """Drop performance indexes."""
+    op.drop_index("ix_file_processing_steps_status", table_name="file_processing_steps")
+    op.drop_index("ix_processing_logs_timestamp", table_name="processing_logs")
+    op.drop_index("ix_processing_logs_file_id", table_name="processing_logs")
+    op.drop_index("ix_files_mime_type", table_name="files")
+    op.drop_index("ix_files_created_at", table_name="files")