refactor(database): enforce Alembic-only database migrations, deprecate manual schema migrations
- Create alembic.ini and migrations/env.py for full Alembic CLI + programmatic support - Add Alembic migration 006: detail column on processing_logs - Add Alembic migration 007: ocr_quality_score column + drop unique filehash index - Add Alembic migration 008: performance indexes - Replace _run_schema_migrations() call in init_db() with _run_alembic_upgrade() - Deprecate _run_schema_migrations() with DeprecationWarning - Update tests for new Alembic-based approach and deprecation - Update DatabaseConfiguration.md documentation Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
+73
@@ -0,0 +1,73 @@
|
|||||||
|
# Alembic Configuration File
|
||||||
|
# Used for managing database schema migrations in DocuElevate.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# alembic upgrade head # Apply all pending migrations
|
||||||
|
# alembic current # Show current revision
|
||||||
|
# alembic history --verbose # Show migration history
|
||||||
|
# alembic downgrade -1 # Roll back one migration
|
||||||
|
# alembic revision --autogenerate -m "description" # Create new migration
|
||||||
|
|
||||||
|
[alembic]
|
||||||
|
# Path to migration scripts
|
||||||
|
script_location = migrations
|
||||||
|
|
||||||
|
# Template used to generate migration file names
|
||||||
|
file_template = %%(rev)s_%%(slug)s
|
||||||
|
|
||||||
|
# Timezone for migration file timestamps (uses UTC by default)
|
||||||
|
# timezone =
|
||||||
|
|
||||||
|
# Maximum length of characters for autogenerate revision names
|
||||||
|
# truncate_slug_length = 40
|
||||||
|
|
||||||
|
# Set to 'true' to run environment during 'revision' command
|
||||||
|
# revision_environment = false
|
||||||
|
|
||||||
|
# Set to 'true' to allow .pyc or .pyo files for migration scripts
|
||||||
|
# sourceless = false
|
||||||
|
|
||||||
|
# Version path separator; default is "os" which uses os.pathsep
|
||||||
|
# version_path_separator = os
|
||||||
|
|
||||||
|
# Output encoding for revision files
|
||||||
|
# output_encoding = utf-8
|
||||||
|
|
||||||
|
# The database URL is loaded from app.config.settings.database_url
|
||||||
|
# in migrations/env.py, not from this file.
|
||||||
|
sqlalchemy.url =
|
||||||
|
|
||||||
|
|
||||||
|
[loggers]
|
||||||
|
keys = root,sqlalchemy,alembic
|
||||||
|
|
||||||
|
[handlers]
|
||||||
|
keys = console
|
||||||
|
|
||||||
|
[formatters]
|
||||||
|
keys = generic
|
||||||
|
|
||||||
|
[logger_root]
|
||||||
|
level = WARN
|
||||||
|
handlers = console
|
||||||
|
qualname =
|
||||||
|
|
||||||
|
[logger_sqlalchemy]
|
||||||
|
level = WARN
|
||||||
|
handlers =
|
||||||
|
qualname = sqlalchemy.engine
|
||||||
|
|
||||||
|
[logger_alembic]
|
||||||
|
level = INFO
|
||||||
|
handlers =
|
||||||
|
qualname = alembic
|
||||||
|
|
||||||
|
[handler_console]
|
||||||
|
class = StreamHandler
|
||||||
|
args = (sys.stderr,)
|
||||||
|
level = NOTSET
|
||||||
|
formatter = generic
|
||||||
|
|
||||||
|
[formatter_generic]
|
||||||
|
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||||
|
datefmt = %H:%M:%S
|
||||||
+66
-6
@@ -2,7 +2,9 @@
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import warnings
|
||||||
from collections.abc import Generator
|
from collections.abc import Generator
|
||||||
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import create_engine, exc
|
from sqlalchemy import create_engine, exc
|
||||||
@@ -24,8 +26,8 @@ SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
|||||||
def init_db() -> None:
|
def init_db() -> None:
|
||||||
"""
|
"""
|
||||||
Ensures the SQLite database file and its parent directory exist (if using sqlite).
|
Ensures the SQLite database file and its parent directory exist (if using sqlite).
|
||||||
Then runs Base.metadata.create_all(bind=engine) to initialize tables.
|
Then runs Base.metadata.create_all(bind=engine) to initialize tables and
|
||||||
Logs a message if a new SQLite DB file is created.
|
applies any pending Alembic migrations.
|
||||||
"""
|
"""
|
||||||
# 1. Parse the DB URL to see if it's sqlite
|
# 1. Parse the DB URL to see if it's sqlite
|
||||||
url = make_url(DB_URL)
|
url = make_url(DB_URL)
|
||||||
@@ -50,18 +52,76 @@ def init_db() -> None:
|
|||||||
Base.metadata.create_all(bind=engine)
|
Base.metadata.create_all(bind=engine)
|
||||||
logger.info("Database initialization complete (tables created if not exist).")
|
logger.info("Database initialization complete (tables created if not exist).")
|
||||||
|
|
||||||
# 6. Run lightweight schema migrations for existing databases
|
# 6. Run Alembic migrations for existing databases
|
||||||
_run_schema_migrations(engine)
|
_run_alembic_upgrade(engine)
|
||||||
except exc.SQLAlchemyError as e:
|
except exc.SQLAlchemyError as e:
|
||||||
logger.error(f"Error initializing database: {e}")
|
logger.error(f"Error initializing database: {e}")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
def _run_schema_migrations(engine: Any) -> None:
|
def _run_alembic_upgrade(engine: Any) -> None:
|
||||||
|
"""Run Alembic migrations programmatically to apply pending schema changes.
|
||||||
|
|
||||||
|
For fresh databases (created via ``Base.metadata.create_all()``), the
|
||||||
|
Alembic version is stamped to ``head`` because all tables already exist.
|
||||||
|
For existing databases with Alembic tracking, any pending migrations are
|
||||||
|
applied via ``alembic upgrade head``.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
engine: The SQLAlchemy engine connected to the target database.
|
||||||
"""
|
"""
|
||||||
Apply lightweight schema migrations for columns added after the initial release.
|
from alembic import command
|
||||||
|
from alembic.config import Config
|
||||||
|
from sqlalchemy import inspect
|
||||||
|
|
||||||
|
inspector = inspect(engine)
|
||||||
|
table_names = inspector.get_table_names()
|
||||||
|
|
||||||
|
# Locate the migrations directory relative to this file
|
||||||
|
migrations_dir = str(Path(__file__).resolve().parent.parent / "migrations")
|
||||||
|
|
||||||
|
# Build an Alembic Config that points at our migration scripts.
|
||||||
|
# The sqlalchemy.url is intentionally left empty because we pass the
|
||||||
|
# live connection via config.attributes["connection"] below.
|
||||||
|
alembic_cfg = Config()
|
||||||
|
alembic_cfg.set_main_option("script_location", migrations_dir)
|
||||||
|
alembic_cfg.set_main_option("sqlalchemy.url", "")
|
||||||
|
|
||||||
|
with engine.begin() as connection:
|
||||||
|
alembic_cfg.attributes["connection"] = connection
|
||||||
|
|
||||||
|
if "alembic_version" not in table_names:
|
||||||
|
# Fresh database or one that predates Alembic tracking.
|
||||||
|
# Base.metadata.create_all() already created everything, so
|
||||||
|
# stamp the current version to head (no migrations need to run).
|
||||||
|
logger.info("No Alembic version table found — stamping database to latest revision.")
|
||||||
|
command.stamp(alembic_cfg, "head")
|
||||||
|
else:
|
||||||
|
# Existing database with Alembic version tracking — apply pending migrations.
|
||||||
|
logger.info("Running pending Alembic migrations…")
|
||||||
|
command.upgrade(alembic_cfg, "head")
|
||||||
|
logger.info("Alembic migration check complete.")
|
||||||
|
|
||||||
|
|
||||||
|
def _run_schema_migrations(engine: Any) -> None:
|
||||||
|
"""Apply lightweight schema migrations for columns added after the initial release.
|
||||||
|
|
||||||
|
.. deprecated::
|
||||||
|
This function is deprecated and will be removed in a future release.
|
||||||
|
All schema migrations are now managed exclusively through Alembic.
|
||||||
|
Run ``alembic upgrade head`` (or let ``init_db()`` handle it
|
||||||
|
automatically) instead of calling this function directly.
|
||||||
|
|
||||||
Each migration is idempotent and safe to run multiple times.
|
Each migration is idempotent and safe to run multiple times.
|
||||||
"""
|
"""
|
||||||
|
warnings.warn(
|
||||||
|
"_run_schema_migrations() is deprecated. "
|
||||||
|
"All schema changes are now managed by Alembic migrations. "
|
||||||
|
"Use 'alembic upgrade head' or init_db() instead.",
|
||||||
|
DeprecationWarning,
|
||||||
|
stacklevel=2,
|
||||||
|
)
|
||||||
|
|
||||||
from sqlalchemy import inspect, text
|
from sqlalchemy import inspect, text
|
||||||
|
|
||||||
inspector = inspect(engine)
|
inspector = inspect(engine)
|
||||||
|
|||||||
@@ -163,7 +163,15 @@ DATABASE_URL=mysql+pymysql://user:password@host:3306/docuelevate?charset=utf8mb4
|
|||||||
|
|
||||||
## Schema Migrations with Alembic
|
## Schema Migrations with Alembic
|
||||||
|
|
||||||
DocuElevate uses Alembic to manage all database schema changes.
|
DocuElevate uses Alembic **exclusively** to manage all database schema changes. On application startup, `init_db()` automatically applies any pending Alembic migrations, so the database schema is always kept in sync with the running code.
|
||||||
|
|
||||||
|
> **Note:** Prior to this change, a manual `_run_schema_migrations()` helper in `app/database.py` applied schema changes outside of Alembic. That function is now **deprecated** and will be removed in a future release. All schema changes are tracked as Alembic revisions in `migrations/versions/`.
|
||||||
|
|
||||||
|
### How It Works
|
||||||
|
|
||||||
|
1. **Fresh databases** — `Base.metadata.create_all()` creates all tables from the SQLAlchemy models, then Alembic stamps the version to `head` (no migrations need to run).
|
||||||
|
2. **Existing databases** — `alembic upgrade head` applies any pending migration scripts.
|
||||||
|
3. **CLI usage** — You can still run `alembic upgrade head` manually or in CI/CD before the application starts.
|
||||||
|
|
||||||
### Apply Migrations
|
### Apply Migrations
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
"""Alembic environment configuration for DocuElevate.
|
||||||
|
|
||||||
|
This module configures Alembic to use the application's database URL
|
||||||
|
from ``app.config.settings`` and the SQLAlchemy ``Base.metadata`` so
|
||||||
|
that autogenerate can detect model changes.
|
||||||
|
|
||||||
|
It also supports receiving an existing connection via
|
||||||
|
``config.attributes["connection"]`` for programmatic invocation from
|
||||||
|
``app.database.init_db()``, which is essential for in-memory SQLite
|
||||||
|
databases used in testing.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from logging.config import fileConfig
|
||||||
|
|
||||||
|
from alembic import context
|
||||||
|
from sqlalchemy import engine_from_config, pool
|
||||||
|
|
||||||
|
# Import application Base and models so target_metadata reflects the full schema.
|
||||||
|
from app.database import Base
|
||||||
|
|
||||||
|
# Ensure all models are imported so Base.metadata is populated.
|
||||||
|
from app.models import ( # noqa: F401
|
||||||
|
ApplicationSettings,
|
||||||
|
DocumentMetadata,
|
||||||
|
FileProcessingStep,
|
||||||
|
FileRecord,
|
||||||
|
ProcessingLog,
|
||||||
|
SavedSearch,
|
||||||
|
SettingsAuditLog,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Alembic Config object – provides access to values in alembic.ini.
|
||||||
|
config = context.config
|
||||||
|
|
||||||
|
# Set up Python logging from the config file (if present).
|
||||||
|
if config.config_file_name is not None:
|
||||||
|
fileConfig(config.config_file_name)
|
||||||
|
|
||||||
|
# MetaData object for autogenerate support.
|
||||||
|
target_metadata = Base.metadata
|
||||||
|
|
||||||
|
|
||||||
|
def _get_url() -> str:
|
||||||
|
"""Return the database URL, preferring the application config."""
|
||||||
|
url = config.get_main_option("sqlalchemy.url")
|
||||||
|
if url:
|
||||||
|
return url
|
||||||
|
# Fall back to application settings
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
|
return settings.database_url
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_offline() -> None:
|
||||||
|
"""Run migrations in 'offline' mode.
|
||||||
|
|
||||||
|
Configures the context with just a URL and not an Engine.
|
||||||
|
Calls to ``context.execute()`` emit the given string to the script output.
|
||||||
|
"""
|
||||||
|
url = _get_url()
|
||||||
|
context.configure(
|
||||||
|
url=url,
|
||||||
|
target_metadata=target_metadata,
|
||||||
|
literal_binds=True,
|
||||||
|
dialect_opts={"paramstyle": "named"},
|
||||||
|
)
|
||||||
|
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_online() -> None:
|
||||||
|
"""Run migrations in 'online' mode.
|
||||||
|
|
||||||
|
Creates an Engine (or reuses a connection passed via
|
||||||
|
``config.attributes["connection"]``) and associates it with the context.
|
||||||
|
"""
|
||||||
|
# If a connection was passed programmatically, reuse it.
|
||||||
|
connectable = config.attributes.get("connection", None)
|
||||||
|
|
||||||
|
if connectable is not None:
|
||||||
|
# Already have a connection — run migrations directly.
|
||||||
|
context.configure(connection=connectable, target_metadata=target_metadata)
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
else:
|
||||||
|
# Create a new engine from configuration.
|
||||||
|
configuration = config.get_section(config.config_ini_section, {})
|
||||||
|
url = _get_url()
|
||||||
|
if url:
|
||||||
|
configuration["sqlalchemy.url"] = url
|
||||||
|
connectable = engine_from_config(
|
||||||
|
configuration,
|
||||||
|
prefix="sqlalchemy.",
|
||||||
|
poolclass=pool.NullPool,
|
||||||
|
)
|
||||||
|
|
||||||
|
with connectable.connect() as connection:
|
||||||
|
context.configure(connection=connection, target_metadata=target_metadata)
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
|
||||||
|
if context.is_offline_mode():
|
||||||
|
run_migrations_offline()
|
||||||
|
else:
|
||||||
|
run_migrations_online()
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
"""Add detail column to processing_logs table
|
||||||
|
|
||||||
|
Revision ID: 006_add_detail_column
|
||||||
|
Revises: 005_add_saved_searches
|
||||||
|
Create Date: 2026-03-01
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = "006_add_detail_column"
|
||||||
|
down_revision: Union[str, None] = "005_add_saved_searches"
|
||||||
|
depends_on: Union[str, None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Add detail column to processing_logs table for verbose worker log output."""
|
||||||
|
op.add_column("processing_logs", sa.Column("detail", sa.Text(), nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Remove detail column from processing_logs table."""
|
||||||
|
op.drop_column("processing_logs", "detail")
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
"""Add ocr_quality_score column and drop unique filehash index
|
||||||
|
|
||||||
|
Revision ID: 007_add_ocr_quality_drop_filehash_unique
|
||||||
|
Revises: 006_add_detail_column
|
||||||
|
Create Date: 2026-03-01
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = "007_add_ocr_quality_drop_filehash_unique"
|
||||||
|
down_revision: Union[str, None] = "006_add_detail_column"
|
||||||
|
depends_on: Union[str, None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Add ocr_quality_score column and drop unique constraint on filehash."""
|
||||||
|
# Add ocr_quality_score column for AI-assessed text quality (0-100)
|
||||||
|
op.add_column("files", sa.Column("ocr_quality_score", sa.Integer(), nullable=True))
|
||||||
|
|
||||||
|
# Drop unique index on filehash to allow duplicate file records.
|
||||||
|
# The filehash column retains a non-unique index for lookups.
|
||||||
|
try:
|
||||||
|
op.drop_index("ix_files_filehash", table_name="files")
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Could not drop ix_files_filehash (may not exist): %s", exc)
|
||||||
|
try:
|
||||||
|
op.create_index("ix_files_filehash", "files", ["filehash"], unique=False)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Could not create ix_files_filehash (may already exist): %s", exc)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Remove ocr_quality_score column and restore unique filehash index."""
|
||||||
|
op.drop_column("files", "ocr_quality_score")
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
"""Add performance indexes for common query patterns
|
||||||
|
|
||||||
|
Revision ID: 008_add_performance_indexes
|
||||||
|
Revises: 007_add_ocr_quality_drop_filehash_unique
|
||||||
|
Create Date: 2026-03-01
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = "008_add_performance_indexes"
|
||||||
|
down_revision: Union[str, None] = "007_add_ocr_quality_drop_filehash_unique"
|
||||||
|
depends_on: Union[str, None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Create performance indexes for file listing, filtering, and log retrieval.
|
||||||
|
|
||||||
|
Uses ``CREATE INDEX IF NOT EXISTS`` to remain idempotent — safe to run
|
||||||
|
even if the indexes were previously created by ``Base.metadata.create_all()``
|
||||||
|
or an earlier manual migration.
|
||||||
|
"""
|
||||||
|
conn = op.get_bind()
|
||||||
|
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_files_created_at ON files (created_at)"))
|
||||||
|
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_files_mime_type ON files (mime_type)"))
|
||||||
|
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_processing_logs_file_id ON processing_logs (file_id)"))
|
||||||
|
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_processing_logs_timestamp ON processing_logs (timestamp)"))
|
||||||
|
conn.execute(
|
||||||
|
text("CREATE INDEX IF NOT EXISTS ix_file_processing_steps_status ON file_processing_steps (status)")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Drop performance indexes."""
|
||||||
|
op.drop_index("ix_file_processing_steps_status", table_name="file_processing_steps")
|
||||||
|
op.drop_index("ix_processing_logs_timestamp", table_name="processing_logs")
|
||||||
|
op.drop_index("ix_processing_logs_file_id", table_name="processing_logs")
|
||||||
|
op.drop_index("ix_files_mime_type", table_name="files")
|
||||||
|
op.drop_index("ix_files_created_at", table_name="files")
|
||||||
+194
-107
@@ -1,5 +1,6 @@
|
|||||||
"""Tests for app/database.py module."""
|
"""Tests for app/database.py module."""
|
||||||
|
|
||||||
|
import warnings
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -110,8 +111,28 @@ class TestSchemaMigrations:
|
|||||||
|
|
||||||
assert log.detail is None
|
assert log.detail is None
|
||||||
|
|
||||||
def test_migration_adds_detail_column(self, tmp_path):
|
def test_deprecated_run_schema_migrations_warns(self, tmp_path):
|
||||||
"""Test that _run_schema_migrations adds detail column to existing tables."""
|
"""Test that _run_schema_migrations emits a DeprecationWarning."""
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
|
||||||
|
from app.database import _run_schema_migrations
|
||||||
|
|
||||||
|
db_path = str(tmp_path / "deprecation_test.db")
|
||||||
|
engine = create_engine(f"sqlite:///{db_path}")
|
||||||
|
|
||||||
|
with warnings.catch_warnings(record=True) as w:
|
||||||
|
warnings.simplefilter("always")
|
||||||
|
_run_schema_migrations(engine)
|
||||||
|
|
||||||
|
assert len(w) == 1
|
||||||
|
assert issubclass(w[0].category, DeprecationWarning)
|
||||||
|
assert "deprecated" in str(w[0].message).lower()
|
||||||
|
assert "Alembic" in str(w[0].message)
|
||||||
|
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
def test_deprecated_migration_still_adds_detail_column(self, tmp_path):
|
||||||
|
"""Test that deprecated _run_schema_migrations still works for legacy callers."""
|
||||||
from sqlalchemy import create_engine, text
|
from sqlalchemy import create_engine, text
|
||||||
|
|
||||||
from app.database import _run_schema_migrations
|
from app.database import _run_schema_migrations
|
||||||
@@ -133,8 +154,10 @@ class TestSchemaMigrations:
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Run migrations
|
# Run deprecated migrations (suppress warning for test clarity)
|
||||||
_run_schema_migrations(engine)
|
with warnings.catch_warnings():
|
||||||
|
warnings.simplefilter("ignore", DeprecationWarning)
|
||||||
|
_run_schema_migrations(engine)
|
||||||
|
|
||||||
# Verify detail column was added
|
# Verify detail column was added
|
||||||
from sqlalchemy import inspect
|
from sqlalchemy import inspect
|
||||||
@@ -145,8 +168,8 @@ class TestSchemaMigrations:
|
|||||||
|
|
||||||
engine.dispose()
|
engine.dispose()
|
||||||
|
|
||||||
def test_migration_adds_file_path_columns(self, tmp_path):
|
def test_deprecated_migration_adds_file_path_columns(self, tmp_path):
|
||||||
"""Test that _run_schema_migrations adds file path columns to files table."""
|
"""Test that deprecated _run_schema_migrations adds file path columns to files table."""
|
||||||
from sqlalchemy import create_engine, text
|
from sqlalchemy import create_engine, text
|
||||||
|
|
||||||
from app.database import _run_schema_migrations
|
from app.database import _run_schema_migrations
|
||||||
@@ -165,8 +188,10 @@ class TestSchemaMigrations:
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Run migrations
|
# Run deprecated migrations
|
||||||
_run_schema_migrations(engine)
|
with warnings.catch_warnings():
|
||||||
|
warnings.simplefilter("ignore", DeprecationWarning)
|
||||||
|
_run_schema_migrations(engine)
|
||||||
|
|
||||||
# Verify columns were added with correct types
|
# Verify columns were added with correct types
|
||||||
from sqlalchemy import inspect
|
from sqlalchemy import inspect
|
||||||
@@ -192,92 +217,8 @@ class TestSchemaMigrations:
|
|||||||
|
|
||||||
engine.dispose()
|
engine.dispose()
|
||||||
|
|
||||||
def test_migration_adds_search_fields(self, tmp_path):
|
def test_deprecated_migration_handles_missing_tables_gracefully(self, tmp_path):
|
||||||
"""Test that _run_schema_migrations adds ocr_text, ai_metadata, document_title to files table."""
|
"""Test that deprecated migrations don't fail when tables don't exist."""
|
||||||
from sqlalchemy import create_engine, text
|
|
||||||
|
|
||||||
from app.database import _run_schema_migrations
|
|
||||||
|
|
||||||
# Create a database with the old schema (no search fields)
|
|
||||||
db_path = str(tmp_path / "migration_search_fields_test.db")
|
|
||||||
engine = create_engine(f"sqlite:///{db_path}")
|
|
||||||
with engine.begin() as conn:
|
|
||||||
conn.execute(
|
|
||||||
text(
|
|
||||||
"CREATE TABLE files ("
|
|
||||||
"id INTEGER PRIMARY KEY, "
|
|
||||||
"filename VARCHAR, "
|
|
||||||
"filehash VARCHAR, "
|
|
||||||
"original_file_path VARCHAR, "
|
|
||||||
"processed_file_path VARCHAR, "
|
|
||||||
"is_duplicate BOOLEAN DEFAULT FALSE NOT NULL, "
|
|
||||||
"duplicate_of_id INTEGER)"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
# Run migrations
|
|
||||||
_run_schema_migrations(engine)
|
|
||||||
|
|
||||||
# Verify search columns were added
|
|
||||||
from sqlalchemy import inspect
|
|
||||||
|
|
||||||
inspector = inspect(engine)
|
|
||||||
columns = {col["name"]: col for col in inspector.get_columns("files")}
|
|
||||||
|
|
||||||
assert "ocr_text" in columns
|
|
||||||
assert "ai_metadata" in columns
|
|
||||||
assert "document_title" in columns
|
|
||||||
|
|
||||||
engine.dispose()
|
|
||||||
|
|
||||||
def test_migration_drops_unique_filehash_index(self, tmp_path):
|
|
||||||
"""Test that _run_schema_migrations drops unique index on filehash."""
|
|
||||||
from sqlalchemy import create_engine, text
|
|
||||||
|
|
||||||
from app.database import _run_schema_migrations
|
|
||||||
|
|
||||||
# Create a database with unique index on filehash
|
|
||||||
db_path = str(tmp_path / "migration_index_test.db")
|
|
||||||
engine = create_engine(f"sqlite:///{db_path}")
|
|
||||||
with engine.begin() as conn:
|
|
||||||
conn.execute(
|
|
||||||
text(
|
|
||||||
"CREATE TABLE files ("
|
|
||||||
"id INTEGER PRIMARY KEY, "
|
|
||||||
"filename VARCHAR, "
|
|
||||||
"filehash VARCHAR, "
|
|
||||||
"upload_date DATETIME, "
|
|
||||||
"original_file_path VARCHAR, "
|
|
||||||
"processed_file_path VARCHAR, "
|
|
||||||
"is_duplicate BOOLEAN DEFAULT FALSE NOT NULL, "
|
|
||||||
"duplicate_of_id INTEGER)"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
conn.execute(text("CREATE UNIQUE INDEX idx_filehash_unique ON files (filehash)"))
|
|
||||||
|
|
||||||
# Verify unique index exists before migration
|
|
||||||
from sqlalchemy import inspect
|
|
||||||
|
|
||||||
inspector = inspect(engine)
|
|
||||||
indexes_before = inspector.get_indexes("files")
|
|
||||||
unique_indexes_before = [idx for idx in indexes_before if idx.get("unique")]
|
|
||||||
assert len(unique_indexes_before) > 0
|
|
||||||
|
|
||||||
# Run migrations
|
|
||||||
_run_schema_migrations(engine)
|
|
||||||
|
|
||||||
# Verify unique index was removed
|
|
||||||
inspector = inspect(engine)
|
|
||||||
indexes_after = inspector.get_indexes("files")
|
|
||||||
unique_filehash_indexes_after = [
|
|
||||||
idx for idx in indexes_after if idx.get("unique") and "filehash" in idx.get("column_names", [])
|
|
||||||
]
|
|
||||||
assert len(unique_filehash_indexes_after) == 0
|
|
||||||
|
|
||||||
engine.dispose()
|
|
||||||
|
|
||||||
def test_migration_handles_missing_tables_gracefully(self, tmp_path):
|
|
||||||
"""Test that migrations don't fail when tables don't exist."""
|
|
||||||
from sqlalchemy import create_engine
|
from sqlalchemy import create_engine
|
||||||
|
|
||||||
from app.database import _run_schema_migrations
|
from app.database import _run_schema_migrations
|
||||||
@@ -287,12 +228,14 @@ class TestSchemaMigrations:
|
|||||||
engine = create_engine(f"sqlite:///{db_path}")
|
engine = create_engine(f"sqlite:///{db_path}")
|
||||||
|
|
||||||
# Run migrations - should not raise any errors
|
# Run migrations - should not raise any errors
|
||||||
_run_schema_migrations(engine)
|
with warnings.catch_warnings():
|
||||||
|
warnings.simplefilter("ignore", DeprecationWarning)
|
||||||
|
_run_schema_migrations(engine)
|
||||||
|
|
||||||
engine.dispose()
|
engine.dispose()
|
||||||
|
|
||||||
def test_migration_is_idempotent(self, tmp_path):
|
def test_deprecated_migration_is_idempotent(self, tmp_path):
|
||||||
"""Test that running migrations multiple times is safe."""
|
"""Test that running deprecated migrations multiple times is safe."""
|
||||||
from sqlalchemy import create_engine, text
|
from sqlalchemy import create_engine, text
|
||||||
|
|
||||||
from app.database import _run_schema_migrations
|
from app.database import _run_schema_migrations
|
||||||
@@ -323,10 +266,12 @@ class TestSchemaMigrations:
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Run migrations multiple times
|
# Run deprecated migrations multiple times
|
||||||
_run_schema_migrations(engine)
|
with warnings.catch_warnings():
|
||||||
_run_schema_migrations(engine)
|
warnings.simplefilter("ignore", DeprecationWarning)
|
||||||
_run_schema_migrations(engine)
|
_run_schema_migrations(engine)
|
||||||
|
_run_schema_migrations(engine)
|
||||||
|
_run_schema_migrations(engine)
|
||||||
|
|
||||||
# Verify all columns exist and no errors occurred
|
# Verify all columns exist and no errors occurred
|
||||||
from sqlalchemy import inspect
|
from sqlalchemy import inspect
|
||||||
@@ -372,7 +317,11 @@ class TestInitDbErrors:
|
|||||||
|
|
||||||
@pytest.mark.unit
|
@pytest.mark.unit
|
||||||
class TestMultiVersionMigrations:
|
class TestMultiVersionMigrations:
|
||||||
"""Test migration scenarios from various database versions."""
|
"""Test deprecated migration scenarios from various database versions.
|
||||||
|
|
||||||
|
These tests verify the deprecated _run_schema_migrations function still
|
||||||
|
works for legacy callers. New schema changes should use Alembic exclusively.
|
||||||
|
"""
|
||||||
|
|
||||||
def test_migration_from_v1_to_v2_processing_logs(self, tmp_path):
|
def test_migration_from_v1_to_v2_processing_logs(self, tmp_path):
|
||||||
"""Test migration from v1 (no detail column) to v2 (with detail)."""
|
"""Test migration from v1 (no detail column) to v2 (with detail)."""
|
||||||
@@ -405,7 +354,9 @@ class TestMultiVersionMigrations:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Run migration to v2
|
# Run migration to v2
|
||||||
_run_schema_migrations(engine)
|
with warnings.catch_warnings():
|
||||||
|
warnings.simplefilter("ignore", DeprecationWarning)
|
||||||
|
_run_schema_migrations(engine)
|
||||||
|
|
||||||
# Verify detail column exists and old data is preserved
|
# Verify detail column exists and old data is preserved
|
||||||
inspector = inspect(engine)
|
inspector = inspect(engine)
|
||||||
@@ -445,7 +396,9 @@ class TestMultiVersionMigrations:
|
|||||||
conn.execute(text("INSERT INTO files (filename, filehash) VALUES ('test.pdf', 'abc123')"))
|
conn.execute(text("INSERT INTO files (filename, filehash) VALUES ('test.pdf', 'abc123')"))
|
||||||
|
|
||||||
# Run migration to v3 (adds path columns and dedup columns)
|
# Run migration to v3 (adds path columns and dedup columns)
|
||||||
_run_schema_migrations(engine)
|
with warnings.catch_warnings():
|
||||||
|
warnings.simplefilter("ignore", DeprecationWarning)
|
||||||
|
_run_schema_migrations(engine)
|
||||||
|
|
||||||
# Verify all new columns exist
|
# Verify all new columns exist
|
||||||
inspector = inspect(engine)
|
inspector = inspect(engine)
|
||||||
@@ -489,7 +442,9 @@ class TestMultiVersionMigrations:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Run migration - should not error even though there's no index to drop
|
# Run migration - should not error even though there's no index to drop
|
||||||
_run_schema_migrations(engine)
|
with warnings.catch_warnings():
|
||||||
|
warnings.simplefilter("ignore", DeprecationWarning)
|
||||||
|
_run_schema_migrations(engine)
|
||||||
|
|
||||||
# Should complete without error
|
# Should complete without error
|
||||||
engine.dispose()
|
engine.dispose()
|
||||||
@@ -529,7 +484,9 @@ class TestMultiVersionMigrations:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Run migration - should add missing columns only
|
# Run migration - should add missing columns only
|
||||||
_run_schema_migrations(engine)
|
with warnings.catch_warnings():
|
||||||
|
warnings.simplefilter("ignore", DeprecationWarning)
|
||||||
|
_run_schema_migrations(engine)
|
||||||
|
|
||||||
# Verify all columns exist now
|
# Verify all columns exist now
|
||||||
inspector = inspect(engine)
|
inspector = inspect(engine)
|
||||||
@@ -570,9 +527,139 @@ class TestMultiVersionMigrations:
|
|||||||
# Run migration - should handle the exception path for index operations
|
# Run migration - should handle the exception path for index operations
|
||||||
# (when get_indexes might have issues)
|
# (when get_indexes might have issues)
|
||||||
try:
|
try:
|
||||||
_run_schema_migrations(engine)
|
with warnings.catch_warnings():
|
||||||
|
warnings.simplefilter("ignore", DeprecationWarning)
|
||||||
|
_run_schema_migrations(engine)
|
||||||
# Should complete without raising
|
# Should complete without raising
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
pytest.fail(f"Migration should handle exceptions gracefully: {e}")
|
pytest.fail(f"Migration should handle exceptions gracefully: {e}")
|
||||||
|
|
||||||
engine.dispose()
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
class TestAlembicUpgrade:
|
||||||
|
"""Tests for Alembic-based migration management."""
|
||||||
|
|
||||||
|
def test_alembic_upgrade_stamps_fresh_database(self, tmp_path):
|
||||||
|
"""Test that _run_alembic_upgrade stamps a fresh database to head."""
|
||||||
|
from sqlalchemy import create_engine, inspect, text
|
||||||
|
|
||||||
|
from app.database import Base, _run_alembic_upgrade
|
||||||
|
|
||||||
|
db_path = str(tmp_path / "fresh_alembic.db")
|
||||||
|
engine = create_engine(f"sqlite:///{db_path}")
|
||||||
|
|
||||||
|
# Create all tables (simulates Base.metadata.create_all)
|
||||||
|
Base.metadata.create_all(bind=engine)
|
||||||
|
|
||||||
|
# Run Alembic upgrade — should stamp to head (not run migrations)
|
||||||
|
_run_alembic_upgrade(engine)
|
||||||
|
|
||||||
|
# Verify alembic_version table exists and has a revision
|
||||||
|
inspector = inspect(engine)
|
||||||
|
assert "alembic_version" in inspector.get_table_names()
|
||||||
|
|
||||||
|
with engine.connect() as conn:
|
||||||
|
result = conn.execute(text("SELECT version_num FROM alembic_version"))
|
||||||
|
row = result.fetchone()
|
||||||
|
assert row is not None
|
||||||
|
# Should be stamped to the latest revision
|
||||||
|
assert row[0] == "008_add_performance_indexes"
|
||||||
|
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
def test_alembic_upgrade_applies_pending_migrations(self, tmp_path):
|
||||||
|
"""Test that _run_alembic_upgrade applies pending migrations to a tracked DB."""
|
||||||
|
from sqlalchemy import create_engine, text
|
||||||
|
|
||||||
|
from app.database import Base, _run_alembic_upgrade
|
||||||
|
|
||||||
|
db_path = str(tmp_path / "tracked_alembic.db")
|
||||||
|
engine = create_engine(f"sqlite:///{db_path}")
|
||||||
|
|
||||||
|
# Create all tables
|
||||||
|
Base.metadata.create_all(bind=engine)
|
||||||
|
|
||||||
|
# Manually create alembic_version table and set to an earlier revision
|
||||||
|
with engine.begin() as conn:
|
||||||
|
conn.execute(text("CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL)"))
|
||||||
|
conn.execute(text("INSERT INTO alembic_version (version_num) VALUES ('008_add_performance_indexes')"))
|
||||||
|
|
||||||
|
# Run Alembic upgrade — should run pending migrations (none in this case)
|
||||||
|
_run_alembic_upgrade(engine)
|
||||||
|
|
||||||
|
# Verify alembic_version table still has the head revision
|
||||||
|
with engine.connect() as conn:
|
||||||
|
result = conn.execute(text("SELECT version_num FROM alembic_version"))
|
||||||
|
row = result.fetchone()
|
||||||
|
assert row is not None
|
||||||
|
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
def test_alembic_upgrade_idempotent(self, tmp_path):
|
||||||
|
"""Test that calling _run_alembic_upgrade multiple times is safe."""
|
||||||
|
from sqlalchemy import create_engine, text
|
||||||
|
|
||||||
|
from app.database import Base, _run_alembic_upgrade
|
||||||
|
|
||||||
|
db_path = str(tmp_path / "idempotent_alembic.db")
|
||||||
|
engine = create_engine(f"sqlite:///{db_path}")
|
||||||
|
Base.metadata.create_all(bind=engine)
|
||||||
|
|
||||||
|
# Run multiple times — should not raise
|
||||||
|
_run_alembic_upgrade(engine)
|
||||||
|
_run_alembic_upgrade(engine)
|
||||||
|
_run_alembic_upgrade(engine)
|
||||||
|
|
||||||
|
# Verify revision is still head
|
||||||
|
with engine.connect() as conn:
|
||||||
|
result = conn.execute(text("SELECT version_num FROM alembic_version"))
|
||||||
|
row = result.fetchone()
|
||||||
|
assert row is not None
|
||||||
|
assert row[0] == "008_add_performance_indexes"
|
||||||
|
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
def test_all_migration_revisions_exist(self):
|
||||||
|
"""Test that all expected Alembic migration revisions are present."""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
migrations_dir = Path(__file__).resolve().parent.parent / "migrations" / "versions"
|
||||||
|
migration_files = sorted(migrations_dir.glob("*.py"))
|
||||||
|
|
||||||
|
expected_prefixes = [
|
||||||
|
"001_file_processing_steps",
|
||||||
|
"002_add_file_paths",
|
||||||
|
"003_add_deduplication_support",
|
||||||
|
"004_add_search_fields",
|
||||||
|
"005_add_saved_searches",
|
||||||
|
"006_add_detail_column",
|
||||||
|
"007_add_ocr_quality_drop_filehash_unique",
|
||||||
|
"008_add_performance_indexes",
|
||||||
|
]
|
||||||
|
|
||||||
|
migration_names = [f.stem for f in migration_files]
|
||||||
|
for prefix in expected_prefixes:
|
||||||
|
assert prefix in migration_names, f"Missing Alembic migration: {prefix}"
|
||||||
|
|
||||||
|
def test_migration_chain_is_connected(self):
|
||||||
|
"""Test that the Alembic migration chain is properly connected."""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from alembic.config import Config
|
||||||
|
from alembic.script import ScriptDirectory
|
||||||
|
|
||||||
|
migrations_dir = str(Path(__file__).resolve().parent.parent / "migrations")
|
||||||
|
alembic_cfg = Config()
|
||||||
|
alembic_cfg.set_main_option("script_location", migrations_dir)
|
||||||
|
|
||||||
|
script = ScriptDirectory.from_config(alembic_cfg)
|
||||||
|
|
||||||
|
# Walk the chain from base to head — should not raise
|
||||||
|
revisions = list(script.walk_revisions())
|
||||||
|
assert len(revisions) == 8 # 001 through 008
|
||||||
|
|
||||||
|
# Verify head is the performance indexes migration
|
||||||
|
heads = script.get_heads()
|
||||||
|
assert "008_add_performance_indexes" in heads
|
||||||
|
|||||||
Reference in New Issue
Block a user