refactor(database): enforce Alembic-only database migrations, deprecate manual schema migrations
- Create alembic.ini and migrations/env.py for full Alembic CLI + programmatic support - Add Alembic migration 006: detail column on processing_logs - Add Alembic migration 007: ocr_quality_score column + drop unique filehash index - Add Alembic migration 008: performance indexes - Replace _run_schema_migrations() call in init_db() with _run_alembic_upgrade() - Deprecate _run_schema_migrations() with DeprecationWarning - Update tests for new Alembic-based approach and deprecation - Update DatabaseConfiguration.md documentation Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
+73
@@ -0,0 +1,73 @@
|
||||
# Alembic Configuration File
|
||||
# Used for managing database schema migrations in DocuElevate.
|
||||
#
|
||||
# Usage:
|
||||
# alembic upgrade head # Apply all pending migrations
|
||||
# alembic current # Show current revision
|
||||
# alembic history --verbose # Show migration history
|
||||
# alembic downgrade -1 # Roll back one migration
|
||||
# alembic revision --autogenerate -m "description" # Create new migration
|
||||
|
||||
[alembic]
|
||||
# Path to migration scripts
|
||||
script_location = migrations
|
||||
|
||||
# Template used to generate migration file names
|
||||
file_template = %%(rev)s_%%(slug)s
|
||||
|
||||
# Timezone for migration file timestamps (uses UTC by default)
|
||||
# timezone =
|
||||
|
||||
# Maximum length of characters for autogenerate revision names
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# Set to 'true' to run environment during 'revision' command
|
||||
# revision_environment = false
|
||||
|
||||
# Set to 'true' to allow .pyc or .pyo files for migration scripts
|
||||
# sourceless = false
|
||||
|
||||
# Version path separator; default is "os" which uses os.pathsep
|
||||
# version_path_separator = os
|
||||
|
||||
# Output encoding for revision files
|
||||
# output_encoding = utf-8
|
||||
|
||||
# The database URL is loaded from app.config.settings.database_url
|
||||
# in migrations/env.py, not from this file.
|
||||
sqlalchemy.url =
|
||||
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
+66
-6
@@ -2,7 +2,9 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
import warnings
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import create_engine, exc
|
||||
@@ -24,8 +26,8 @@ SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
def init_db() -> None:
|
||||
"""
|
||||
Ensures the SQLite database file and its parent directory exist (if using sqlite).
|
||||
Then runs Base.metadata.create_all(bind=engine) to initialize tables.
|
||||
Logs a message if a new SQLite DB file is created.
|
||||
Then runs Base.metadata.create_all(bind=engine) to initialize tables and
|
||||
applies any pending Alembic migrations.
|
||||
"""
|
||||
# 1. Parse the DB URL to see if it's sqlite
|
||||
url = make_url(DB_URL)
|
||||
@@ -50,18 +52,76 @@ def init_db() -> None:
|
||||
Base.metadata.create_all(bind=engine)
|
||||
logger.info("Database initialization complete (tables created if not exist).")
|
||||
|
||||
# 6. Run lightweight schema migrations for existing databases
|
||||
_run_schema_migrations(engine)
|
||||
# 6. Run Alembic migrations for existing databases
|
||||
_run_alembic_upgrade(engine)
|
||||
except exc.SQLAlchemyError as e:
|
||||
logger.error(f"Error initializing database: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def _run_schema_migrations(engine: Any) -> None:
|
||||
def _run_alembic_upgrade(engine: Any) -> None:
|
||||
"""Run Alembic migrations programmatically to apply pending schema changes.
|
||||
|
||||
For fresh databases (created via ``Base.metadata.create_all()``), the
|
||||
Alembic version is stamped to ``head`` because all tables already exist.
|
||||
For existing databases with Alembic tracking, any pending migrations are
|
||||
applied via ``alembic upgrade head``.
|
||||
|
||||
Args:
|
||||
engine: The SQLAlchemy engine connected to the target database.
|
||||
"""
|
||||
Apply lightweight schema migrations for columns added after the initial release.
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import inspect
|
||||
|
||||
inspector = inspect(engine)
|
||||
table_names = inspector.get_table_names()
|
||||
|
||||
# Locate the migrations directory relative to this file
|
||||
migrations_dir = str(Path(__file__).resolve().parent.parent / "migrations")
|
||||
|
||||
# Build an Alembic Config that points at our migration scripts.
|
||||
# The sqlalchemy.url is intentionally left empty because we pass the
|
||||
# live connection via config.attributes["connection"] below.
|
||||
alembic_cfg = Config()
|
||||
alembic_cfg.set_main_option("script_location", migrations_dir)
|
||||
alembic_cfg.set_main_option("sqlalchemy.url", "")
|
||||
|
||||
with engine.begin() as connection:
|
||||
alembic_cfg.attributes["connection"] = connection
|
||||
|
||||
if "alembic_version" not in table_names:
|
||||
# Fresh database or one that predates Alembic tracking.
|
||||
# Base.metadata.create_all() already created everything, so
|
||||
# stamp the current version to head (no migrations need to run).
|
||||
logger.info("No Alembic version table found — stamping database to latest revision.")
|
||||
command.stamp(alembic_cfg, "head")
|
||||
else:
|
||||
# Existing database with Alembic version tracking — apply pending migrations.
|
||||
logger.info("Running pending Alembic migrations…")
|
||||
command.upgrade(alembic_cfg, "head")
|
||||
logger.info("Alembic migration check complete.")
|
||||
|
||||
|
||||
def _run_schema_migrations(engine: Any) -> None:
|
||||
"""Apply lightweight schema migrations for columns added after the initial release.
|
||||
|
||||
.. deprecated::
|
||||
This function is deprecated and will be removed in a future release.
|
||||
All schema migrations are now managed exclusively through Alembic.
|
||||
Run ``alembic upgrade head`` (or let ``init_db()`` handle it
|
||||
automatically) instead of calling this function directly.
|
||||
|
||||
Each migration is idempotent and safe to run multiple times.
|
||||
"""
|
||||
warnings.warn(
|
||||
"_run_schema_migrations() is deprecated. "
|
||||
"All schema changes are now managed by Alembic migrations. "
|
||||
"Use 'alembic upgrade head' or init_db() instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
from sqlalchemy import inspect, text
|
||||
|
||||
inspector = inspect(engine)
|
||||
|
||||
@@ -163,7 +163,15 @@ DATABASE_URL=mysql+pymysql://user:password@host:3306/docuelevate?charset=utf8mb4
|
||||
|
||||
## Schema Migrations with Alembic
|
||||
|
||||
DocuElevate uses Alembic to manage all database schema changes.
|
||||
DocuElevate uses Alembic **exclusively** to manage all database schema changes. On application startup, `init_db()` automatically applies any pending Alembic migrations, so the database schema is always kept in sync with the running code.
|
||||
|
||||
> **Note:** Prior to this change, a manual `_run_schema_migrations()` helper in `app/database.py` applied schema changes outside of Alembic. That function is now **deprecated** and will be removed in a future release. All schema changes are tracked as Alembic revisions in `migrations/versions/`.
|
||||
|
||||
### How It Works
|
||||
|
||||
1. **Fresh databases** — `Base.metadata.create_all()` creates all tables from the SQLAlchemy models, then Alembic stamps the version to `head` (no migrations need to run).
|
||||
2. **Existing databases** — `alembic upgrade head` applies any pending migration scripts.
|
||||
3. **CLI usage** — You can still run `alembic upgrade head` manually or in CI/CD before the application starts.
|
||||
|
||||
### Apply Migrations
|
||||
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Alembic environment configuration for DocuElevate.
|
||||
|
||||
This module configures Alembic to use the application's database URL
|
||||
from ``app.config.settings`` and the SQLAlchemy ``Base.metadata`` so
|
||||
that autogenerate can detect model changes.
|
||||
|
||||
It also supports receiving an existing connection via
|
||||
``config.attributes["connection"]`` for programmatic invocation from
|
||||
``app.database.init_db()``, which is essential for in-memory SQLite
|
||||
databases used in testing.
|
||||
"""
|
||||
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
# Import application Base and models so target_metadata reflects the full schema.
|
||||
from app.database import Base
|
||||
|
||||
# Ensure all models are imported so Base.metadata is populated.
|
||||
from app.models import ( # noqa: F401
|
||||
ApplicationSettings,
|
||||
DocumentMetadata,
|
||||
FileProcessingStep,
|
||||
FileRecord,
|
||||
ProcessingLog,
|
||||
SavedSearch,
|
||||
SettingsAuditLog,
|
||||
)
|
||||
|
||||
# Alembic Config object – provides access to values in alembic.ini.
|
||||
config = context.config
|
||||
|
||||
# Set up Python logging from the config file (if present).
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# MetaData object for autogenerate support.
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def _get_url() -> str:
|
||||
"""Return the database URL, preferring the application config."""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
if url:
|
||||
return url
|
||||
# Fall back to application settings
|
||||
from app.config import settings
|
||||
|
||||
return settings.database_url
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
Configures the context with just a URL and not an Engine.
|
||||
Calls to ``context.execute()`` emit the given string to the script output.
|
||||
"""
|
||||
url = _get_url()
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
Creates an Engine (or reuses a connection passed via
|
||||
``config.attributes["connection"]``) and associates it with the context.
|
||||
"""
|
||||
# If a connection was passed programmatically, reuse it.
|
||||
connectable = config.attributes.get("connection", None)
|
||||
|
||||
if connectable is not None:
|
||||
# Already have a connection — run migrations directly.
|
||||
context.configure(connection=connectable, target_metadata=target_metadata)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
else:
|
||||
# Create a new engine from configuration.
|
||||
configuration = config.get_section(config.config_ini_section, {})
|
||||
url = _get_url()
|
||||
if url:
|
||||
configuration["sqlalchemy.url"] = url
|
||||
connectable = engine_from_config(
|
||||
configuration,
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Add detail column to processing_logs table
|
||||
|
||||
Revision ID: 006_add_detail_column
|
||||
Revises: 005_add_saved_searches
|
||||
Create Date: 2026-03-01
|
||||
|
||||
"""
|
||||
|
||||
from typing import Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "006_add_detail_column"
|
||||
down_revision: Union[str, None] = "005_add_saved_searches"
|
||||
depends_on: Union[str, None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add detail column to processing_logs table for verbose worker log output."""
|
||||
op.add_column("processing_logs", sa.Column("detail", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove detail column from processing_logs table."""
|
||||
op.drop_column("processing_logs", "detail")
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Add ocr_quality_score column and drop unique filehash index
|
||||
|
||||
Revision ID: 007_add_ocr_quality_drop_filehash_unique
|
||||
Revises: 006_add_detail_column
|
||||
Create Date: 2026-03-01
|
||||
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "007_add_ocr_quality_drop_filehash_unique"
|
||||
down_revision: Union[str, None] = "006_add_detail_column"
|
||||
depends_on: Union[str, None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add ocr_quality_score column and drop unique constraint on filehash."""
|
||||
# Add ocr_quality_score column for AI-assessed text quality (0-100)
|
||||
op.add_column("files", sa.Column("ocr_quality_score", sa.Integer(), nullable=True))
|
||||
|
||||
# Drop unique index on filehash to allow duplicate file records.
|
||||
# The filehash column retains a non-unique index for lookups.
|
||||
try:
|
||||
op.drop_index("ix_files_filehash", table_name="files")
|
||||
except Exception as exc:
|
||||
logger.debug("Could not drop ix_files_filehash (may not exist): %s", exc)
|
||||
try:
|
||||
op.create_index("ix_files_filehash", "files", ["filehash"], unique=False)
|
||||
except Exception as exc:
|
||||
logger.debug("Could not create ix_files_filehash (may already exist): %s", exc)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove ocr_quality_score column and restore unique filehash index."""
|
||||
op.drop_column("files", "ocr_quality_score")
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Add performance indexes for common query patterns
|
||||
|
||||
Revision ID: 008_add_performance_indexes
|
||||
Revises: 007_add_ocr_quality_drop_filehash_unique
|
||||
Create Date: 2026-03-01
|
||||
|
||||
"""
|
||||
|
||||
from typing import Union
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "008_add_performance_indexes"
|
||||
down_revision: Union[str, None] = "007_add_ocr_quality_drop_filehash_unique"
|
||||
depends_on: Union[str, None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Create performance indexes for file listing, filtering, and log retrieval.
|
||||
|
||||
Uses ``CREATE INDEX IF NOT EXISTS`` to remain idempotent — safe to run
|
||||
even if the indexes were previously created by ``Base.metadata.create_all()``
|
||||
or an earlier manual migration.
|
||||
"""
|
||||
conn = op.get_bind()
|
||||
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_files_created_at ON files (created_at)"))
|
||||
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_files_mime_type ON files (mime_type)"))
|
||||
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_processing_logs_file_id ON processing_logs (file_id)"))
|
||||
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_processing_logs_timestamp ON processing_logs (timestamp)"))
|
||||
conn.execute(
|
||||
text("CREATE INDEX IF NOT EXISTS ix_file_processing_steps_status ON file_processing_steps (status)")
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Drop performance indexes."""
|
||||
op.drop_index("ix_file_processing_steps_status", table_name="file_processing_steps")
|
||||
op.drop_index("ix_processing_logs_timestamp", table_name="processing_logs")
|
||||
op.drop_index("ix_processing_logs_file_id", table_name="processing_logs")
|
||||
op.drop_index("ix_files_mime_type", table_name="files")
|
||||
op.drop_index("ix_files_created_at", table_name="files")
|
||||
+194
-107
@@ -1,5 +1,6 @@
|
||||
"""Tests for app/database.py module."""
|
||||
|
||||
import warnings
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -110,8 +111,28 @@ class TestSchemaMigrations:
|
||||
|
||||
assert log.detail is None
|
||||
|
||||
def test_migration_adds_detail_column(self, tmp_path):
|
||||
"""Test that _run_schema_migrations adds detail column to existing tables."""
|
||||
def test_deprecated_run_schema_migrations_warns(self, tmp_path):
|
||||
"""Test that _run_schema_migrations emits a DeprecationWarning."""
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
from app.database import _run_schema_migrations
|
||||
|
||||
db_path = str(tmp_path / "deprecation_test.db")
|
||||
engine = create_engine(f"sqlite:///{db_path}")
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warnings.simplefilter("always")
|
||||
_run_schema_migrations(engine)
|
||||
|
||||
assert len(w) == 1
|
||||
assert issubclass(w[0].category, DeprecationWarning)
|
||||
assert "deprecated" in str(w[0].message).lower()
|
||||
assert "Alembic" in str(w[0].message)
|
||||
|
||||
engine.dispose()
|
||||
|
||||
def test_deprecated_migration_still_adds_detail_column(self, tmp_path):
|
||||
"""Test that deprecated _run_schema_migrations still works for legacy callers."""
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from app.database import _run_schema_migrations
|
||||
@@ -133,8 +154,10 @@ class TestSchemaMigrations:
|
||||
)
|
||||
)
|
||||
|
||||
# Run migrations
|
||||
_run_schema_migrations(engine)
|
||||
# Run deprecated migrations (suppress warning for test clarity)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
_run_schema_migrations(engine)
|
||||
|
||||
# Verify detail column was added
|
||||
from sqlalchemy import inspect
|
||||
@@ -145,8 +168,8 @@ class TestSchemaMigrations:
|
||||
|
||||
engine.dispose()
|
||||
|
||||
def test_migration_adds_file_path_columns(self, tmp_path):
|
||||
"""Test that _run_schema_migrations adds file path columns to files table."""
|
||||
def test_deprecated_migration_adds_file_path_columns(self, tmp_path):
|
||||
"""Test that deprecated _run_schema_migrations adds file path columns to files table."""
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from app.database import _run_schema_migrations
|
||||
@@ -165,8 +188,10 @@ class TestSchemaMigrations:
|
||||
)
|
||||
)
|
||||
|
||||
# Run migrations
|
||||
_run_schema_migrations(engine)
|
||||
# Run deprecated migrations
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
_run_schema_migrations(engine)
|
||||
|
||||
# Verify columns were added with correct types
|
||||
from sqlalchemy import inspect
|
||||
@@ -192,92 +217,8 @@ class TestSchemaMigrations:
|
||||
|
||||
engine.dispose()
|
||||
|
||||
def test_migration_adds_search_fields(self, tmp_path):
|
||||
"""Test that _run_schema_migrations adds ocr_text, ai_metadata, document_title to files table."""
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from app.database import _run_schema_migrations
|
||||
|
||||
# Create a database with the old schema (no search fields)
|
||||
db_path = str(tmp_path / "migration_search_fields_test.db")
|
||||
engine = create_engine(f"sqlite:///{db_path}")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(
|
||||
text(
|
||||
"CREATE TABLE files ("
|
||||
"id INTEGER PRIMARY KEY, "
|
||||
"filename VARCHAR, "
|
||||
"filehash VARCHAR, "
|
||||
"original_file_path VARCHAR, "
|
||||
"processed_file_path VARCHAR, "
|
||||
"is_duplicate BOOLEAN DEFAULT FALSE NOT NULL, "
|
||||
"duplicate_of_id INTEGER)"
|
||||
)
|
||||
)
|
||||
|
||||
# Run migrations
|
||||
_run_schema_migrations(engine)
|
||||
|
||||
# Verify search columns were added
|
||||
from sqlalchemy import inspect
|
||||
|
||||
inspector = inspect(engine)
|
||||
columns = {col["name"]: col for col in inspector.get_columns("files")}
|
||||
|
||||
assert "ocr_text" in columns
|
||||
assert "ai_metadata" in columns
|
||||
assert "document_title" in columns
|
||||
|
||||
engine.dispose()
|
||||
|
||||
def test_migration_drops_unique_filehash_index(self, tmp_path):
|
||||
"""Test that _run_schema_migrations drops unique index on filehash."""
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from app.database import _run_schema_migrations
|
||||
|
||||
# Create a database with unique index on filehash
|
||||
db_path = str(tmp_path / "migration_index_test.db")
|
||||
engine = create_engine(f"sqlite:///{db_path}")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(
|
||||
text(
|
||||
"CREATE TABLE files ("
|
||||
"id INTEGER PRIMARY KEY, "
|
||||
"filename VARCHAR, "
|
||||
"filehash VARCHAR, "
|
||||
"upload_date DATETIME, "
|
||||
"original_file_path VARCHAR, "
|
||||
"processed_file_path VARCHAR, "
|
||||
"is_duplicate BOOLEAN DEFAULT FALSE NOT NULL, "
|
||||
"duplicate_of_id INTEGER)"
|
||||
)
|
||||
)
|
||||
conn.execute(text("CREATE UNIQUE INDEX idx_filehash_unique ON files (filehash)"))
|
||||
|
||||
# Verify unique index exists before migration
|
||||
from sqlalchemy import inspect
|
||||
|
||||
inspector = inspect(engine)
|
||||
indexes_before = inspector.get_indexes("files")
|
||||
unique_indexes_before = [idx for idx in indexes_before if idx.get("unique")]
|
||||
assert len(unique_indexes_before) > 0
|
||||
|
||||
# Run migrations
|
||||
_run_schema_migrations(engine)
|
||||
|
||||
# Verify unique index was removed
|
||||
inspector = inspect(engine)
|
||||
indexes_after = inspector.get_indexes("files")
|
||||
unique_filehash_indexes_after = [
|
||||
idx for idx in indexes_after if idx.get("unique") and "filehash" in idx.get("column_names", [])
|
||||
]
|
||||
assert len(unique_filehash_indexes_after) == 0
|
||||
|
||||
engine.dispose()
|
||||
|
||||
def test_migration_handles_missing_tables_gracefully(self, tmp_path):
|
||||
"""Test that migrations don't fail when tables don't exist."""
|
||||
def test_deprecated_migration_handles_missing_tables_gracefully(self, tmp_path):
|
||||
"""Test that deprecated migrations don't fail when tables don't exist."""
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
from app.database import _run_schema_migrations
|
||||
@@ -287,12 +228,14 @@ class TestSchemaMigrations:
|
||||
engine = create_engine(f"sqlite:///{db_path}")
|
||||
|
||||
# Run migrations - should not raise any errors
|
||||
_run_schema_migrations(engine)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
_run_schema_migrations(engine)
|
||||
|
||||
engine.dispose()
|
||||
|
||||
def test_migration_is_idempotent(self, tmp_path):
|
||||
"""Test that running migrations multiple times is safe."""
|
||||
def test_deprecated_migration_is_idempotent(self, tmp_path):
|
||||
"""Test that running deprecated migrations multiple times is safe."""
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from app.database import _run_schema_migrations
|
||||
@@ -323,10 +266,12 @@ class TestSchemaMigrations:
|
||||
)
|
||||
)
|
||||
|
||||
# Run migrations multiple times
|
||||
_run_schema_migrations(engine)
|
||||
_run_schema_migrations(engine)
|
||||
_run_schema_migrations(engine)
|
||||
# Run deprecated migrations multiple times
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
_run_schema_migrations(engine)
|
||||
_run_schema_migrations(engine)
|
||||
_run_schema_migrations(engine)
|
||||
|
||||
# Verify all columns exist and no errors occurred
|
||||
from sqlalchemy import inspect
|
||||
@@ -372,7 +317,11 @@ class TestInitDbErrors:
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestMultiVersionMigrations:
|
||||
"""Test migration scenarios from various database versions."""
|
||||
"""Test deprecated migration scenarios from various database versions.
|
||||
|
||||
These tests verify the deprecated _run_schema_migrations function still
|
||||
works for legacy callers. New schema changes should use Alembic exclusively.
|
||||
"""
|
||||
|
||||
def test_migration_from_v1_to_v2_processing_logs(self, tmp_path):
|
||||
"""Test migration from v1 (no detail column) to v2 (with detail)."""
|
||||
@@ -405,7 +354,9 @@ class TestMultiVersionMigrations:
|
||||
)
|
||||
|
||||
# Run migration to v2
|
||||
_run_schema_migrations(engine)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
_run_schema_migrations(engine)
|
||||
|
||||
# Verify detail column exists and old data is preserved
|
||||
inspector = inspect(engine)
|
||||
@@ -445,7 +396,9 @@ class TestMultiVersionMigrations:
|
||||
conn.execute(text("INSERT INTO files (filename, filehash) VALUES ('test.pdf', 'abc123')"))
|
||||
|
||||
# Run migration to v3 (adds path columns and dedup columns)
|
||||
_run_schema_migrations(engine)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
_run_schema_migrations(engine)
|
||||
|
||||
# Verify all new columns exist
|
||||
inspector = inspect(engine)
|
||||
@@ -489,7 +442,9 @@ class TestMultiVersionMigrations:
|
||||
)
|
||||
|
||||
# Run migration - should not error even though there's no index to drop
|
||||
_run_schema_migrations(engine)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
_run_schema_migrations(engine)
|
||||
|
||||
# Should complete without error
|
||||
engine.dispose()
|
||||
@@ -529,7 +484,9 @@ class TestMultiVersionMigrations:
|
||||
)
|
||||
|
||||
# Run migration - should add missing columns only
|
||||
_run_schema_migrations(engine)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
_run_schema_migrations(engine)
|
||||
|
||||
# Verify all columns exist now
|
||||
inspector = inspect(engine)
|
||||
@@ -570,9 +527,139 @@ class TestMultiVersionMigrations:
|
||||
# Run migration - should handle the exception path for index operations
|
||||
# (when get_indexes might have issues)
|
||||
try:
|
||||
_run_schema_migrations(engine)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
_run_schema_migrations(engine)
|
||||
# Should complete without raising
|
||||
except Exception as e:
|
||||
pytest.fail(f"Migration should handle exceptions gracefully: {e}")
|
||||
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestAlembicUpgrade:
|
||||
"""Tests for Alembic-based migration management."""
|
||||
|
||||
def test_alembic_upgrade_stamps_fresh_database(self, tmp_path):
|
||||
"""Test that _run_alembic_upgrade stamps a fresh database to head."""
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
|
||||
from app.database import Base, _run_alembic_upgrade
|
||||
|
||||
db_path = str(tmp_path / "fresh_alembic.db")
|
||||
engine = create_engine(f"sqlite:///{db_path}")
|
||||
|
||||
# Create all tables (simulates Base.metadata.create_all)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
# Run Alembic upgrade — should stamp to head (not run migrations)
|
||||
_run_alembic_upgrade(engine)
|
||||
|
||||
# Verify alembic_version table exists and has a revision
|
||||
inspector = inspect(engine)
|
||||
assert "alembic_version" in inspector.get_table_names()
|
||||
|
||||
with engine.connect() as conn:
|
||||
result = conn.execute(text("SELECT version_num FROM alembic_version"))
|
||||
row = result.fetchone()
|
||||
assert row is not None
|
||||
# Should be stamped to the latest revision
|
||||
assert row[0] == "008_add_performance_indexes"
|
||||
|
||||
engine.dispose()
|
||||
|
||||
def test_alembic_upgrade_applies_pending_migrations(self, tmp_path):
|
||||
"""Test that _run_alembic_upgrade applies pending migrations to a tracked DB."""
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from app.database import Base, _run_alembic_upgrade
|
||||
|
||||
db_path = str(tmp_path / "tracked_alembic.db")
|
||||
engine = create_engine(f"sqlite:///{db_path}")
|
||||
|
||||
# Create all tables
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
# Manually create alembic_version table and set to an earlier revision
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL)"))
|
||||
conn.execute(text("INSERT INTO alembic_version (version_num) VALUES ('008_add_performance_indexes')"))
|
||||
|
||||
# Run Alembic upgrade — should run pending migrations (none in this case)
|
||||
_run_alembic_upgrade(engine)
|
||||
|
||||
# Verify alembic_version table still has the head revision
|
||||
with engine.connect() as conn:
|
||||
result = conn.execute(text("SELECT version_num FROM alembic_version"))
|
||||
row = result.fetchone()
|
||||
assert row is not None
|
||||
|
||||
engine.dispose()
|
||||
|
||||
def test_alembic_upgrade_idempotent(self, tmp_path):
|
||||
"""Test that calling _run_alembic_upgrade multiple times is safe."""
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from app.database import Base, _run_alembic_upgrade
|
||||
|
||||
db_path = str(tmp_path / "idempotent_alembic.db")
|
||||
engine = create_engine(f"sqlite:///{db_path}")
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
# Run multiple times — should not raise
|
||||
_run_alembic_upgrade(engine)
|
||||
_run_alembic_upgrade(engine)
|
||||
_run_alembic_upgrade(engine)
|
||||
|
||||
# Verify revision is still head
|
||||
with engine.connect() as conn:
|
||||
result = conn.execute(text("SELECT version_num FROM alembic_version"))
|
||||
row = result.fetchone()
|
||||
assert row is not None
|
||||
assert row[0] == "008_add_performance_indexes"
|
||||
|
||||
engine.dispose()
|
||||
|
||||
def test_all_migration_revisions_exist(self):
|
||||
"""Test that all expected Alembic migration revisions are present."""
|
||||
from pathlib import Path
|
||||
|
||||
migrations_dir = Path(__file__).resolve().parent.parent / "migrations" / "versions"
|
||||
migration_files = sorted(migrations_dir.glob("*.py"))
|
||||
|
||||
expected_prefixes = [
|
||||
"001_file_processing_steps",
|
||||
"002_add_file_paths",
|
||||
"003_add_deduplication_support",
|
||||
"004_add_search_fields",
|
||||
"005_add_saved_searches",
|
||||
"006_add_detail_column",
|
||||
"007_add_ocr_quality_drop_filehash_unique",
|
||||
"008_add_performance_indexes",
|
||||
]
|
||||
|
||||
migration_names = [f.stem for f in migration_files]
|
||||
for prefix in expected_prefixes:
|
||||
assert prefix in migration_names, f"Missing Alembic migration: {prefix}"
|
||||
|
||||
def test_migration_chain_is_connected(self):
|
||||
"""Test that the Alembic migration chain is properly connected."""
|
||||
from pathlib import Path
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic.script import ScriptDirectory
|
||||
|
||||
migrations_dir = str(Path(__file__).resolve().parent.parent / "migrations")
|
||||
alembic_cfg = Config()
|
||||
alembic_cfg.set_main_option("script_location", migrations_dir)
|
||||
|
||||
script = ScriptDirectory.from_config(alembic_cfg)
|
||||
|
||||
# Walk the chain from base to head — should not raise
|
||||
revisions = list(script.walk_revisions())
|
||||
assert len(revisions) == 8 # 001 through 008
|
||||
|
||||
# Verify head is the performance indexes migration
|
||||
heads = script.get_heads()
|
||||
assert "008_add_performance_indexes" in heads
|
||||
|
||||
Reference in New Issue
Block a user