Christian Krakau-Louis
4c11cfd6dd
- Add enable_deduplication and show_deduplication_step config options - Rename hash_file step to check_for_duplicates - Make deduplication step conditional based on configuration - Add is_duplicate and duplicate_of_id fields to FileRecord model - Create database migration for new deduplication fields - Update process_document task to log deduplication results - Update step visualization to show/hide step based on config - Update status calculations to include deduplication step conditionally - Default: deduplication enabled, step displayed - Can be configured to hide from UI while still processing
46 lines
1.5 KiB
Python
46 lines
1.5 KiB
Python
"""Add deduplication support with is_duplicate and duplicate_of_id fields
|
|
|
|
Revision ID: 003_add_deduplication_support
|
|
Revises: 002_add_file_paths
|
|
Create Date: 2026-02-12
|
|
|
|
"""
|
|
|
|
from typing import Sequence, Union
|
|
|
|
import sqlalchemy as sa
|
|
from alembic import op
|
|
|
|
# revision identifiers, used by Alembic.
|
|
revision: str = "003_add_deduplication_support"
|
|
down_revision: Union[str, None] = "002_add_file_paths"
|
|
depends_on: Union[str, None] = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
"""Add is_duplicate and duplicate_of_id columns to files table."""
|
|
# Add is_duplicate column with default False
|
|
op.add_column("files", sa.Column("is_duplicate", sa.Boolean(), nullable=False, server_default="0"))
|
|
|
|
# Add duplicate_of_id column as foreign key to self
|
|
op.add_column("files", sa.Column("duplicate_of_id", sa.Integer(), nullable=True))
|
|
|
|
# Create index on is_duplicate for efficient filtering
|
|
op.create_index("ix_files_is_duplicate", "files", ["is_duplicate"])
|
|
|
|
# Create foreign key relationship
|
|
op.create_foreign_key("fk_files_duplicate_of_id", "files", "files", ["duplicate_of_id"], ["id"])
|
|
|
|
|
|
def downgrade() -> None:
|
|
"""Remove is_duplicate and duplicate_of_id columns from files table."""
|
|
# Drop foreign key
|
|
op.drop_constraint("fk_files_duplicate_of_id", "files", type_="foreignkey")
|
|
|
|
# Drop index
|
|
op.drop_index("ix_files_is_duplicate", table_name="files")
|
|
|
|
# Drop columns
|
|
op.drop_column("files", "duplicate_of_id")
|
|
op.drop_column("files", "is_duplicate")
|