fix: merge main branch and renumber migration 027→037

Resolve 3 merge conflicts and renumber the automation_hooks migration
to follow main's migration chain (036_add_document_translation_fields).

Conflicts resolved:
- app/api/__init__.py: add automation_router alongside main's new routers
- app/utils/settings_service.py: add automation_hooks_enabled alongside compliance_enabled
- tests/conftest.py: add AutomationHook alongside AuditLog/ComplianceTemplate imports

Migration renumbered:
- 027_add_automation_hooks → 037_add_automation_hooks
- down_revision: 026_add_scheduled_jobs → 036_add_document_translation_fields

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-16 22:40:15 +00:00
parent 6a83d51d88
commit 204000aabc
318 changed files with 317516 additions and 2858 deletions
+20
View File
@@ -20,13 +20,33 @@ from app.database import Base
# Ensure all models are imported so Base.metadata is populated.
from app.models import ( # noqa: F401
ApiToken,
ApplicationSettings,
AuditLog,
BackupRecord,
ComplianceTemplate,
DocumentMetadata,
FileProcessingStep,
FileRecord,
ImapIngestionProfile,
InAppNotification,
LocalUser,
MobileDevice,
Pipeline,
PipelineRoutingRule,
PipelineStep,
ProcessingLog,
SavedSearch,
ScheduledJob,
SettingsAuditLog,
SharedLink,
SubscriptionPlan,
UserImapAccount,
UserIntegration,
UserNotificationPreference,
UserNotificationTarget,
UserProfile,
WebhookConfig,
)
# Alembic Config object provides access to values in alembic.ini.
@@ -0,0 +1,62 @@
"""Ensure shared_links table exists for databases that skipped migration 025.
Databases that were already at revision 025_add_user_notifications or
026_add_scheduled_jobs before 025_add_shared_links was inserted into the
migration chain will never have had the ``shared_links`` table created.
This migration creates the table idempotently so those databases are
repaired on the next ``alembic upgrade head``.
Revision ID: 027_ensure_shared_links_table
Revises: 026_add_scheduled_jobs
Create Date: 2026-03-09
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "027_ensure_shared_links_table"
down_revision: Union[str, None] = "026_add_scheduled_jobs"
depends_on: Union[str, None] = None
def upgrade() -> None:
"""Create shared_links table if it does not already exist."""
conn = op.get_bind()
inspector = sa.inspect(conn)
if "shared_links" not in inspector.get_table_names():
op.create_table(
"shared_links",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("token", sa.String(64), nullable=False),
sa.Column("file_id", sa.Integer(), nullable=False),
sa.Column("owner_id", sa.String(), nullable=False),
sa.Column("label", sa.String(255), nullable=True),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("max_views", sa.Integer(), nullable=True),
sa.Column("view_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("password_hash", sa.String(128), nullable=True),
sa.Column("is_active", sa.Boolean(), nullable=False, server_default="1"),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
sa.ForeignKeyConstraint(["file_id"], ["files.id"]),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("token"),
)
op.create_index("ix_shared_links_id", "shared_links", ["id"])
op.create_index("ix_shared_links_token", "shared_links", ["token"])
op.create_index("ix_shared_links_file_id", "shared_links", ["file_id"])
op.create_index("ix_shared_links_owner_id", "shared_links", ["owner_id"])
def downgrade() -> None:
"""Drop shared_links table only if this migration created it."""
conn = op.get_bind()
inspector = sa.inspect(conn)
if "shared_links" in inspector.get_table_names():
op.drop_index("ix_shared_links_owner_id", "shared_links")
op.drop_index("ix_shared_links_file_id", "shared_links")
op.drop_index("ix_shared_links_token", "shared_links")
op.drop_index("ix_shared_links_id", "shared_links")
op.drop_table("shared_links")
+47
View File
@@ -0,0 +1,47 @@
"""Add audit_logs table for comprehensive compliance audit logging.
Revision ID: 028_add_audit_logs
Revises: 027_ensure_shared_links_table
Create Date: 2026-03-09
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "028_add_audit_logs"
down_revision: Union[str, None] = "027_ensure_shared_links_table"
depends_on: Union[str, None] = None
def upgrade() -> None:
"""Create audit_logs table."""
op.create_table(
"audit_logs",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("timestamp", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("user", sa.String(), nullable=False),
sa.Column("action", sa.String(), nullable=False),
sa.Column("resource_type", sa.String(), nullable=True),
sa.Column("resource_id", sa.String(), nullable=True),
sa.Column("ip_address", sa.String(), nullable=True),
sa.Column("details", sa.Text(), nullable=True),
sa.Column("severity", sa.String(16), nullable=False, server_default="info"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_audit_logs_id", "audit_logs", ["id"])
op.create_index("ix_audit_logs_timestamp", "audit_logs", ["timestamp"])
op.create_index("ix_audit_logs_user", "audit_logs", ["user"])
op.create_index("ix_audit_logs_action", "audit_logs", ["action"])
op.create_index("ix_audit_logs_resource_type", "audit_logs", ["resource_type"])
def downgrade() -> None:
"""Drop audit_logs table."""
op.drop_index("ix_audit_logs_resource_type", "audit_logs")
op.drop_index("ix_audit_logs_action", "audit_logs")
op.drop_index("ix_audit_logs_user", "audit_logs")
op.drop_index("ix_audit_logs_timestamp", "audit_logs")
op.drop_index("ix_audit_logs_id", "audit_logs")
op.drop_table("audit_logs")
@@ -0,0 +1,40 @@
"""Add preferred_language column to user_profiles for i18n support.
Revision ID: 029_add_user_language_preference
Revises: 028_add_audit_logs
Create Date: 2026-03-09
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "029_add_user_language_preference"
down_revision: Union[str, None] = "028_add_audit_logs"
depends_on: Union[str, None] = None
def upgrade() -> None:
"""Add preferred_language column to user_profiles table."""
conn = op.get_bind()
inspector = sa.inspect(conn)
if "user_profiles" not in inspector.get_table_names():
return
existing_columns = {col["name"] for col in inspector.get_columns("user_profiles")}
if "preferred_language" not in existing_columns:
op.add_column(
"user_profiles",
sa.Column("preferred_language", sa.String(10), nullable=True, server_default=None),
)
def downgrade() -> None:
"""Remove preferred_language column from user_profiles table."""
conn = op.get_bind()
inspector = sa.inspect(conn)
if "user_profiles" not in inspector.get_table_names():
return
existing_columns = {col["name"] for col in inspector.get_columns("user_profiles")}
if "preferred_language" in existing_columns:
op.drop_column("user_profiles", "preferred_language")
@@ -0,0 +1,41 @@
"""Add mobile_devices table for push notification device registration.
Revision ID: 030_add_mobile_devices
Revises: 029_add_user_language_preference
Create Date: 2026-03-10
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "030_add_mobile_devices"
down_revision: Union[str, None] = "029_add_user_language_preference"
depends_on: Union[str, None] = None
def upgrade() -> None:
"""Create mobile_devices table."""
op.create_table(
"mobile_devices",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("owner_id", sa.String(), nullable=False),
sa.Column("device_name", sa.String(255), nullable=True),
sa.Column("platform", sa.String(20), nullable=False, server_default="ios"),
sa.Column("push_token", sa.String(512), nullable=False),
sa.Column("is_active", sa.Boolean(), nullable=False, server_default="1"),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=True),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("owner_id", "push_token", name="uq_mobile_device_owner_token"),
)
op.create_index("ix_mobile_devices_id", "mobile_devices", ["id"])
op.create_index("ix_mobile_devices_owner_id", "mobile_devices", ["owner_id"])
def downgrade() -> None:
"""Drop mobile_devices table."""
op.drop_index("ix_mobile_devices_owner_id", table_name="mobile_devices")
op.drop_index("ix_mobile_devices_id", table_name="mobile_devices")
op.drop_table("mobile_devices")
@@ -0,0 +1,44 @@
"""Add compliance_templates table for GDPR, HIPAA, SOC2 compliance templates.
Revision ID: 031_add_compliance_templates
Revises: 030_add_mobile_devices
Create Date: 2026-03-09
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "031_add_compliance_templates"
down_revision: Union[str, None] = "030_add_mobile_devices"
depends_on: Union[str, None] = None
def upgrade() -> None:
"""Create compliance_templates table."""
op.create_table(
"compliance_templates",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("name", sa.String(50), nullable=False),
sa.Column("display_name", sa.String(100), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("settings_json", sa.Text(), nullable=False, server_default="{}"),
sa.Column("enabled", sa.Boolean(), nullable=False, server_default="0"),
sa.Column("status", sa.String(20), nullable=False, server_default="not_applied"),
sa.Column("applied_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("applied_by", sa.String(255), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("name", name="uq_compliance_templates_name"),
)
op.create_index("ix_compliance_templates_id", "compliance_templates", ["id"])
op.create_index("ix_compliance_templates_name", "compliance_templates", ["name"])
def downgrade() -> None:
"""Drop compliance_templates table."""
op.drop_index("ix_compliance_templates_name", "compliance_templates")
op.drop_index("ix_compliance_templates_id", "compliance_templates")
op.drop_table("compliance_templates")
@@ -0,0 +1,28 @@
"""Add attachment_filter column to user_imap_accounts table.
Revision ID: 032_add_imap_attachment_filter
Revises: 031_add_compliance_templates
Create Date: 2026-03-12
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "032_add_imap_attachment_filter"
down_revision: Union[str, None] = "031_add_compliance_templates"
depends_on: Union[str, None] = None
def upgrade() -> None:
"""Add attachment_filter column to user_imap_accounts."""
op.add_column(
"user_imap_accounts",
sa.Column("attachment_filter", sa.String(50), nullable=True),
)
def downgrade() -> None:
"""Remove attachment_filter column from user_imap_accounts."""
op.drop_column("user_imap_accounts", "attachment_filter")
@@ -0,0 +1,146 @@
"""Add imap_ingestion_profiles table and migrate user_imap_accounts.
Creates the ``imap_ingestion_profiles`` table, seeds the two built-in profiles
("Documents Only" and "All Files"), and replaces the ``attachment_filter``
string column on ``user_imap_accounts`` with a ``profile_id`` FK that references
the new table.
Revision ID: 033_add_imap_ingestion_profiles
Revises: 032_add_imap_attachment_filter
Create Date: 2026-03-12
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "033_add_imap_ingestion_profiles"
down_revision: Union[str, None] = "032_add_imap_attachment_filter"
depends_on: Union[str, None] = None
# Fixed IDs for the built-in profiles so that the FK migration is reproducible.
_BUILTIN_DOCUMENTS_ONLY_ID = 1
_BUILTIN_ALL_FILES_ID = 2
def upgrade() -> None:
"""Create profiles table, seed built-ins, migrate accounts column."""
# 1 — Create the imap_ingestion_profiles table
op.create_table(
"imap_ingestion_profiles",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("name", sa.String(255), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("owner_id", sa.String(), nullable=True),
sa.Column(
"allowed_categories",
sa.Text(),
nullable=False,
server_default='["pdf","office","opendocument","text","web"]',
),
sa.Column("is_builtin", sa.Boolean(), nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_imap_ingestion_profiles_id", "imap_ingestion_profiles", ["id"])
op.create_index("ix_imap_ingestion_profiles_owner_id", "imap_ingestion_profiles", ["owner_id"])
# 2 — Seed the two built-in profiles
bind = op.get_bind()
bind.execute(
sa.text(
"INSERT INTO imap_ingestion_profiles "
"(id, name, description, owner_id, allowed_categories, is_builtin) "
"VALUES (:id, :name, :desc, NULL, :cats, 1)"
),
[
{
"id": _BUILTIN_DOCUMENTS_ONLY_ID,
"name": "Documents Only",
"desc": (
"Ingest PDFs, Microsoft Office files, OpenDocument files, "
"plain text, CSV, RTF, HTML and Markdown. Images are excluded."
),
"cats": '["pdf","office","opendocument","text","web"]',
},
{
"id": _BUILTIN_ALL_FILES_ID,
"name": "All Files",
"desc": "Ingest all supported file types, including images.",
"cats": '["pdf","office","opendocument","text","web","images"]',
},
],
)
# 3 — Add profile_id column to user_imap_accounts
op.add_column(
"user_imap_accounts",
sa.Column("profile_id", sa.Integer(), nullable=True),
)
# 4 — Migrate existing attachment_filter values to profile_id
bind.execute(
sa.text(
"UPDATE user_imap_accounts SET profile_id = :pid "
"WHERE attachment_filter = 'all'"
),
{"pid": _BUILTIN_ALL_FILES_ID},
)
bind.execute(
sa.text(
"UPDATE user_imap_accounts SET profile_id = :pid "
"WHERE attachment_filter = 'documents_only'"
),
{"pid": _BUILTIN_DOCUMENTS_ONLY_ID},
)
# Rows with NULL attachment_filter keep profile_id = NULL (use global default)
# 5 — Add FK constraint (skip for SQLite which does not enforce FKs at DDL time)
# We use batch_alter_table so this works across SQLite and PostgreSQL
with op.batch_alter_table("user_imap_accounts") as batch_op:
batch_op.create_foreign_key(
"fk_user_imap_accounts_profile_id",
"imap_ingestion_profiles",
["profile_id"],
["id"],
)
# 6 — Drop the now-redundant attachment_filter column
with op.batch_alter_table("user_imap_accounts") as batch_op:
batch_op.drop_column("attachment_filter")
def downgrade() -> None:
"""Reverse the migration: restore attachment_filter, drop profiles table."""
# 1 — Re-add attachment_filter column
with op.batch_alter_table("user_imap_accounts") as batch_op:
batch_op.add_column(sa.Column("attachment_filter", sa.String(50), nullable=True))
# 2 — Restore string values from profile_id
bind = op.get_bind()
bind.execute(
sa.text(
"UPDATE user_imap_accounts SET attachment_filter = 'all' "
"WHERE profile_id = :pid"
),
{"pid": _BUILTIN_ALL_FILES_ID},
)
bind.execute(
sa.text(
"UPDATE user_imap_accounts SET attachment_filter = 'documents_only' "
"WHERE profile_id = :pid"
),
{"pid": _BUILTIN_DOCUMENTS_ONLY_ID},
)
# 3 — Drop the FK and profile_id column
with op.batch_alter_table("user_imap_accounts") as batch_op:
batch_op.drop_constraint("fk_user_imap_accounts_profile_id", type_="foreignkey")
batch_op.drop_column("profile_id")
# 4 — Drop the profiles table
op.drop_index("ix_imap_ingestion_profiles_owner_id", "imap_ingestion_profiles")
op.drop_index("ix_imap_ingestion_profiles_id", "imap_ingestion_profiles")
op.drop_table("imap_ingestion_profiles")
@@ -0,0 +1,47 @@
"""Add preferred_theme and avatar_data columns to user_profiles.
Revision ID: 034_add_user_profile_settings
Revises: 033_add_imap_ingestion_profiles
Create Date: 2026-03-12
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "034_add_user_profile_settings"
down_revision: Union[str, None] = "033_add_imap_ingestion_profiles"
depends_on: Union[str, None] = None
def upgrade() -> None:
"""Add preferred_theme and avatar_data columns to user_profiles table."""
conn = op.get_bind()
inspector = sa.inspect(conn)
if "user_profiles" not in inspector.get_table_names():
return
existing_columns = {col["name"] for col in inspector.get_columns("user_profiles")}
if "preferred_theme" not in existing_columns:
op.add_column(
"user_profiles",
sa.Column("preferred_theme", sa.String(10), nullable=True, server_default=None),
)
if "avatar_data" not in existing_columns:
op.add_column(
"user_profiles",
sa.Column("avatar_data", sa.Text, nullable=True, server_default=None),
)
def downgrade() -> None:
"""Remove preferred_theme and avatar_data columns from user_profiles table."""
conn = op.get_bind()
inspector = sa.inspect(conn)
if "user_profiles" not in inspector.get_table_names():
return
existing_columns = {col["name"] for col in inspector.get_columns("user_profiles")}
if "avatar_data" in existing_columns:
op.drop_column("user_profiles", "avatar_data")
if "preferred_theme" in existing_columns:
op.drop_column("user_profiles", "preferred_theme")
@@ -0,0 +1,46 @@
"""Add pipeline_routing_rules table for conditional document routing.
Revision ID: 035_add_routing_rules
Revises: 034_add_user_profile_settings
Create Date: 2026-03-09
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "035_add_routing_rules"
down_revision: Union[str, None] = "034_add_user_profile_settings"
depends_on: Union[str, None] = None
def upgrade() -> None:
"""Create pipeline_routing_rules table."""
op.create_table(
"pipeline_routing_rules",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("owner_id", sa.String(), nullable=True),
sa.Column("name", sa.String(255), nullable=False),
sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
sa.Column("field", sa.String(255), nullable=False),
sa.Column("operator", sa.String(50), nullable=False),
sa.Column("value", sa.String(1024), nullable=False),
sa.Column("target_pipeline_id", sa.Integer(), nullable=False),
sa.Column("is_active", sa.Boolean(), nullable=False, server_default="1"),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.PrimaryKeyConstraint("id"),
sa.ForeignKeyConstraint(["target_pipeline_id"], ["pipelines.id"]),
)
op.create_index("ix_routing_rules_id", "pipeline_routing_rules", ["id"])
op.create_index("ix_routing_rules_owner_id", "pipeline_routing_rules", ["owner_id"])
op.create_index("ix_routing_rules_target_pipeline_id", "pipeline_routing_rules", ["target_pipeline_id"])
def downgrade() -> None:
"""Drop pipeline_routing_rules table."""
op.drop_index("ix_routing_rules_target_pipeline_id", "pipeline_routing_rules")
op.drop_index("ix_routing_rules_owner_id", "pipeline_routing_rules")
op.drop_index("ix_routing_rules_id", "pipeline_routing_rules")
op.drop_table("pipeline_routing_rules")
@@ -0,0 +1,70 @@
"""Add document translation fields to files and user_profiles tables.
Adds detected_language, default_language_text, and default_language_code to
the files table so that a translated version of the document text can be
stored alongside the original.
Adds default_document_language to user_profiles so each user can override
the system-wide default translation target language.
Revision ID: 036_add_document_translation_fields
Revises: 035_add_routing_rules
Create Date: 2026-03-16
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "036_add_document_translation_fields"
down_revision: Union[str, None] = "035_add_routing_rules"
depends_on: Union[str, None] = None
def upgrade() -> None:
"""Add translation columns to files and user_profiles."""
conn = op.get_bind()
inspector = sa.inspect(conn)
if "files" in inspector.get_table_names():
existing_file_cols = {col["name"] for col in inspector.get_columns("files")}
cols_to_add = {"detected_language", "default_language_text", "default_language_code"} - existing_file_cols
if cols_to_add:
with op.batch_alter_table("files") as batch_op:
if "detected_language" in cols_to_add:
batch_op.add_column(sa.Column("detected_language", sa.String(10), nullable=True))
if "default_language_text" in cols_to_add:
batch_op.add_column(sa.Column("default_language_text", sa.Text(), nullable=True))
if "default_language_code" in cols_to_add:
batch_op.add_column(sa.Column("default_language_code", sa.String(10), nullable=True))
if "user_profiles" in inspector.get_table_names():
existing_profile_cols = {col["name"] for col in inspector.get_columns("user_profiles")}
if "default_document_language" not in existing_profile_cols:
with op.batch_alter_table("user_profiles") as batch_op:
batch_op.add_column(sa.Column("default_document_language", sa.String(10), nullable=True))
def downgrade() -> None:
"""Remove translation columns."""
conn = op.get_bind()
inspector = sa.inspect(conn)
if "user_profiles" in inspector.get_table_names():
existing_profile_cols = {col["name"] for col in inspector.get_columns("user_profiles")}
if "default_document_language" in existing_profile_cols:
with op.batch_alter_table("user_profiles") as batch_op:
batch_op.drop_column("default_document_language")
if "files" in inspector.get_table_names():
existing_file_cols = {col["name"] for col in inspector.get_columns("files")}
cols_to_drop = {"detected_language", "default_language_text", "default_language_code"} & existing_file_cols
if cols_to_drop:
with op.batch_alter_table("files") as batch_op:
if "default_language_code" in cols_to_drop:
batch_op.drop_column("default_language_code")
if "default_language_text" in cols_to_drop:
batch_op.drop_column("default_language_text")
if "detected_language" in cols_to_drop:
batch_op.drop_column("detected_language")
@@ -1,7 +1,7 @@
"""Add automation_hooks table for Zapier / Make.com webhook subscriptions.
Revision ID: 027_add_automation_hooks
Revises: 026_add_scheduled_jobs
Revision ID: 037_add_automation_hooks
Revises: 036_add_document_translation_fields
Create Date: 2026-03-09
"""
@@ -10,8 +10,8 @@ from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "027_add_automation_hooks"
down_revision: Union[str, None] = "026_add_scheduled_jobs"
revision: str = "037_add_automation_hooks"
down_revision: Union[str, None] = "036_add_document_translation_fields"
depends_on: Union[str, None] = None