feat(models): add UserIntegration model, encrypt IMAP passwords, add integrations API

- Add IntegrationDirection/IntegrationType constants and UserIntegration SQLAlchemy model with owner_id, direction, integration_type, name, config (JSON), credentials (encrypted), is_active, last_used_at, last_error, created_at, updated_at fields - Fix UserImapAccount password encryption: encrypt_value() on create/update, decrypt_value() in IMAP tasks and test-connection endpoint (backward compatible) - Add Alembic migration 023_add_user_integrations - Add app/api/integrations.py: CRUD + credentials endpoint with owner-scoped access - Register integrations router in app/api/__init__.py - Add 39 tests in tests/test_api_integrations.py covering model, CRUD, encryption Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
2026-03-08 15:02:38 +00:00
parent 077672d8c1
commit 743627ecc2
8 changed files with 1099 additions and 7 deletions
@@ -17,6 +17,7 @@ from app.api.duplicates import router as duplicates_router
 from app.api.files import router as files_router
 from app.api.google_drive import router as google_drive_router
 from app.api.imap_accounts import router as imap_accounts_router
+from app.api.integrations import router as integrations_router
 from app.api.logs import router as logs_router
 from app.api.onboarding import router as onboarding_router
 from app.api.onedrive import router as onedrive_router
@@ -70,3 +71,4 @@ router.include_router(onboarding_router)
 router.include_router(billing_router)
 router.include_router(pipelines_router)
 router.include_router(imap_accounts_router)
+router.include_router(integrations_router)
@@ -16,6 +16,7 @@ from sqlalchemy.orm import Session

 from app.database import get_db
 from app.models import UserImapAccount
+from app.utils.encryption import decrypt_value, encrypt_value
 from app.utils.subscription import get_tier, get_user_tier_id
 from app.utils.user_scope import get_current_owner_id

@@ -217,7 +218,7 @@ def create_imap_account(
        host=body.host,
        port=body.port,
        username=body.username,
-        password=body.password,
+        password=encrypt_value(body.password),
        use_ssl=body.use_ssl,
        delete_after_process=body.delete_after_process,
        is_active=body.is_active,
@@ -269,7 +270,7 @@ def update_imap_account(
    if body.username is not None:
        acct.username = body.username
    if body.password is not None:
-        acct.password = body.password
+        acct.password = encrypt_value(body.password)
    if body.use_ssl is not None:
        acct.use_ssl = body.use_ssl
    if body.delete_after_process is not None:
@@ -320,7 +321,7 @@ def test_saved_imap_account(account_id: int, request: Request, db: DbSession, ow
    if not acct:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="IMAP account not found")

-    return _test_imap_connection(acct.host, acct.port, acct.username, acct.password, acct.use_ssl)
+    return _test_imap_connection(acct.host, acct.port, acct.username, decrypt_value(acct.password), acct.use_ssl)


@router.post("/test", summary="Test an IMAP connection without saving")
@@ -0,0 +1,344 @@
+"""API endpoints for managing per-user integrations (sources and destinations).
+
+Provides CRUD operations for :class:`~app.models.UserIntegration` records.
+Each record represents one ingestion source (e.g. IMAP, Watch Folder) or
+storage destination (e.g. S3, Dropbox, Google Drive) configured by a user.
+
+Sensitive credentials are encrypted at rest using Fernet symmetric encryption
+(keyed from ``SESSION_SECRET``) via :mod:`app.utils.encryption`.  Credential
+values are **never** returned in API responses.
+"""
+
+import json
+import logging
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, HTTPException, Request, status
+from pydantic import BaseModel, Field
+from sqlalchemy.orm import Session
+
+from app.database import get_db
+from app.models import IntegrationDirection, IntegrationType, UserIntegration
+from app.utils.encryption import decrypt_value, encrypt_value
+from app.utils.user_scope import get_current_owner_id
+
+logger = logging.getLogger(__name__)
+router = APIRouter(prefix="/integrations", tags=["integrations"])
+
+DbSession = Annotated[Session, Depends(get_db)]
+
+# ---------------------------------------------------------------------------
+# Auth helpers
+# ---------------------------------------------------------------------------
+
+
+def _get_owner_id(request: Request) -> str:
+    """Return the current user's owner ID, raising 401 if unauthenticated."""
+    owner_id = get_current_owner_id(request)
+    if owner_id is None:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
+    return owner_id
+
+
+CurrentOwner = Annotated[str, Depends(_get_owner_id)]
+
+# ---------------------------------------------------------------------------
+# Pydantic schemas
+# ---------------------------------------------------------------------------
+
+_VALID_DIRECTIONS = IntegrationDirection.ALL
+_VALID_TYPES = IntegrationType.ALL
+
+
+class IntegrationCreate(BaseModel):
+    """Schema for creating a new integration."""
+
+    direction: str = Field(..., description="'SOURCE' or 'DESTINATION'")
+    integration_type: str = Field(..., description="Integration type (e.g. 'IMAP', 'S3', 'DROPBOX')")
+    name: str = Field(..., min_length=1, max_length=255, description="Human-readable label")
+    config: dict[str, Any] | None = Field(default=None, description="Non-sensitive configuration (JSON object)")
+    credentials: dict[str, Any] | None = Field(
+        default=None, description="Sensitive credentials (JSON object, encrypted at rest)"
+    )
+    is_active: bool = Field(default=True, description="Whether the integration is active")
+
+
+class IntegrationUpdate(BaseModel):
+    """Schema for updating an existing integration (all fields optional)."""
+
+    name: str | None = Field(default=None, min_length=1, max_length=255)
+    config: dict[str, Any] | None = None
+    credentials: dict[str, Any] | None = None
+    is_active: bool | None = None
+
+
+# ---------------------------------------------------------------------------
+# Validation helpers
+# ---------------------------------------------------------------------------
+
+
+def _validate_direction(direction: str) -> None:
+    """Raise 400 if *direction* is not a known value."""
+    if direction not in _VALID_DIRECTIONS:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Invalid direction '{direction}'. Must be one of: {sorted(_VALID_DIRECTIONS)}",
+        )
+
+
+def _validate_integration_type(integration_type: str) -> None:
+    """Raise 400 if *integration_type* is not a known value."""
+    if integration_type not in _VALID_TYPES:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Invalid integration_type '{integration_type}'. Must be one of: {sorted(_VALID_TYPES)}",
+        )
+
+
+# ---------------------------------------------------------------------------
+# Serialisation helpers
+# ---------------------------------------------------------------------------
+
+
+def _to_response(integration: UserIntegration) -> dict[str, Any]:
+    """Serialise a :class:`UserIntegration` row to a response dict.
+
+    Credentials are **never** included; only a boolean flag indicating
+    whether credentials have been configured is returned.
+    """
+    config_data: dict[str, Any] | None = None
+    if integration.config:
+        try:
+            config_data = json.loads(integration.config)
+        except (json.JSONDecodeError, TypeError):
+            config_data = None
+
+    return {
+        "id": integration.id,
+        "owner_id": integration.owner_id,
+        "direction": integration.direction,
+        "integration_type": integration.integration_type,
+        "name": integration.name,
+        "config": config_data,
+        "has_credentials": bool(integration.credentials),
+        "is_active": integration.is_active,
+        "last_used_at": integration.last_used_at.isoformat() if integration.last_used_at else None,
+        "last_error": integration.last_error,
+        "created_at": integration.created_at.isoformat() if integration.created_at else None,
+        "updated_at": integration.updated_at.isoformat() if integration.updated_at else None,
+    }
+
+
+def _encode_credentials(credentials: dict[str, Any] | None) -> str | None:
+    """Serialise *credentials* dict to an encrypted JSON string for storage."""
+    if not credentials:
+        return None
+    plaintext = json.dumps(credentials)
+    return encrypt_value(plaintext)
+
+
+def _decode_credentials(stored: str | None) -> dict[str, Any] | None:
+    """Decrypt and deserialise stored credentials back to a dict.
+
+    Returns ``None`` when *stored* is empty or cannot be decoded.
+    """
+    if not stored:
+        return None
+    plaintext = decrypt_value(stored)
+    if not plaintext:
+        return None
+    try:
+        return json.loads(plaintext)
+    except (json.JSONDecodeError, TypeError):
+        logger.error("Failed to decode credentials JSON after decryption")
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Endpoints
+# ---------------------------------------------------------------------------
+
+
+@router.get("/", summary="List integrations for the current user")
+def list_integrations(
+    request: Request,
+    db: DbSession,
+    owner_id: CurrentOwner,
+    direction: str | None = None,
+    integration_type: str | None = None,
+) -> list[dict[str, Any]]:
+    """Return all integrations belonging to the authenticated user.
+
+    Optional query-string filters:
+
+    - ``direction`` — ``SOURCE`` or ``DESTINATION``
+    - ``integration_type`` — e.g. ``IMAP``, ``S3``, ``DROPBOX``
+    """
+    query = db.query(UserIntegration).filter(UserIntegration.owner_id == owner_id)
+
+    if direction is not None:
+        _validate_direction(direction)
+        query = query.filter(UserIntegration.direction == direction)
+
+    if integration_type is not None:
+        _validate_integration_type(integration_type)
+        query = query.filter(UserIntegration.integration_type == integration_type)
+
+    integrations = query.order_by(UserIntegration.id).all()
+    return [_to_response(i) for i in integrations]
+
+
+@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new integration")
+def create_integration(
+    request: Request,
+    body: IntegrationCreate,
+    db: DbSession,
+    owner_id: CurrentOwner,
+) -> dict[str, Any]:
+    """Create a new source or destination integration for the current user.
+
+    ``credentials`` are encrypted at rest using Fernet symmetric encryption
+    before being persisted and are **never** returned in API responses.
+    """
+    _validate_direction(body.direction)
+    _validate_integration_type(body.integration_type)
+
+    integration = UserIntegration(
+        owner_id=owner_id,
+        direction=body.direction,
+        integration_type=body.integration_type,
+        name=body.name,
+        config=json.dumps(body.config) if body.config is not None else None,
+        credentials=_encode_credentials(body.credentials),
+        is_active=body.is_active,
+    )
+
+    try:
+        db.add(integration)
+        db.commit()
+        db.refresh(integration)
+    except Exception:
+        db.rollback()
+        raise
+
+    logger.info(
+        "User %s created %s integration %d (%s)",
+        owner_id,
+        body.direction,
+        integration.id,
+        body.integration_type,
+    )
+    return _to_response(integration)
+
+
+@router.get("/{integration_id}", summary="Get a single integration")
+def get_integration(
+    integration_id: int,
+    request: Request,
+    db: DbSession,
+    owner_id: CurrentOwner,
+) -> dict[str, Any]:
+    """Return a single integration by ID (must belong to the current user)."""
+    integration = (
+        db.query(UserIntegration)
+        .filter(UserIntegration.id == integration_id, UserIntegration.owner_id == owner_id)
+        .first()
+    )
+    if not integration:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found")
+    return _to_response(integration)
+
+
+@router.put("/{integration_id}", summary="Update an integration")
+def update_integration(
+    integration_id: int,
+    request: Request,
+    body: IntegrationUpdate,
+    db: DbSession,
+    owner_id: CurrentOwner,
+) -> dict[str, Any]:
+    """Update an existing integration.  Only provided fields are changed.
+
+    When ``credentials`` is supplied the stored value is replaced in full
+    with the freshly encrypted version of the new credentials dict.
+    """
+    integration = (
+        db.query(UserIntegration)
+        .filter(UserIntegration.id == integration_id, UserIntegration.owner_id == owner_id)
+        .first()
+    )
+    if not integration:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found")
+
+    if body.name is not None:
+        integration.name = body.name
+    if body.config is not None:
+        integration.config = json.dumps(body.config)
+    if body.credentials is not None:
+        integration.credentials = _encode_credentials(body.credentials)
+    if body.is_active is not None:
+        integration.is_active = body.is_active
+
+    # Reset last_error so the next operation gives a fresh result
+    integration.last_error = None
+
+    try:
+        db.commit()
+        db.refresh(integration)
+    except Exception:
+        db.rollback()
+        raise
+
+    logger.info("User %s updated integration %d", owner_id, integration_id)
+    return _to_response(integration)
+
+
+@router.delete("/{integration_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an integration")
+def delete_integration(
+    integration_id: int,
+    request: Request,
+    db: DbSession,
+    owner_id: CurrentOwner,
+) -> None:
+    """Delete an integration permanently."""
+    integration = (
+        db.query(UserIntegration)
+        .filter(UserIntegration.id == integration_id, UserIntegration.owner_id == owner_id)
+        .first()
+    )
+    if not integration:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found")
+
+    try:
+        db.delete(integration)
+        db.commit()
+    except Exception:
+        db.rollback()
+        raise
+
+    logger.info("User %s deleted integration %d", owner_id, integration_id)
+
+
+@router.get("/{integration_id}/credentials", summary="Retrieve decrypted credentials for an integration")
+def get_integration_credentials(
+    integration_id: int,
+    request: Request,
+    db: DbSession,
+    owner_id: CurrentOwner,
+) -> dict[str, Any]:
+    """Return the decrypted credentials dict for a saved integration.
+
+    This endpoint is intended for internal use by background tasks that need
+    to authenticate with a third-party service.  Treat the response as
+    sensitive — it contains plaintext secrets.
+    """
+    integration = (
+        db.query(UserIntegration)
+        .filter(UserIntegration.id == integration_id, UserIntegration.owner_id == owner_id)
+        .first()
+    )
+    if not integration:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found")
+
+    credentials = _decode_credentials(integration.credentials)
+    return {"credentials": credentials or {}}
@@ -402,9 +402,11 @@ class UserImapAccount(Base):
    host = Column(String(255), nullable=False)
    port = Column(Integer, nullable=False, default=993)
    username = Column(String(255), nullable=False)
-    # Password stored in plain text — the admin is responsible for access control.
-    # TODO: Encrypt at rest using cryptography.fernet before deploying in high-security
-    # environments. See SECURITY_AUDIT.md for full risk assessment and mitigation notes.
+    # Password stored encrypted using Fernet symmetric encryption via
+    # app.utils.encryption.encrypt_value / decrypt_value (keyed from SESSION_SECRET).
+    # New records are always encrypted; legacy plaintext records are transparently
+    # handled by decrypt_value which returns the value unchanged when no "enc:" prefix
+    # is present.
    password = Column(String(1024), nullable=False)
    use_ssl = Column(Boolean, nullable=False, default=True)

@@ -467,3 +469,134 @@ class BackupRecord(Base):
    remote_path = Column(String(1024), nullable=True)

    created_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
+
+
+# ---------------------------------------------------------------------------
+# Integration direction / type constants
+# ---------------------------------------------------------------------------
+
+
+class IntegrationDirection:
+    """Direction of data flow for a UserIntegration."""
+
+    SOURCE = "SOURCE"
+    DESTINATION = "DESTINATION"
+
+    ALL = {SOURCE, DESTINATION}
+
+
+class IntegrationType:
+    """Supported integration types for UserIntegration."""
+
+    # Source integrations (ingestion)
+    IMAP = "IMAP"
+    WATCH_FOLDER = "WATCH_FOLDER"
+    WEBHOOK = "WEBHOOK"
+
+    # Destination integrations (storage / output)
+    S3 = "S3"
+    DROPBOX = "DROPBOX"
+    GOOGLE_DRIVE = "GOOGLE_DRIVE"
+    ONEDRIVE = "ONEDRIVE"
+    WEBDAV = "WEBDAV"
+    NEXTCLOUD = "NEXTCLOUD"
+    FTP = "FTP"
+    SFTP = "SFTP"
+    EMAIL = "EMAIL"
+    PAPERLESS = "PAPERLESS"
+    RCLONE = "RCLONE"
+
+    ALL = {
+        IMAP,
+        WATCH_FOLDER,
+        WEBHOOK,
+        S3,
+        DROPBOX,
+        GOOGLE_DRIVE,
+        ONEDRIVE,
+        WEBDAV,
+        NEXTCLOUD,
+        FTP,
+        SFTP,
+        EMAIL,
+        PAPERLESS,
+        RCLONE,
+    }
+
+
+class UserIntegration(Base):
+    """Generic per-user integration record (source or destination).
+
+    Replaces ad-hoc per-integration-type tables with a single, extensible
+    model that supports any combination of ingestion sources and storage
+    destinations without schema changes when new integrations are added.
+
+    ``config`` holds non-sensitive connection settings as a JSON string
+    (e.g. host, port, bucket name, folder path).
+
+    ``credentials`` holds sensitive secrets (passwords, tokens, API keys)
+    as a Fernet-encrypted JSON string.  Always use
+    ``app.utils.encryption.encrypt_value`` / ``decrypt_value`` when
+    writing / reading this field.
+
+    Example config + credentials shapes by integration type:
+
+    IMAP:
+      config      = {"host": "imap.example.com", "port": 993,
+                     "username": "user@example.com", "use_ssl": true,
+                     "delete_after_process": false}
+      credentials = {"password": "secret"}
+
+    S3:
+      config      = {"bucket": "my-bucket", "region": "us-east-1",
+                     "endpoint_url": null, "folder_prefix": ""}
+      credentials = {"access_key_id": "AKI…", "secret_access_key": "…"}
+
+    DROPBOX:
+      config      = {"folder": "/DocuElevate"}
+      credentials = {"refresh_token": "…", "app_key": "…",
+                     "app_secret": "…"}
+
+    GOOGLE_DRIVE:
+      config      = {"folder_id": "1abc…"}
+      credentials = {"credentials_json": "{…service-account or OAuth…}"}
+
+    WEBDAV / NEXTCLOUD:
+      config      = {"url": "https://cloud.example.com/dav/",
+                     "folder": "/Documents"}
+      credentials = {"username": "user", "password": "secret"}
+    """
+
+    __tablename__ = "user_integrations"
+
+    id = Column(Integer, primary_key=True, index=True)
+
+    # Stable owner identifier — matches FileRecord.owner_id / UserImapAccount.owner_id
+    owner_id = Column(String, nullable=False, index=True)
+
+    # "SOURCE" or "DESTINATION"  (see IntegrationDirection)
+    direction = Column(String(20), nullable=False, index=True)
+
+    # One of the IntegrationType constants (e.g. "IMAP", "S3", "DROPBOX")
+    integration_type = Column(String(50), nullable=False, index=True)
+
+    # Human-readable label chosen by the user (e.g. "Work Gmail", "S3 Archive")
+    name = Column(String(255), nullable=False)
+
+    # Non-sensitive connection configuration (JSON string)
+    config = Column(Text, nullable=True)
+
+    # Sensitive credentials — always stored encrypted via encrypt_value()
+    credentials = Column(Text, nullable=True)
+
+    # When False the integration is not polled / used by background tasks
+    is_active = Column(Boolean, nullable=False, default=True)
+
+    # Timestamp of the last successful use of this integration
+    last_used_at = Column(DateTime(timezone=True), nullable=True)
+
+    # Last error message if the most recent operation failed (NULL = last op succeeded)
+    last_error = Column(Text, nullable=True)
+
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
@@ -37,6 +37,19 @@ logger = logging.getLogger(__name__)
 # Initialize Redis connection using Celery's Redis settings
 redis_client = redis.StrictRedis.from_url(settings.redis_url, decode_responses=True)

+
+def _decrypt_imap_password(password: str | None) -> str | None:
+    """Decrypt an IMAP account password stored in the database.
+
+    Passwords are stored encrypted (Fernet, ``enc:`` prefix) for new records;
+    legacy plaintext records are returned unchanged so existing accounts
+    continue to work until they are next updated via the API.
+    """
+    from app.utils.encryption import decrypt_value
+
+    return decrypt_value(password)
+
+
 LOCK_KEY = "imap_lock"  # Unique key for locking
 LOCK_EXPIRE = 300  # Lock expires in 5 minutes

@@ -160,7 +173,7 @@ def _pull_user_imap_accounts() -> None:
                        host=acct.host,
                        port=acct.port,
                        username=acct.username,
-                        password=acct.password,
+                        password=_decrypt_imap_password(acct.password),
                        use_ssl=acct.use_ssl,
                        delete_after_process=acct.delete_after_process,
                    )