feat(imap): add ImapIngestionProfile model, API, migration and UI
Replaces the simple binary attachment_filter string with a full ingestion profiles system: - Add FILE_TYPE_CATEGORIES dict to allowed_types.py (6 categories: pdf, office, opendocument, text, web, images) + DEFAULT_CATEGORIES / ALL_CATEGORIES + get_allowed_types_for_categories() helper - Add ImapIngestionProfile model (id, name, description, owner_id, allowed_categories JSON, is_builtin) - Update UserImapAccount: replace attachment_filter string with profile_id FK to imap_ingestion_profiles - Migration 033: creates profiles table, seeds 2 built-in profiles (Documents Only, All Files), migrates attachment_filter → profile_id - New /api/imap-profiles/ CRUD endpoints (list, create, get, update, delete) with category validation - Register imap_profiles router in app/api/__init__.py - Update imap_tasks.py: replace attachment_filter string param with profile-based allowed_categories; add _resolve_categories_for_profile() - Update imap_accounts.py API to use profile_id instead of attachment_filter - Update imap_accounts view to pass profiles + categories to template - Full UI overhaul: profiles panel + profile create/edit modal with category checkboxes; profile selector in account modal - 17 new tests (141 total), all passing Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
@@ -21,6 +21,7 @@ from app.api.files import router as files_router
|
||||
from app.api.google_drive import router as google_drive_router
|
||||
from app.api.i18n import router as i18n_router
|
||||
from app.api.imap_accounts import router as imap_accounts_router
|
||||
from app.api.imap_profiles import router as imap_profiles_router
|
||||
from app.api.integrations import router as integrations_router
|
||||
from app.api.logs import router as logs_router
|
||||
from app.api.mobile import router as mobile_router
|
||||
@@ -83,6 +84,7 @@ router.include_router(onboarding_router)
|
||||
router.include_router(billing_router)
|
||||
router.include_router(pipelines_router)
|
||||
router.include_router(imap_accounts_router)
|
||||
router.include_router(imap_profiles_router)
|
||||
router.include_router(integrations_router)
|
||||
router.include_router(notifications_router)
|
||||
router.include_router(scheduled_jobs_router)
|
||||
|
||||
+11
-17
@@ -110,12 +110,10 @@ class ImapAccountCreate(BaseModel):
|
||||
use_ssl: bool = Field(default=True, description="Use SSL/TLS connection")
|
||||
delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing")
|
||||
is_active: bool = Field(default=True, description="Whether to poll this mailbox")
|
||||
attachment_filter: str | None = Field(
|
||||
profile_id: int | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Controls which attachment types to ingest. "
|
||||
"'documents_only' – PDFs and office files only (default when None). "
|
||||
"'all' – all supported types including images. "
|
||||
"ID of the ImapIngestionProfile that controls which attachment types to ingest. "
|
||||
"Null inherits the global imap_attachment_filter setting."
|
||||
),
|
||||
)
|
||||
@@ -132,13 +130,11 @@ class ImapAccountUpdate(BaseModel):
|
||||
use_ssl: bool | None = None
|
||||
delete_after_process: bool | None = None
|
||||
is_active: bool | None = None
|
||||
attachment_filter: str | None = Field(
|
||||
profile_id: int | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Controls which attachment types to ingest. "
|
||||
"'documents_only' – PDFs and office files only. "
|
||||
"'all' – all supported types including images. "
|
||||
"Null or empty string clears the override (inherits global setting)."
|
||||
"ID of the ImapIngestionProfile to use. "
|
||||
"Explicitly sending null clears the override (falls back to global setting)."
|
||||
),
|
||||
)
|
||||
|
||||
@@ -173,7 +169,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]:
|
||||
"use_ssl": acct.use_ssl,
|
||||
"delete_after_process": acct.delete_after_process,
|
||||
"is_active": acct.is_active,
|
||||
"attachment_filter": acct.attachment_filter,
|
||||
"profile_id": acct.profile_id,
|
||||
"last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None,
|
||||
"last_error": acct.last_error,
|
||||
"created_at": acct.created_at.isoformat() if acct.created_at else None,
|
||||
@@ -241,7 +237,7 @@ def create_imap_account(
|
||||
use_ssl=body.use_ssl,
|
||||
delete_after_process=body.delete_after_process,
|
||||
is_active=body.is_active,
|
||||
attachment_filter=body.attachment_filter or None,
|
||||
profile_id=body.profile_id,
|
||||
)
|
||||
try:
|
||||
db.add(acct)
|
||||
@@ -297,12 +293,10 @@ def update_imap_account(
|
||||
acct.delete_after_process = body.delete_after_process
|
||||
if body.is_active is not None:
|
||||
acct.is_active = body.is_active
|
||||
# attachment_filter uses a sentinel check: the field is always present in the
|
||||
# model (defaulting to None in Pydantic) so we update it unconditionally when
|
||||
# the caller sends any value (including explicit null to clear the override).
|
||||
# An empty string is normalised to None to avoid storing a non-meaningful value.
|
||||
if "attachment_filter" in body.model_fields_set:
|
||||
acct.attachment_filter = body.attachment_filter or None
|
||||
# profile_id: update whenever the field is explicitly present in the request payload
|
||||
# (including sending null to clear the override).
|
||||
if "profile_id" in body.model_fields_set:
|
||||
acct.profile_id = body.profile_id
|
||||
|
||||
# Reset last_error so the next poll gives a fresh result
|
||||
acct.last_error = None
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
"""API endpoints for managing IMAP ingestion profiles.
|
||||
|
||||
Ingestion profiles allow fine-grained control over which attachment types are
|
||||
accepted when ingesting emails via IMAP. Each profile carries a list of enabled
|
||||
file-type categories (e.g. ``["pdf", "office", "images"]``) drawn from the
|
||||
canonical set defined in :mod:`app.utils.allowed_types`.
|
||||
|
||||
Built-in system profiles (``is_builtin=True``) are read-only and cannot be
|
||||
deleted or modified. Users may create their own profiles which are private to
|
||||
their ``owner_id``. System-level global profiles (``owner_id=None``) are visible
|
||||
to all users but can only be created by administrators.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import ImapIngestionProfile
|
||||
from app.utils.allowed_types import FILE_TYPE_CATEGORIES
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/imap-profiles", tags=["imap-profiles"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_owner_id(request: Request) -> str:
|
||||
"""Return the current user's owner ID, raising 401 if unauthenticated."""
|
||||
owner_id = get_current_owner_id(request)
|
||||
if owner_id is None:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
return owner_id
|
||||
|
||||
|
||||
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_VALID_CATEGORIES = set(FILE_TYPE_CATEGORIES.keys())
|
||||
|
||||
|
||||
class ImapProfileCreate(BaseModel):
|
||||
"""Schema for creating a new ingestion profile."""
|
||||
|
||||
name: str = Field(..., min_length=1, max_length=255, description="Human-readable profile name")
|
||||
description: str | None = Field(default=None, description="Optional description")
|
||||
allowed_categories: list[str] = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description=(f"List of enabled file-type category keys. Valid values: {sorted(_VALID_CATEGORIES)}"),
|
||||
)
|
||||
|
||||
|
||||
class ImapProfileUpdate(BaseModel):
|
||||
"""Schema for updating an existing profile (all fields optional)."""
|
||||
|
||||
name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
description: str | None = None
|
||||
allowed_categories: list[str] | None = Field(default=None, min_length=1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _validate_categories(categories: list[str]) -> list[str]:
|
||||
"""Raise 422 if any category key is unknown; return the cleaned list."""
|
||||
unknown = [c for c in categories if c not in _VALID_CATEGORIES]
|
||||
if unknown:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Unknown category key(s): {unknown}. Valid keys: {sorted(_VALID_CATEGORIES)}",
|
||||
)
|
||||
# Deduplicate while preserving order
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for cat in categories:
|
||||
if cat not in seen:
|
||||
seen.add(cat)
|
||||
result.append(cat)
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Serialisation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _to_response(profile: ImapIngestionProfile) -> dict[str, Any]:
|
||||
"""Serialize a profile row to a response dict."""
|
||||
try:
|
||||
categories = json.loads(profile.allowed_categories)
|
||||
except (ValueError, TypeError):
|
||||
categories = []
|
||||
|
||||
# Enrich categories with display metadata
|
||||
categories_detail = [
|
||||
{
|
||||
"key": cat,
|
||||
"label": FILE_TYPE_CATEGORIES[cat]["label"] if cat in FILE_TYPE_CATEGORIES else cat,
|
||||
"description": FILE_TYPE_CATEGORIES[cat]["description"] if cat in FILE_TYPE_CATEGORIES else "",
|
||||
}
|
||||
for cat in categories
|
||||
]
|
||||
|
||||
return {
|
||||
"id": profile.id,
|
||||
"name": profile.name,
|
||||
"description": profile.description,
|
||||
"owner_id": profile.owner_id,
|
||||
"allowed_categories": categories,
|
||||
"categories_detail": categories_detail,
|
||||
"is_builtin": profile.is_builtin,
|
||||
"created_at": profile.created_at.isoformat() if profile.created_at else None,
|
||||
"updated_at": profile.updated_at.isoformat() if profile.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/categories", summary="List available file-type categories")
|
||||
def list_categories(request: Request, owner_id: CurrentOwner) -> list[dict[str, Any]]:
|
||||
"""Return the full list of file-type categories that can be used in profiles."""
|
||||
return [
|
||||
{
|
||||
"key": key,
|
||||
"label": info["label"],
|
||||
"description": info["description"],
|
||||
}
|
||||
for key, info in FILE_TYPE_CATEGORIES.items()
|
||||
]
|
||||
|
||||
|
||||
@router.get("/", summary="List ingestion profiles visible to the current user")
|
||||
def list_profiles(request: Request, db: DbSession, owner_id: CurrentOwner) -> list[dict[str, Any]]:
|
||||
"""Return all profiles: system-global (owner_id=NULL) and the user's own profiles."""
|
||||
profiles = (
|
||||
db.query(ImapIngestionProfile)
|
||||
.filter(
|
||||
(ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711
|
||||
)
|
||||
.order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
|
||||
.all()
|
||||
)
|
||||
return [_to_response(p) for p in profiles]
|
||||
|
||||
|
||||
@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new ingestion profile")
|
||||
def create_profile(request: Request, body: ImapProfileCreate, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
|
||||
"""Create a new ingestion profile owned by the current user."""
|
||||
categories = _validate_categories(body.allowed_categories)
|
||||
|
||||
profile = ImapIngestionProfile(
|
||||
name=body.name,
|
||||
description=body.description,
|
||||
owner_id=owner_id,
|
||||
allowed_categories=json.dumps(categories),
|
||||
is_builtin=False,
|
||||
)
|
||||
try:
|
||||
db.add(profile)
|
||||
db.commit()
|
||||
db.refresh(profile)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s created IMAP ingestion profile %d ('%s')", owner_id, profile.id, body.name)
|
||||
return _to_response(profile)
|
||||
|
||||
|
||||
@router.get("/{profile_id}", summary="Get a single ingestion profile")
|
||||
def get_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
|
||||
"""Return a single profile by ID. Only the owner or system profiles are accessible."""
|
||||
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
|
||||
return _to_response(profile)
|
||||
|
||||
|
||||
@router.put("/{profile_id}", summary="Update an ingestion profile")
|
||||
def update_profile(
|
||||
profile_id: int,
|
||||
request: Request,
|
||||
body: ImapProfileUpdate,
|
||||
db: DbSession,
|
||||
owner_id: CurrentOwner,
|
||||
) -> dict[str, Any]:
|
||||
"""Update an existing ingestion profile. Built-in profiles cannot be modified."""
|
||||
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
|
||||
if profile.is_builtin:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Built-in profiles cannot be modified.",
|
||||
)
|
||||
|
||||
if body.name is not None:
|
||||
profile.name = body.name
|
||||
if "description" in body.model_fields_set:
|
||||
profile.description = body.description
|
||||
if body.allowed_categories is not None:
|
||||
categories = _validate_categories(body.allowed_categories)
|
||||
profile.allowed_categories = json.dumps(categories)
|
||||
|
||||
profile.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(profile)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s updated IMAP ingestion profile %d", owner_id, profile_id)
|
||||
return _to_response(profile)
|
||||
|
||||
|
||||
@router.delete("/{profile_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an ingestion profile")
|
||||
def delete_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> None:
|
||||
"""Delete an ingestion profile. Built-in profiles cannot be deleted."""
|
||||
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
|
||||
if profile.is_builtin:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Built-in profiles cannot be deleted.",
|
||||
)
|
||||
|
||||
try:
|
||||
db.delete(profile)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s deleted IMAP ingestion profile %d", owner_id, profile_id)
|
||||
+45
-4
@@ -401,6 +401,48 @@ class PipelineStep(Base):
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class ImapIngestionProfile(Base):
|
||||
"""Named ingestion profile controlling which attachment types are accepted from IMAP emails.
|
||||
|
||||
Profiles group file-type categories (e.g. "pdf", "office", "images") so users
|
||||
can precisely control what gets ingested from each mailbox.
|
||||
|
||||
System-provided built-in profiles (``is_builtin=True``) are seeded by the
|
||||
migration and cannot be deleted or renamed. Users may create their own profiles
|
||||
(``owner_id`` set to their identifier) or rely on the global system profiles
|
||||
(``owner_id=None``).
|
||||
|
||||
``allowed_categories`` stores a JSON list of category strings, e.g.::
|
||||
|
||||
'["pdf", "office", "opendocument", "text", "web"]'
|
||||
|
||||
Valid category names are defined in ``app.utils.allowed_types.FILE_TYPE_CATEGORIES``.
|
||||
"""
|
||||
|
||||
__tablename__ = "imap_ingestion_profiles"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
|
||||
# Human-readable profile name (e.g. "Documents Only", "Documents + Images")
|
||||
name = Column(String(255), nullable=False)
|
||||
|
||||
# Optional description shown in the UI
|
||||
description = Column(Text, nullable=True)
|
||||
|
||||
# Owner of this profile. NULL = global/system profile available to all users.
|
||||
owner_id = Column(String, nullable=True, index=True)
|
||||
|
||||
# JSON-encoded list of enabled category keys. Example: '["pdf","office","text"]'
|
||||
# See FILE_TYPE_CATEGORIES in app/utils/allowed_types.py for valid values.
|
||||
allowed_categories = Column(Text, nullable=False, default='["pdf","office","opendocument","text","web"]')
|
||||
|
||||
# Built-in system profiles that cannot be deleted or modified via the API.
|
||||
is_builtin = Column(Boolean, nullable=False, default=False)
|
||||
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class UserImapAccount(Base):
|
||||
"""Per-user IMAP ingestion account.
|
||||
|
||||
@@ -439,10 +481,9 @@ class UserImapAccount(Base):
|
||||
# When True, emails are deleted from the mailbox after their attachments are processed
|
||||
delete_after_process = Column(Boolean, nullable=False, default=False)
|
||||
|
||||
# Override for which attachment types to ingest.
|
||||
# NULL means "inherit the global imap_attachment_filter setting".
|
||||
# Allowed values: 'documents_only', 'all'
|
||||
attachment_filter = Column(String(50), nullable=True, default=None)
|
||||
# Optional reference to an ImapIngestionProfile.
|
||||
# NULL means "use the global imap_attachment_filter setting" (system default).
|
||||
profile_id = Column(Integer, ForeignKey("imap_ingestion_profiles.id"), nullable=True)
|
||||
|
||||
# When False the account is not polled by the periodic task (but not deleted)
|
||||
is_active = Column(Boolean, nullable=False, default=True)
|
||||
|
||||
+79
-71
@@ -13,7 +13,11 @@ from celery import shared_task
|
||||
from app.config import settings
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
|
||||
from app.tasks.process_document import process_document # Updated import
|
||||
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, DOCUMENT_MIME_TYPES, IMAGE_MIME_TYPES
|
||||
from app.utils.allowed_types import (
|
||||
ALL_CATEGORIES,
|
||||
DEFAULT_CATEGORIES,
|
||||
get_allowed_types_for_categories,
|
||||
)
|
||||
|
||||
# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
|
||||
_db_session_factory = None
|
||||
@@ -50,6 +54,33 @@ def _decrypt_imap_password(password: str | None) -> str | None:
|
||||
return decrypt_value(password)
|
||||
|
||||
|
||||
def _resolve_categories_for_profile(profile_id: int | None) -> list[str]:
|
||||
"""Return the list of allowed categories for a profile ID.
|
||||
|
||||
Loads the profile from the database. If ``profile_id`` is ``None`` or the
|
||||
profile is not found, falls back to the global ``settings.imap_attachment_filter``
|
||||
string (``'documents_only'`` → default categories; ``'all'`` → all categories).
|
||||
"""
|
||||
if profile_id is not None:
|
||||
try:
|
||||
from app.models import ImapIngestionProfile
|
||||
|
||||
db = _get_db_session()
|
||||
try:
|
||||
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||
if profile:
|
||||
return json.loads(profile.allowed_categories)
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Could not load IMAP ingestion profile %d: %s — using global default", profile_id, exc)
|
||||
|
||||
# Fall back to global setting
|
||||
if settings.imap_attachment_filter == "all":
|
||||
return ALL_CATEGORIES
|
||||
return DEFAULT_CATEGORIES
|
||||
|
||||
|
||||
LOCK_KEY = "imap_lock" # Unique key for locking
|
||||
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
|
||||
|
||||
@@ -180,7 +211,7 @@ def _pull_user_imap_accounts() -> None:
|
||||
use_ssl=acct.use_ssl,
|
||||
delete_after_process=acct.delete_after_process,
|
||||
owner_id=acct.owner_id,
|
||||
attachment_filter=acct.attachment_filter or settings.imap_attachment_filter,
|
||||
allowed_categories=_resolve_categories_for_profile(acct.profile_id),
|
||||
)
|
||||
# Record successful poll
|
||||
acct.last_checked_at = datetime.now(timezone.utc)
|
||||
@@ -251,7 +282,9 @@ def _pull_user_integration_imap() -> None:
|
||||
use_ssl = cfg.get("use_ssl", True)
|
||||
delete_after = cfg.get("delete_after_process", False)
|
||||
gmail_labels = cfg.get("gmail_apply_labels", True)
|
||||
attachment_filter = cfg.get("attachment_filter") or settings.imap_attachment_filter
|
||||
# Integrations can store a profile_id in config; fall back to global default
|
||||
profile_id = cfg.get("profile_id")
|
||||
allowed_categories = _resolve_categories_for_profile(profile_id)
|
||||
|
||||
if not (host and username and password):
|
||||
logger.warning(
|
||||
@@ -271,7 +304,7 @@ def _pull_user_integration_imap() -> None:
|
||||
delete_after_process=delete_after,
|
||||
owner_id=integ.owner_id,
|
||||
gmail_apply_labels=gmail_labels,
|
||||
attachment_filter=attachment_filter,
|
||||
allowed_categories=allowed_categories,
|
||||
)
|
||||
integ.last_used_at = datetime.now(timezone.utc)
|
||||
integ.last_error = None
|
||||
@@ -332,7 +365,7 @@ def pull_inbox(
|
||||
delete_after_process,
|
||||
owner_id=None,
|
||||
gmail_apply_labels=True,
|
||||
attachment_filter=None,
|
||||
allowed_categories=None,
|
||||
):
|
||||
"""
|
||||
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
|
||||
@@ -349,13 +382,22 @@ def pull_inbox(
|
||||
attributed to this user via ``process_document`` / ``convert_to_pdf``.
|
||||
gmail_apply_labels: Whether to apply Gmail-specific labels and stars to
|
||||
processed emails. Only relevant for Gmail hosts. Defaults to True.
|
||||
attachment_filter: Controls which attachment types to ingest.
|
||||
``'documents_only'`` (default) – PDFs and office files only.
|
||||
``'all'`` – all supported types including images.
|
||||
``None`` falls back to the global ``settings.imap_attachment_filter``.
|
||||
allowed_categories: List of file-type category keys to ingest (e.g.
|
||||
``["pdf", "office", "images"]``). ``None`` falls back to the
|
||||
global ``settings.imap_attachment_filter`` mapping.
|
||||
"""
|
||||
resolved_filter = attachment_filter or settings.imap_attachment_filter
|
||||
logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
|
||||
if allowed_categories is None:
|
||||
allowed_categories = _resolve_categories_for_profile(None)
|
||||
|
||||
effective_mime_types, effective_extensions = get_allowed_types_for_categories(allowed_categories)
|
||||
logger.info(
|
||||
"Connecting to %s at %s:%s (SSL=%s) — categories: %s",
|
||||
mailbox_key,
|
||||
host,
|
||||
port,
|
||||
use_ssl,
|
||||
allowed_categories,
|
||||
)
|
||||
processed_emails = load_processed_emails()
|
||||
|
||||
try:
|
||||
@@ -414,9 +456,13 @@ def pull_inbox(
|
||||
logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
|
||||
continue
|
||||
|
||||
# Process attachments (and convert non-PDF files).
|
||||
# We call the function without assigning its return value since it is not used.
|
||||
fetch_attachments_and_enqueue(email_message, owner_id=owner_id, attachment_filter=resolved_filter)
|
||||
# Process attachments using the resolved mime types / extensions.
|
||||
fetch_attachments_and_enqueue(
|
||||
email_message,
|
||||
owner_id=owner_id,
|
||||
effective_mime_types=effective_mime_types,
|
||||
effective_extensions=effective_extensions,
|
||||
)
|
||||
|
||||
if settings.imap_readonly_mode:
|
||||
logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
|
||||
@@ -448,19 +494,20 @@ def pull_inbox(
|
||||
def fetch_attachments_and_enqueue(
|
||||
email_message,
|
||||
owner_id: str | None = None,
|
||||
attachment_filter: str | None = None,
|
||||
effective_mime_types: frozenset[str] | None = None,
|
||||
effective_extensions: frozenset[str] | None = None,
|
||||
):
|
||||
"""
|
||||
Extracts attachments from the email and processes only allowed file types.
|
||||
|
||||
Files are first checked against the ``attachment_filter`` to determine which
|
||||
broad categories are permitted, then validated against known MIME types /
|
||||
extensions.
|
||||
The caller is responsible for computing ``effective_mime_types`` and
|
||||
``effective_extensions`` from the relevant :class:`ImapIngestionProfile` (or
|
||||
the global default) via :func:`app.utils.allowed_types.get_allowed_types_for_categories`
|
||||
before calling this function. ``pull_inbox`` does this automatically.
|
||||
|
||||
Attachment filter values:
|
||||
- ``'documents_only'`` (default): PDFs, office files (Word, Excel, PowerPoint,
|
||||
OpenDocument, RTF), plain text, CSV, HTML, and Markdown. Images are skipped.
|
||||
- ``'all'``: All supported file types, including images (JPEG, PNG, GIF, etc.).
|
||||
If either set is ``None`` the function falls back to the default category list
|
||||
so the function still works correctly when called directly in tests or from
|
||||
other contexts.
|
||||
|
||||
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
|
||||
any other allowed file is enqueued for conversion to PDF.
|
||||
@@ -469,36 +516,13 @@ def fetch_attachments_and_enqueue(
|
||||
email_message: The parsed email message to extract attachments from.
|
||||
owner_id: Optional user identifier forwarded to ``process_document`` /
|
||||
``convert_to_pdf`` for multi-tenant attribution.
|
||||
attachment_filter: Override for the filter level. Defaults to
|
||||
``settings.imap_attachment_filter`` when not provided.
|
||||
effective_mime_types: Pre-computed frozenset of allowed MIME type strings.
|
||||
effective_extensions: Pre-computed frozenset of allowed file extension strings.
|
||||
|
||||
Returns True if at least one allowed attachment was processed.
|
||||
"""
|
||||
resolved_filter = attachment_filter or settings.imap_attachment_filter
|
||||
|
||||
# Build the effective allowed MIME type set based on the filter
|
||||
if resolved_filter == "all":
|
||||
effective_mime_types = ALLOWED_MIME_TYPES
|
||||
else:
|
||||
# 'documents_only' (and any unrecognised value): exclude images
|
||||
effective_mime_types = DOCUMENT_MIME_TYPES
|
||||
|
||||
# Build the effective allowed extensions set (images excluded for documents_only)
|
||||
if resolved_filter == "all":
|
||||
effective_extensions = ALLOWED_EXTENSIONS
|
||||
else:
|
||||
image_extensions = {
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".png",
|
||||
".gif",
|
||||
".bmp",
|
||||
".tiff",
|
||||
".tif",
|
||||
".webp",
|
||||
".svg",
|
||||
}
|
||||
effective_extensions = ALLOWED_EXTENSIONS - image_extensions
|
||||
if effective_mime_types is None or effective_extensions is None:
|
||||
effective_mime_types, effective_extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
|
||||
|
||||
has_attachment = False
|
||||
for part in email_message.walk():
|
||||
@@ -515,30 +539,14 @@ def fetch_attachments_and_enqueue(
|
||||
mime_type = part.get_content_type()
|
||||
file_ext = os.path.splitext(filename)[1].lower()
|
||||
|
||||
# Skip images when filter is documents_only
|
||||
is_image = mime_type in IMAGE_MIME_TYPES or file_ext in {
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".png",
|
||||
".gif",
|
||||
".bmp",
|
||||
".tiff",
|
||||
".tif",
|
||||
".webp",
|
||||
".svg",
|
||||
}
|
||||
if is_image and resolved_filter != "all":
|
||||
logger.info(
|
||||
"Skipping image attachment %s (MIME: %s) — attachment_filter=%s",
|
||||
filename,
|
||||
mime_type,
|
||||
resolved_filter,
|
||||
)
|
||||
continue
|
||||
|
||||
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
|
||||
if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension:
|
||||
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
|
||||
logger.info(
|
||||
"Skipping attachment %s (MIME: %s, ext: %s) — not in effective allowed set",
|
||||
filename,
|
||||
mime_type,
|
||||
file_ext,
|
||||
)
|
||||
continue
|
||||
|
||||
file_path = os.path.join(settings.workdir, filename)
|
||||
|
||||
@@ -131,3 +131,157 @@ ALLOWED_EXTENSIONS: set[str] = {
|
||||
".md",
|
||||
".markdown",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fine-grained file-type categories used by IMAP ingestion profiles.
|
||||
# Each category groups related MIME types and extensions so that users can
|
||||
# enable/disable a logical collection of formats (e.g. "images") rather than
|
||||
# having to manage individual MIME strings.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FILE_TYPE_CATEGORIES: dict[str, dict] = {
|
||||
"pdf": {
|
||||
"label": "PDF",
|
||||
"description": "PDF documents (.pdf)",
|
||||
"mime_types": frozenset({"application/pdf"}),
|
||||
"extensions": frozenset({".pdf"}),
|
||||
},
|
||||
"office": {
|
||||
"label": "Microsoft Office",
|
||||
"description": "Word, Excel and PowerPoint files (.doc, .docx, .xls, .xlsx, .ppt, .pptx, …)",
|
||||
"mime_types": frozenset(
|
||||
{
|
||||
"application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
|
||||
"application/vnd.ms-word.document.macroEnabled.12",
|
||||
"application/vnd.ms-word.template.macroEnabled.12",
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.template",
|
||||
"application/vnd.ms-excel.sheet.macroEnabled.12",
|
||||
"application/vnd.ms-excel.sheet.binary.macroEnabled.12",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.template",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.slideshow",
|
||||
"application/vnd.ms-powerpoint.presentation.macroEnabled.12",
|
||||
}
|
||||
),
|
||||
"extensions": frozenset(
|
||||
{
|
||||
".doc",
|
||||
".docx",
|
||||
".docm",
|
||||
".dot",
|
||||
".dotx",
|
||||
".dotm",
|
||||
".xls",
|
||||
".xlsx",
|
||||
".xlsm",
|
||||
".xlsb",
|
||||
".xlt",
|
||||
".xltx",
|
||||
".xlw",
|
||||
".ppt",
|
||||
".pptx",
|
||||
".pptm",
|
||||
".pps",
|
||||
".ppsx",
|
||||
".pot",
|
||||
".potx",
|
||||
}
|
||||
),
|
||||
},
|
||||
"opendocument": {
|
||||
"label": "OpenDocument (LibreOffice)",
|
||||
"description": "LibreOffice / OpenOffice files (.odt, .ods, .odp, …)",
|
||||
"mime_types": frozenset(
|
||||
{
|
||||
"application/vnd.oasis.opendocument.text",
|
||||
"application/vnd.oasis.opendocument.spreadsheet",
|
||||
"application/vnd.oasis.opendocument.presentation",
|
||||
"application/vnd.oasis.opendocument.graphics",
|
||||
"application/vnd.oasis.opendocument.formula",
|
||||
}
|
||||
),
|
||||
"extensions": frozenset({".odt", ".ods", ".odp", ".odg", ".odf"}),
|
||||
},
|
||||
"text": {
|
||||
"label": "Text & Data",
|
||||
"description": "Plain text, CSV and RTF files (.txt, .csv, .rtf)",
|
||||
"mime_types": frozenset(
|
||||
{
|
||||
"text/plain",
|
||||
"text/csv",
|
||||
"application/rtf",
|
||||
"text/rtf",
|
||||
}
|
||||
),
|
||||
"extensions": frozenset({".txt", ".csv", ".rtf"}),
|
||||
},
|
||||
"web": {
|
||||
"label": "Web & Markup",
|
||||
"description": "HTML and Markdown files (.html, .htm, .md, .markdown)",
|
||||
"mime_types": frozenset(
|
||||
{
|
||||
"text/html",
|
||||
"text/markdown",
|
||||
"text/x-markdown",
|
||||
}
|
||||
),
|
||||
"extensions": frozenset({".html", ".htm", ".md", ".markdown"}),
|
||||
},
|
||||
"images": {
|
||||
"label": "Images",
|
||||
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)",
|
||||
"mime_types": frozenset(
|
||||
{
|
||||
"image/jpeg",
|
||||
"image/jpg",
|
||||
"image/png",
|
||||
"image/gif",
|
||||
"image/bmp",
|
||||
"image/tiff",
|
||||
"image/webp",
|
||||
"image/svg+xml",
|
||||
}
|
||||
),
|
||||
"extensions": frozenset(
|
||||
{
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".png",
|
||||
".gif",
|
||||
".bmp",
|
||||
".tiff",
|
||||
".tif",
|
||||
".webp",
|
||||
".svg",
|
||||
}
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
# Default categories for the "documents only" built-in profile (no images)
|
||||
DEFAULT_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web"]
|
||||
# All categories including images
|
||||
ALL_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web", "images"]
|
||||
|
||||
|
||||
def get_allowed_types_for_categories(
|
||||
categories: list[str],
|
||||
) -> tuple[frozenset[str], frozenset[str]]:
|
||||
"""Return ``(mime_types, extensions)`` for the given category list.
|
||||
|
||||
Unknown category names are silently ignored so that future categories
|
||||
don't break existing profiles.
|
||||
"""
|
||||
mime_types: set[str] = set()
|
||||
extensions: set[str] = set()
|
||||
for cat in categories:
|
||||
info = FILE_TYPE_CATEGORIES.get(cat)
|
||||
if info:
|
||||
mime_types |= info["mime_types"]
|
||||
extensions |= info["extensions"]
|
||||
return frozenset(mime_types), frozenset(extensions)
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
"""User-facing view for the per-user IMAP ingestion dashboard."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from fastapi import Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import UserImapAccount
|
||||
from app.models import ImapIngestionProfile, UserImapAccount
|
||||
from app.utils.allowed_types import FILE_TYPE_CATEGORIES
|
||||
from app.utils.subscription import get_tier, get_user_tier_id
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
from app.views.base import APIRouter, Depends, get_db, require_login, templates
|
||||
@@ -25,6 +27,22 @@ def _get_max_mailboxes(tier: dict) -> int | None:
|
||||
return max_mb
|
||||
|
||||
|
||||
def _serialize_profile(profile: ImapIngestionProfile) -> dict:
|
||||
"""Serialize a profile for JSON embedding in the template."""
|
||||
try:
|
||||
categories = json.loads(profile.allowed_categories)
|
||||
except (ValueError, TypeError):
|
||||
categories = []
|
||||
return {
|
||||
"id": profile.id,
|
||||
"name": profile.name,
|
||||
"description": profile.description,
|
||||
"owner_id": profile.owner_id,
|
||||
"allowed_categories": categories,
|
||||
"is_builtin": profile.is_builtin,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/imap-accounts")
|
||||
@require_login
|
||||
async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
|
||||
@@ -49,11 +67,39 @@ async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
|
||||
max_mailboxes = _get_max_mailboxes(tier)
|
||||
can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes)
|
||||
|
||||
# Load ingestion profiles: system-global + user's own
|
||||
profiles: list[ImapIngestionProfile] = (
|
||||
db.query(ImapIngestionProfile)
|
||||
.filter(
|
||||
(ImapIngestionProfile.owner_id == None) # noqa: E711
|
||||
| (ImapIngestionProfile.owner_id == owner_id)
|
||||
)
|
||||
.order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
|
||||
.all()
|
||||
if owner_id
|
||||
else db.query(ImapIngestionProfile)
|
||||
.filter(ImapIngestionProfile.owner_id == None) # noqa: E711
|
||||
.order_by(ImapIngestionProfile.id)
|
||||
.all()
|
||||
)
|
||||
|
||||
# Category definitions for the UI checkbox builder
|
||||
categories = [
|
||||
{
|
||||
"key": key,
|
||||
"label": info["label"],
|
||||
"description": info["description"],
|
||||
}
|
||||
for key, info in FILE_TYPE_CATEGORIES.items()
|
||||
]
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"imap_accounts.html",
|
||||
{
|
||||
"request": request,
|
||||
"accounts": accounts,
|
||||
"profiles": [_serialize_profile(p) for p in profiles],
|
||||
"categories": categories,
|
||||
"current_count": current_count,
|
||||
"max_mailboxes": max_mailboxes,
|
||||
"can_add": can_add,
|
||||
|
||||
@@ -172,14 +172,9 @@
|
||||
<i class="fas fa-trash-alt mr-1" aria-hidden="true"></i>Delete after process
|
||||
</span>
|
||||
</template>
|
||||
<template x-if="acct.attachment_filter === 'all'">
|
||||
<template x-if="acct.profile_id">
|
||||
<span class="inline-flex items-center px-2 py-0.5 rounded bg-blue-50 text-blue-700">
|
||||
<i class="fas fa-paperclip mr-1" aria-hidden="true"></i>All attachments
|
||||
</span>
|
||||
</template>
|
||||
<template x-if="acct.attachment_filter === 'documents_only'">
|
||||
<span class="inline-flex items-center px-2 py-0.5 rounded bg-gray-50 text-gray-600">
|
||||
<i class="fas fa-file-alt mr-1" aria-hidden="true"></i>Documents only
|
||||
<i class="fas fa-filter mr-1" aria-hidden="true"></i><span x-text="profileName(acct.profile_id)"></span>
|
||||
</span>
|
||||
</template>
|
||||
<template x-if="acct.last_checked_at">
|
||||
@@ -398,22 +393,45 @@
|
||||
|
||||
<!-- Attachment filter -->
|
||||
<div>
|
||||
<label for="acct-attachment-filter" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
|
||||
Attachment Types to Ingest
|
||||
<label for="acct-profile" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
|
||||
Ingestion Profile
|
||||
</label>
|
||||
<select
|
||||
id="acct-attachment-filter"
|
||||
x-model="form.attachment_filter"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
|
||||
aria-describedby="acct-attachment-filter-hint"
|
||||
>
|
||||
<option value="">Use global default</option>
|
||||
<option value="documents_only">Documents only (PDF, Office, ODT, RTF, TXT, CSV) — no images</option>
|
||||
<option value="all">All supported types (including images)</option>
|
||||
</select>
|
||||
<p id="acct-attachment-filter-hint" class="mt-1 text-xs text-gray-400 dark:text-gray-500">
|
||||
Override the system-wide attachment filter for this mailbox. Leave blank to use the global default (documents only).
|
||||
<div class="flex gap-2 items-center">
|
||||
<select
|
||||
id="acct-profile"
|
||||
x-model.number="form.profile_id"
|
||||
class="flex-1 px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
|
||||
aria-describedby="acct-profile-hint"
|
||||
>
|
||||
<option :value="null">Use global default</option>
|
||||
<template x-for="profile in profiles" :key="profile.id">
|
||||
<option :value="profile.id" x-text="profile.name + (profile.is_builtin ? '' : ' (custom)')"></option>
|
||||
</template>
|
||||
</select>
|
||||
<button
|
||||
type="button"
|
||||
@click="openProfileModal(null)"
|
||||
class="inline-flex items-center px-3 py-2 text-sm font-medium rounded border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-blue-500"
|
||||
style="min-height:40px;"
|
||||
title="Create new profile"
|
||||
aria-label="Create new ingestion profile"
|
||||
>
|
||||
<i class="fas fa-plus" aria-hidden="true"></i>
|
||||
</button>
|
||||
</div>
|
||||
<p id="acct-profile-hint" class="mt-1 text-xs text-gray-400 dark:text-gray-500">
|
||||
Controls which attachment types are ingested. Leave blank to use the global default (documents only).
|
||||
<button type="button" @click="showProfilesSection = !showProfilesSection" class="underline hover:text-blue-600 focus:outline-none">
|
||||
Manage profiles
|
||||
</button>
|
||||
</p>
|
||||
<!-- Inline selected profile summary -->
|
||||
<template x-if="form.profile_id">
|
||||
<div class="mt-2 text-xs bg-blue-50 dark:bg-blue-900/20 rounded px-2 py-1.5 text-blue-800 dark:text-blue-200">
|
||||
<strong x-text="profileName(form.profile_id)"></strong>:
|
||||
<span x-text="profileCategorySummary(form.profile_id)"></span>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
|
||||
<!-- Test connection result -->
|
||||
@@ -524,10 +542,276 @@
|
||||
|
||||
</div>
|
||||
|
||||
<!-- ── Ingestion Profiles panel ─────────────────────────────────────────────── -->
|
||||
<div
|
||||
x-show="showProfilesSection"
|
||||
x-transition
|
||||
class="fixed inset-0 z-50 flex items-center justify-center bg-black/50 px-4"
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby="profiles-panel-title"
|
||||
@keydown.escape.window="showProfilesSection = false"
|
||||
style="display:none;"
|
||||
>
|
||||
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-2xl overflow-y-auto max-h-[90vh]" @click.stop>
|
||||
<div class="px-6 pt-6 pb-2 flex items-center justify-between">
|
||||
<h2 id="profiles-panel-title" class="text-lg font-semibold text-gray-900 dark:text-white flex items-center gap-2">
|
||||
<i class="fas fa-filter text-blue-500" aria-hidden="true"></i>
|
||||
Ingestion Profiles
|
||||
</h2>
|
||||
<button
|
||||
type="button"
|
||||
@click="showProfilesSection = false"
|
||||
class="text-gray-400 hover:text-gray-600 dark:hover:text-gray-200 focus:outline-none focus:ring-2 focus:ring-blue-500 rounded"
|
||||
aria-label="Close profiles panel"
|
||||
>
|
||||
<i class="fas fa-times text-xl" aria-hidden="true"></i>
|
||||
</button>
|
||||
</div>
|
||||
<div class="px-6 pb-6 pt-2">
|
||||
<p class="text-sm text-gray-500 dark:text-gray-400 mb-4">
|
||||
Profiles define which attachment types are ingested from emails. Assign a profile to each IMAP account for fine-grained control.
|
||||
</p>
|
||||
|
||||
<!-- Profile list -->
|
||||
<div class="space-y-3 mb-4">
|
||||
<template x-for="profile in profiles" :key="profile.id">
|
||||
<div class="border border-gray-100 dark:border-gray-700 rounded-lg p-4 flex items-start gap-3">
|
||||
<div class="flex-1 min-w-0">
|
||||
<div class="flex items-center gap-2 flex-wrap">
|
||||
<span class="font-medium text-sm text-gray-900 dark:text-white" x-text="profile.name"></span>
|
||||
<template x-if="profile.is_builtin">
|
||||
<span class="text-xs px-1.5 py-0.5 rounded bg-gray-100 dark:bg-gray-700 text-gray-500">built-in</span>
|
||||
</template>
|
||||
</div>
|
||||
<p class="text-xs text-gray-500 dark:text-gray-400 mt-0.5" x-text="profile.description || ''"></p>
|
||||
<!-- Category badges -->
|
||||
<div class="flex flex-wrap gap-1 mt-2">
|
||||
<template x-for="cat in profile.allowed_categories" :key="cat">
|
||||
<span class="inline-flex items-center px-2 py-0.5 rounded text-xs font-medium bg-blue-50 dark:bg-blue-900/30 text-blue-700 dark:text-blue-300">
|
||||
<span x-text="categoryLabel(cat)"></span>
|
||||
</span>
|
||||
</template>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex gap-1 shrink-0">
|
||||
<button
|
||||
type="button"
|
||||
@click="openProfileModal(profile)"
|
||||
:disabled="profile.is_builtin"
|
||||
:class="profile.is_builtin ? 'opacity-40 cursor-not-allowed' : ''"
|
||||
class="p-1.5 text-gray-400 hover:text-blue-600 focus:outline-none focus:ring-2 focus:ring-blue-500 rounded"
|
||||
:aria-label="`Edit profile ${profile.name}`"
|
||||
:title="profile.is_builtin ? 'Built-in profiles cannot be edited' : 'Edit profile'"
|
||||
>
|
||||
<i class="fas fa-edit text-sm" aria-hidden="true"></i>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
@click="confirmDeleteProfile(profile)"
|
||||
:disabled="profile.is_builtin"
|
||||
:class="profile.is_builtin ? 'opacity-40 cursor-not-allowed' : ''"
|
||||
class="p-1.5 text-gray-400 hover:text-red-600 focus:outline-none focus:ring-2 focus:ring-red-500 rounded"
|
||||
:aria-label="`Delete profile ${profile.name}`"
|
||||
:title="profile.is_builtin ? 'Built-in profiles cannot be deleted' : 'Delete profile'"
|
||||
>
|
||||
<i class="fas fa-trash-alt text-sm" aria-hidden="true"></i>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
@click="openProfileModal(null)"
|
||||
class="inline-flex items-center px-4 py-2 text-sm font-medium rounded bg-blue-600 hover:bg-blue-700 text-white focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
style="min-height:40px;"
|
||||
>
|
||||
<i class="fas fa-plus mr-2" aria-hidden="true"></i> New Profile
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ── Profile create / edit modal ────────────────────────────────────────────── -->
|
||||
<div
|
||||
x-show="profileModalOpen"
|
||||
x-transition:enter="ease-out duration-200"
|
||||
x-transition:enter-start="opacity-0"
|
||||
x-transition:enter-end="opacity-100"
|
||||
x-transition:leave="ease-in duration-150"
|
||||
x-transition:leave-start="opacity-100"
|
||||
x-transition:leave-end="opacity-0"
|
||||
class="fixed inset-0 z-[60] flex items-center justify-center bg-black/50 px-4"
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
:aria-labelledby="editingProfile ? 'profile-modal-title-edit' : 'profile-modal-title-create'"
|
||||
@keydown.escape.window="closeProfileModal()"
|
||||
style="display:none;"
|
||||
>
|
||||
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-lg overflow-y-auto max-h-[90vh]" @click.stop>
|
||||
<div class="px-6 pt-6 pb-2 flex items-center justify-between">
|
||||
<h2
|
||||
:id="editingProfile ? 'profile-modal-title-edit' : 'profile-modal-title-create'"
|
||||
class="text-lg font-semibold text-gray-900 dark:text-white"
|
||||
x-text="editingProfile ? 'Edit Profile' : 'New Ingestion Profile'"
|
||||
></h2>
|
||||
<button
|
||||
type="button"
|
||||
@click="closeProfileModal()"
|
||||
class="text-gray-400 hover:text-gray-600 dark:hover:text-gray-200 focus:outline-none focus:ring-2 focus:ring-blue-500 rounded"
|
||||
aria-label="Close modal"
|
||||
>
|
||||
<i class="fas fa-times text-xl" aria-hidden="true"></i>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<form @submit.prevent="saveProfile()" class="px-6 pb-6 pt-4 space-y-4">
|
||||
|
||||
<!-- Profile name -->
|
||||
<div>
|
||||
<label for="profile-name" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
|
||||
Profile Name <span class="text-red-500" aria-hidden="true">*</span>
|
||||
</label>
|
||||
<input
|
||||
id="profile-name"
|
||||
type="text"
|
||||
x-model="profileForm.name"
|
||||
placeholder="e.g. Scanner Inbox, Finance Documents"
|
||||
required
|
||||
maxlength="255"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
|
||||
aria-required="true"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<!-- Description -->
|
||||
<div>
|
||||
<label for="profile-description" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
|
||||
Description
|
||||
</label>
|
||||
<input
|
||||
id="profile-description"
|
||||
type="text"
|
||||
x-model="profileForm.description"
|
||||
placeholder="Optional description"
|
||||
maxlength="500"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<!-- Category checkboxes -->
|
||||
<div>
|
||||
<fieldset>
|
||||
<legend class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-2">
|
||||
File Type Categories <span class="text-red-500" aria-hidden="true">*</span>
|
||||
</legend>
|
||||
<div class="space-y-2">
|
||||
<template x-for="cat in categories" :key="cat.key">
|
||||
<label class="flex items-start gap-3 cursor-pointer group">
|
||||
<input
|
||||
type="checkbox"
|
||||
:value="cat.key"
|
||||
:checked="profileForm.allowed_categories.includes(cat.key)"
|
||||
@change="toggleCategory(cat.key)"
|
||||
class="mt-0.5 h-4 w-4 rounded border-gray-300 text-blue-600 focus:ring-blue-500"
|
||||
/>
|
||||
<span class="flex-1">
|
||||
<span class="text-sm font-medium text-gray-800 dark:text-gray-100" x-text="cat.label"></span>
|
||||
<span class="block text-xs text-gray-500 dark:text-gray-400" x-text="cat.description"></span>
|
||||
</span>
|
||||
</label>
|
||||
</template>
|
||||
</div>
|
||||
<p x-show="profileForm.allowed_categories.length === 0" class="mt-2 text-xs text-red-600" role="alert">
|
||||
Select at least one category.
|
||||
</p>
|
||||
</fieldset>
|
||||
</div>
|
||||
|
||||
<!-- Form error -->
|
||||
<template x-if="profileFormError">
|
||||
<p class="text-sm text-red-600" role="alert" x-text="profileFormError"></p>
|
||||
</template>
|
||||
|
||||
<!-- Actions -->
|
||||
<div class="flex justify-end gap-2 pt-2 border-t border-gray-100 dark:border-gray-700">
|
||||
<button
|
||||
type="button"
|
||||
@click="closeProfileModal()"
|
||||
class="px-4 py-2 text-sm font-medium rounded border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-blue-500"
|
||||
style="min-height:40px;"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="submit"
|
||||
:disabled="savingProfile || profileForm.allowed_categories.length === 0"
|
||||
class="inline-flex items-center px-4 py-2 text-sm font-medium rounded bg-blue-600 hover:bg-blue-700 text-white focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50"
|
||||
style="min-height:40px;"
|
||||
>
|
||||
<template x-if="savingProfile">
|
||||
<i class="fas fa-spinner fa-spin mr-2" aria-hidden="true"></i>
|
||||
</template>
|
||||
<span x-text="editingProfile ? 'Save Changes' : 'Create Profile'"></span>
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ── Delete profile confirmation modal ────────────────────────────────────── -->
|
||||
<div
|
||||
x-show="deleteProfileModalOpen"
|
||||
x-transition
|
||||
class="fixed inset-0 z-[60] flex items-center justify-center bg-black/50 px-4"
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby="delete-profile-modal-title"
|
||||
@keydown.escape.window="deleteProfileModalOpen = false"
|
||||
style="display:none;"
|
||||
>
|
||||
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-sm p-6" @click.stop>
|
||||
<h2 id="delete-profile-modal-title" class="text-lg font-semibold text-gray-900 dark:text-white mb-2">
|
||||
Delete Profile?
|
||||
</h2>
|
||||
<p class="text-sm text-gray-600 dark:text-gray-300 mb-4">
|
||||
Are you sure you want to delete the profile
|
||||
<strong x-text="profileToDelete ? profileToDelete.name : ''"></strong>?
|
||||
IMAP accounts using this profile will fall back to the global default.
|
||||
</p>
|
||||
<div class="flex justify-end gap-2">
|
||||
<button
|
||||
type="button"
|
||||
@click="deleteProfileModalOpen = false; profileToDelete = null"
|
||||
class="px-4 py-2 text-sm font-medium rounded border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-gray-500"
|
||||
style="min-height:40px;"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
@click="deleteProfile()"
|
||||
:disabled="deletingProfile"
|
||||
class="inline-flex items-center px-4 py-2 text-sm font-medium rounded bg-red-600 hover:bg-red-700 text-white focus:outline-none focus:ring-2 focus:ring-red-500 disabled:opacity-50"
|
||||
style="min-height:40px;"
|
||||
>
|
||||
<template x-if="deletingProfile">
|
||||
<i class="fas fa-spinner fa-spin mr-2" aria-hidden="true"></i>
|
||||
</template>
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
function imapAccountsApp() {
|
||||
return {
|
||||
accounts: {{ accounts | tojson }},
|
||||
profiles: {{ profiles | tojson }},
|
||||
categories: {{ categories | tojson }},
|
||||
quota: {
|
||||
current_count: {{ current_count }},
|
||||
max_mailboxes: {{ max_mailboxes | tojson }},
|
||||
@@ -538,17 +822,17 @@ function imapAccountsApp() {
|
||||
loading: false,
|
||||
alert: { show: false, type: '', title: '', message: '' },
|
||||
|
||||
// Modal state
|
||||
// Account modal state
|
||||
modalOpen: false,
|
||||
editingAccount: null,
|
||||
form: { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true },
|
||||
form: { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true, profile_id: null },
|
||||
showPassword: false,
|
||||
saving: false,
|
||||
testing: false,
|
||||
testResult: null,
|
||||
formError: null,
|
||||
|
||||
// Delete state
|
||||
// Account delete state
|
||||
deleteModalOpen: false,
|
||||
accountToDelete: null,
|
||||
deleting: false,
|
||||
@@ -556,6 +840,17 @@ function imapAccountsApp() {
|
||||
// Test (saved account) state
|
||||
testingId: null,
|
||||
|
||||
// Profiles panel & modal state
|
||||
showProfilesSection: false,
|
||||
profileModalOpen: false,
|
||||
editingProfile: null,
|
||||
profileForm: { name: '', description: '', allowed_categories: [] },
|
||||
savingProfile: false,
|
||||
profileFormError: null,
|
||||
deleteProfileModalOpen: false,
|
||||
profileToDelete: null,
|
||||
deletingProfile: false,
|
||||
|
||||
get canAdd() {
|
||||
return this.quota.can_add;
|
||||
},
|
||||
@@ -579,9 +874,33 @@ function imapAccountsApp() {
|
||||
setTimeout(() => { this.alert.show = false; }, 5000);
|
||||
},
|
||||
|
||||
// ── Profile helpers ──────────────────────────────────────────────────────
|
||||
|
||||
profileById(id) {
|
||||
return this.profiles.find(p => p.id === id) || null;
|
||||
},
|
||||
|
||||
profileName(id) {
|
||||
const p = this.profileById(id);
|
||||
return p ? p.name : 'Unknown profile';
|
||||
},
|
||||
|
||||
profileCategorySummary(id) {
|
||||
const p = this.profileById(id);
|
||||
if (!p) return '';
|
||||
return p.allowed_categories.map(k => this.categoryLabel(k)).join(', ');
|
||||
},
|
||||
|
||||
categoryLabel(key) {
|
||||
const cat = this.categories.find(c => c.key === key);
|
||||
return cat ? cat.label : key;
|
||||
},
|
||||
|
||||
// ── Account modal ────────────────────────────────────────────────────────
|
||||
|
||||
openCreateModal() {
|
||||
this.editingAccount = null;
|
||||
this.form = { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true, attachment_filter: '' };
|
||||
this.form = { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true, profile_id: null };
|
||||
this.showPassword = false;
|
||||
this.testResult = null;
|
||||
this.formError = null;
|
||||
@@ -599,7 +918,7 @@ function imapAccountsApp() {
|
||||
use_ssl: acct.use_ssl,
|
||||
delete_after_process: acct.delete_after_process,
|
||||
is_active: acct.is_active,
|
||||
attachment_filter: acct.attachment_filter || '',
|
||||
profile_id: acct.profile_id || null,
|
||||
};
|
||||
this.showPassword = false;
|
||||
this.testResult = null;
|
||||
@@ -623,6 +942,8 @@ function imapAccountsApp() {
|
||||
if (this.editingAccount && payload.password === '') {
|
||||
delete payload.password;
|
||||
}
|
||||
// Normalise profile_id: empty string → null
|
||||
if (payload.profile_id === '' || payload.profile_id === 0) payload.profile_id = null;
|
||||
let resp;
|
||||
if (this.editingAccount) {
|
||||
resp = await fetch(`/api/imap-accounts/${this.editingAccount.id}`, {
|
||||
@@ -750,6 +1071,126 @@ function imapAccountsApp() {
|
||||
this.deleting = false;
|
||||
}
|
||||
},
|
||||
|
||||
// ── Profile CRUD ─────────────────────────────────────────────────────────
|
||||
|
||||
openProfileModal(profile) {
|
||||
this.editingProfile = profile;
|
||||
if (profile) {
|
||||
this.profileForm = {
|
||||
name: profile.name,
|
||||
description: profile.description || '',
|
||||
allowed_categories: [...profile.allowed_categories],
|
||||
};
|
||||
} else {
|
||||
this.profileForm = { name: '', description: '', allowed_categories: ['pdf', 'office', 'opendocument', 'text', 'web'] };
|
||||
}
|
||||
this.profileFormError = null;
|
||||
this.profileModalOpen = true;
|
||||
},
|
||||
|
||||
closeProfileModal() {
|
||||
this.profileModalOpen = false;
|
||||
this.editingProfile = null;
|
||||
this.profileFormError = null;
|
||||
},
|
||||
|
||||
toggleCategory(key) {
|
||||
const idx = this.profileForm.allowed_categories.indexOf(key);
|
||||
if (idx === -1) {
|
||||
this.profileForm.allowed_categories.push(key);
|
||||
} else {
|
||||
this.profileForm.allowed_categories.splice(idx, 1);
|
||||
}
|
||||
},
|
||||
|
||||
async saveProfile() {
|
||||
if (this.profileForm.allowed_categories.length === 0) {
|
||||
this.profileFormError = 'Select at least one category.';
|
||||
return;
|
||||
}
|
||||
this.profileFormError = null;
|
||||
this.savingProfile = true;
|
||||
try {
|
||||
const payload = {
|
||||
name: this.profileForm.name,
|
||||
description: this.profileForm.description || null,
|
||||
allowed_categories: this.profileForm.allowed_categories,
|
||||
};
|
||||
let resp;
|
||||
if (this.editingProfile) {
|
||||
resp = await fetch(`/api/imap-profiles/${this.editingProfile.id}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': getCsrfToken() },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
} else {
|
||||
resp = await fetch('/api/imap-profiles/', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': getCsrfToken() },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
}
|
||||
const data = await resp.json();
|
||||
if (!resp.ok) {
|
||||
this.profileFormError = data.detail || 'Failed to save profile.';
|
||||
return;
|
||||
}
|
||||
// Flatten response into the same shape used locally
|
||||
const normalized = {
|
||||
id: data.id,
|
||||
name: data.name,
|
||||
description: data.description,
|
||||
owner_id: data.owner_id,
|
||||
allowed_categories: data.allowed_categories,
|
||||
is_builtin: data.is_builtin,
|
||||
};
|
||||
if (this.editingProfile) {
|
||||
const idx = this.profiles.findIndex(p => p.id === normalized.id);
|
||||
if (idx !== -1) this.profiles.splice(idx, 1, normalized);
|
||||
} else {
|
||||
this.profiles.push(normalized);
|
||||
}
|
||||
this.closeProfileModal();
|
||||
this.showAlert('success', 'Saved', this.editingProfile ? 'Profile updated.' : 'Profile created.');
|
||||
} catch (err) {
|
||||
this.profileFormError = `Network error: ${err.message || 'Unknown error'}. Please try again.`;
|
||||
} finally {
|
||||
this.savingProfile = false;
|
||||
}
|
||||
},
|
||||
|
||||
confirmDeleteProfile(profile) {
|
||||
this.profileToDelete = profile;
|
||||
this.deleteProfileModalOpen = true;
|
||||
},
|
||||
|
||||
async deleteProfile() {
|
||||
if (!this.profileToDelete) return;
|
||||
this.deletingProfile = true;
|
||||
try {
|
||||
const resp = await fetch(`/api/imap-profiles/${this.profileToDelete.id}`, {
|
||||
method: 'DELETE',
|
||||
headers: { 'X-CSRF-Token': getCsrfToken() },
|
||||
});
|
||||
if (resp.ok || resp.status === 204) {
|
||||
const deletedId = this.profileToDelete.id;
|
||||
this.profiles = this.profiles.filter(p => p.id !== deletedId);
|
||||
// Clear profile_id on accounts that referenced this profile
|
||||
this.accounts.forEach(a => { if (a.profile_id === deletedId) a.profile_id = null; });
|
||||
this.showAlert('success', 'Deleted', `Profile "${this.profileToDelete.name}" removed.`);
|
||||
this.deleteProfileModalOpen = false;
|
||||
this.profileToDelete = null;
|
||||
} else {
|
||||
const data = await resp.json();
|
||||
this.showAlert('error', 'Delete Failed', data.detail || 'Could not delete profile.');
|
||||
}
|
||||
} catch (err) {
|
||||
this.showAlert('error', 'Delete Failed', `Network error: ${err.message || 'Unknown error'}`);
|
||||
} finally {
|
||||
this.deletingProfile = false;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ from app.models import ( # noqa: F401
|
||||
DocumentMetadata,
|
||||
FileProcessingStep,
|
||||
FileRecord,
|
||||
ImapIngestionProfile,
|
||||
InAppNotification,
|
||||
LocalUser,
|
||||
MobileDevice,
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Add imap_ingestion_profiles table and migrate user_imap_accounts.
|
||||
|
||||
Creates the ``imap_ingestion_profiles`` table, seeds the two built-in profiles
|
||||
("Documents Only" and "All Files"), and replaces the ``attachment_filter``
|
||||
string column on ``user_imap_accounts`` with a ``profile_id`` FK that references
|
||||
the new table.
|
||||
|
||||
Revision ID: 033_add_imap_ingestion_profiles
|
||||
Revises: 032_add_imap_attachment_filter
|
||||
Create Date: 2026-03-12
|
||||
"""
|
||||
|
||||
from typing import Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "033_add_imap_ingestion_profiles"
|
||||
down_revision: Union[str, None] = "032_add_imap_attachment_filter"
|
||||
depends_on: Union[str, None] = None
|
||||
|
||||
# Fixed IDs for the built-in profiles so that the FK migration is reproducible.
|
||||
_BUILTIN_DOCUMENTS_ONLY_ID = 1
|
||||
_BUILTIN_ALL_FILES_ID = 2
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Create profiles table, seed built-ins, migrate accounts column."""
|
||||
# 1 — Create the imap_ingestion_profiles table
|
||||
op.create_table(
|
||||
"imap_ingestion_profiles",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("name", sa.String(255), nullable=False),
|
||||
sa.Column("description", sa.Text(), nullable=True),
|
||||
sa.Column("owner_id", sa.String(), nullable=True),
|
||||
sa.Column(
|
||||
"allowed_categories",
|
||||
sa.Text(),
|
||||
nullable=False,
|
||||
server_default='["pdf","office","opendocument","text","web"]',
|
||||
),
|
||||
sa.Column("is_builtin", sa.Boolean(), nullable=False, server_default="0"),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index("ix_imap_ingestion_profiles_id", "imap_ingestion_profiles", ["id"])
|
||||
op.create_index("ix_imap_ingestion_profiles_owner_id", "imap_ingestion_profiles", ["owner_id"])
|
||||
|
||||
# 2 — Seed the two built-in profiles
|
||||
bind = op.get_bind()
|
||||
bind.execute(
|
||||
sa.text(
|
||||
"INSERT INTO imap_ingestion_profiles "
|
||||
"(id, name, description, owner_id, allowed_categories, is_builtin) "
|
||||
"VALUES (:id, :name, :desc, NULL, :cats, 1)"
|
||||
),
|
||||
[
|
||||
{
|
||||
"id": _BUILTIN_DOCUMENTS_ONLY_ID,
|
||||
"name": "Documents Only",
|
||||
"desc": (
|
||||
"Ingest PDFs, Microsoft Office files, OpenDocument files, "
|
||||
"plain text, CSV, RTF, HTML and Markdown. Images are excluded."
|
||||
),
|
||||
"cats": '["pdf","office","opendocument","text","web"]',
|
||||
},
|
||||
{
|
||||
"id": _BUILTIN_ALL_FILES_ID,
|
||||
"name": "All Files",
|
||||
"desc": "Ingest all supported file types, including images.",
|
||||
"cats": '["pdf","office","opendocument","text","web","images"]',
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
# 3 — Add profile_id column to user_imap_accounts
|
||||
op.add_column(
|
||||
"user_imap_accounts",
|
||||
sa.Column("profile_id", sa.Integer(), nullable=True),
|
||||
)
|
||||
|
||||
# 4 — Migrate existing attachment_filter values to profile_id
|
||||
bind.execute(
|
||||
sa.text(
|
||||
"UPDATE user_imap_accounts SET profile_id = :pid "
|
||||
"WHERE attachment_filter = 'all'"
|
||||
),
|
||||
{"pid": _BUILTIN_ALL_FILES_ID},
|
||||
)
|
||||
bind.execute(
|
||||
sa.text(
|
||||
"UPDATE user_imap_accounts SET profile_id = :pid "
|
||||
"WHERE attachment_filter = 'documents_only'"
|
||||
),
|
||||
{"pid": _BUILTIN_DOCUMENTS_ONLY_ID},
|
||||
)
|
||||
# Rows with NULL attachment_filter keep profile_id = NULL (use global default)
|
||||
|
||||
# 5 — Add FK constraint (skip for SQLite which does not enforce FKs at DDL time)
|
||||
# We use batch_alter_table so this works across SQLite and PostgreSQL
|
||||
with op.batch_alter_table("user_imap_accounts") as batch_op:
|
||||
batch_op.create_foreign_key(
|
||||
"fk_user_imap_accounts_profile_id",
|
||||
"imap_ingestion_profiles",
|
||||
["profile_id"],
|
||||
["id"],
|
||||
)
|
||||
|
||||
# 6 — Drop the now-redundant attachment_filter column
|
||||
with op.batch_alter_table("user_imap_accounts") as batch_op:
|
||||
batch_op.drop_column("attachment_filter")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Reverse the migration: restore attachment_filter, drop profiles table."""
|
||||
# 1 — Re-add attachment_filter column
|
||||
with op.batch_alter_table("user_imap_accounts") as batch_op:
|
||||
batch_op.add_column(sa.Column("attachment_filter", sa.String(50), nullable=True))
|
||||
|
||||
# 2 — Restore string values from profile_id
|
||||
bind = op.get_bind()
|
||||
bind.execute(
|
||||
sa.text(
|
||||
"UPDATE user_imap_accounts SET attachment_filter = 'all' "
|
||||
"WHERE profile_id = :pid"
|
||||
),
|
||||
{"pid": _BUILTIN_ALL_FILES_ID},
|
||||
)
|
||||
bind.execute(
|
||||
sa.text(
|
||||
"UPDATE user_imap_accounts SET attachment_filter = 'documents_only' "
|
||||
"WHERE profile_id = :pid"
|
||||
),
|
||||
{"pid": _BUILTIN_DOCUMENTS_ONLY_ID},
|
||||
)
|
||||
|
||||
# 3 — Drop the FK and profile_id column
|
||||
with op.batch_alter_table("user_imap_accounts") as batch_op:
|
||||
batch_op.drop_constraint("fk_user_imap_accounts_profile_id", type_="foreignkey")
|
||||
batch_op.drop_column("profile_id")
|
||||
|
||||
# 4 — Drop the profiles table
|
||||
op.drop_index("ix_imap_ingestion_profiles_owner_id", "imap_ingestion_profiles")
|
||||
op.drop_index("ix_imap_ingestion_profiles_id", "imap_ingestion_profiles")
|
||||
op.drop_table("imap_ingestion_profiles")
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Tests for app/api/imap_profiles.py and app/utils/allowed_types category helpers."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.utils.allowed_types import (
|
||||
ALL_CATEGORIES,
|
||||
DEFAULT_CATEGORIES,
|
||||
FILE_TYPE_CATEGORIES,
|
||||
get_allowed_types_for_categories,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestFileTypeCategories:
|
||||
"""Tests for FILE_TYPE_CATEGORIES and get_allowed_types_for_categories."""
|
||||
|
||||
def test_all_category_keys_present(self):
|
||||
"""Test that the six expected categories exist."""
|
||||
assert set(FILE_TYPE_CATEGORIES.keys()) == {"pdf", "office", "opendocument", "text", "web", "images"}
|
||||
|
||||
def test_each_category_has_required_fields(self):
|
||||
"""Test that every category entry has label, description, mime_types, extensions."""
|
||||
for key, info in FILE_TYPE_CATEGORIES.items():
|
||||
assert "label" in info, f"Category '{key}' missing 'label'"
|
||||
assert "description" in info, f"Category '{key}' missing 'description'"
|
||||
assert "mime_types" in info, f"Category '{key}' missing 'mime_types'"
|
||||
assert "extensions" in info, f"Category '{key}' missing 'extensions'"
|
||||
|
||||
def test_pdf_category_contains_pdf_mime(self):
|
||||
"""Test that the pdf category includes application/pdf."""
|
||||
assert "application/pdf" in FILE_TYPE_CATEGORIES["pdf"]["mime_types"]
|
||||
assert ".pdf" in FILE_TYPE_CATEGORIES["pdf"]["extensions"]
|
||||
|
||||
def test_images_category_contains_jpeg(self):
|
||||
"""Test that the images category includes image/jpeg."""
|
||||
assert "image/jpeg" in FILE_TYPE_CATEGORIES["images"]["mime_types"]
|
||||
assert ".jpg" in FILE_TYPE_CATEGORIES["images"]["extensions"]
|
||||
assert ".png" in FILE_TYPE_CATEGORIES["images"]["extensions"]
|
||||
|
||||
def test_get_allowed_types_for_default_categories(self):
|
||||
"""Test that DEFAULT_CATEGORIES excludes image MIME types."""
|
||||
mime_types, extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
|
||||
assert "application/pdf" in mime_types
|
||||
assert "application/msword" in mime_types
|
||||
# images should NOT be in the default
|
||||
assert "image/jpeg" not in mime_types
|
||||
assert ".jpg" not in extensions
|
||||
|
||||
def test_get_allowed_types_for_all_categories(self):
|
||||
"""Test that ALL_CATEGORIES includes image MIME types."""
|
||||
mime_types, extensions = get_allowed_types_for_categories(ALL_CATEGORIES)
|
||||
assert "image/jpeg" in mime_types
|
||||
assert ".jpg" in extensions
|
||||
assert "application/pdf" in mime_types
|
||||
|
||||
def test_get_allowed_types_returns_frozensets(self):
|
||||
"""Test that returned sets are frozensets."""
|
||||
mime_types, extensions = get_allowed_types_for_categories(["pdf"])
|
||||
assert isinstance(mime_types, frozenset)
|
||||
assert isinstance(extensions, frozenset)
|
||||
|
||||
def test_get_allowed_types_unknown_category_ignored(self):
|
||||
"""Test that unknown category keys are silently ignored."""
|
||||
mime_types, extensions = get_allowed_types_for_categories(["pdf", "nonexistent_category"])
|
||||
assert "application/pdf" in mime_types # 'pdf' still works
|
||||
# No crash for unknown key
|
||||
|
||||
def test_get_allowed_types_empty_list(self):
|
||||
"""Test empty category list returns empty sets."""
|
||||
mime_types, extensions = get_allowed_types_for_categories([])
|
||||
assert mime_types == frozenset()
|
||||
assert extensions == frozenset()
|
||||
|
||||
def test_default_categories_excludes_images(self):
|
||||
"""Test that DEFAULT_CATEGORIES does not include 'images'."""
|
||||
assert "images" not in DEFAULT_CATEGORIES
|
||||
|
||||
def test_all_categories_includes_images(self):
|
||||
"""Test that ALL_CATEGORIES includes 'images'."""
|
||||
assert "images" in ALL_CATEGORIES
|
||||
|
||||
def test_all_categories_is_superset_of_default(self):
|
||||
"""Test that ALL_CATEGORIES contains all DEFAULT_CATEGORIES."""
|
||||
for cat in DEFAULT_CATEGORIES:
|
||||
assert cat in ALL_CATEGORIES
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestImapProfilesApiLogic:
|
||||
"""Tests for ingestion profile validation helpers."""
|
||||
|
||||
def test_validate_categories_accepts_valid_keys(self):
|
||||
"""Test that valid category keys pass validation."""
|
||||
from app.api.imap_profiles import _validate_categories
|
||||
|
||||
result = _validate_categories(["pdf", "office", "images"])
|
||||
assert set(result) == {"pdf", "office", "images"}
|
||||
|
||||
def test_validate_categories_rejects_unknown_key(self):
|
||||
"""Test that unknown category keys raise 422."""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.api.imap_profiles import _validate_categories
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_validate_categories(["pdf", "nonexistent"])
|
||||
assert exc_info.value.status_code == 422
|
||||
assert "nonexistent" in str(exc_info.value.detail)
|
||||
|
||||
def test_validate_categories_deduplicates(self):
|
||||
"""Test that duplicate category keys are de-duplicated while preserving order."""
|
||||
from app.api.imap_profiles import _validate_categories
|
||||
|
||||
result = _validate_categories(["pdf", "pdf", "office", "pdf"])
|
||||
assert result == ["pdf", "office"]
|
||||
|
||||
def test_to_response_serializes_profile(self, tmp_path):
|
||||
"""Test _to_response produces expected dict shape."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.api.imap_profiles import _to_response
|
||||
|
||||
profile = MagicMock()
|
||||
profile.id = 42
|
||||
profile.name = "My Profile"
|
||||
profile.description = "Test description"
|
||||
profile.owner_id = "user@example.com"
|
||||
profile.allowed_categories = '["pdf","office"]'
|
||||
profile.is_builtin = False
|
||||
profile.created_at = None
|
||||
profile.updated_at = None
|
||||
|
||||
result = _to_response(profile)
|
||||
assert result["id"] == 42
|
||||
assert result["name"] == "My Profile"
|
||||
assert result["allowed_categories"] == ["pdf", "office"]
|
||||
assert len(result["categories_detail"]) == 2
|
||||
assert result["categories_detail"][0]["key"] == "pdf"
|
||||
assert result["is_builtin"] is False
|
||||
|
||||
def test_to_response_handles_invalid_categories_json(self):
|
||||
"""Test _to_response gracefully handles invalid JSON in allowed_categories."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.api.imap_profiles import _to_response
|
||||
|
||||
profile = MagicMock()
|
||||
profile.id = 1
|
||||
profile.name = "Broken"
|
||||
profile.description = None
|
||||
profile.owner_id = None
|
||||
profile.allowed_categories = "this is not valid json {"
|
||||
profile.is_builtin = True
|
||||
profile.created_at = None
|
||||
profile.updated_at = None
|
||||
|
||||
result = _to_response(profile)
|
||||
assert result["allowed_categories"] == []
|
||||
+19
-16
@@ -24,6 +24,7 @@ from app.tasks.imap_tasks import (
|
||||
release_lock,
|
||||
save_processed_emails,
|
||||
)
|
||||
from app.utils.allowed_types import ALL_CATEGORIES, DEFAULT_CATEGORIES, get_allowed_types_for_categories
|
||||
|
||||
_TEST_CREDENTIAL = "pass" # noqa: S105
|
||||
|
||||
@@ -182,27 +183,29 @@ class TestFetchAttachmentsAndEnqueue:
|
||||
@patch("app.tasks.imap_tasks.process_document")
|
||||
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
||||
def test_skips_image_when_documents_only(self, mock_convert, mock_process):
|
||||
"""Test that image attachments are skipped with documents_only filter."""
|
||||
"""Test that image attachments are skipped with the default (documents-only) categories."""
|
||||
msg = EmailMessage()
|
||||
msg["Subject"] = "Photo"
|
||||
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
|
||||
|
||||
result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
|
||||
doc_mime, doc_ext = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
|
||||
result = fetch_attachments_and_enqueue(msg, effective_mime_types=doc_mime, effective_extensions=doc_ext)
|
||||
assert result is False
|
||||
mock_process.delay.assert_not_called()
|
||||
mock_convert.delay.assert_not_called()
|
||||
|
||||
@patch("app.tasks.imap_tasks.process_document")
|
||||
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
||||
def test_processes_image_when_all_filter(self, mock_convert, mock_process, tmp_path):
|
||||
"""Test that image attachments are processed with 'all' filter."""
|
||||
def test_processes_image_when_all_categories(self, mock_convert, mock_process, tmp_path):
|
||||
"""Test that image attachments are processed when images category is included."""
|
||||
msg = EmailMessage()
|
||||
msg["Subject"] = "Photo"
|
||||
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
|
||||
|
||||
all_mime, all_ext = get_allowed_types_for_categories(ALL_CATEGORIES)
|
||||
with patch("app.tasks.imap_tasks.settings") as mock_settings:
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
result = fetch_attachments_and_enqueue(msg, attachment_filter="all")
|
||||
result = fetch_attachments_and_enqueue(msg, effective_mime_types=all_mime, effective_extensions=all_ext)
|
||||
|
||||
assert result is True
|
||||
mock_convert.delay.assert_called_once()
|
||||
@@ -210,43 +213,43 @@ class TestFetchAttachmentsAndEnqueue:
|
||||
@patch("app.tasks.imap_tasks.process_document")
|
||||
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
||||
def test_skips_image_with_image_extension_documents_only(self, mock_convert, mock_process):
|
||||
"""Test image files identified by extension are skipped with documents_only."""
|
||||
"""Test image files identified by extension are skipped with documents-only categories."""
|
||||
msg = EmailMessage()
|
||||
msg["Subject"] = "Screenshot"
|
||||
msg.add_attachment(b"\x89PNG", maintype="application", subtype="octet-stream", filename="screenshot.png")
|
||||
|
||||
result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
|
||||
doc_mime, doc_ext = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
|
||||
result = fetch_attachments_and_enqueue(msg, effective_mime_types=doc_mime, effective_extensions=doc_ext)
|
||||
assert result is False
|
||||
mock_process.delay.assert_not_called()
|
||||
mock_convert.delay.assert_not_called()
|
||||
|
||||
@patch("app.tasks.imap_tasks.process_document")
|
||||
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
||||
def test_processes_pdf_regardless_of_filter(self, mock_convert, mock_process, tmp_path):
|
||||
"""Test that PDFs are always processed, even with documents_only filter."""
|
||||
def test_processes_pdf_regardless_of_categories(self, mock_convert, mock_process, tmp_path):
|
||||
"""Test that PDFs are always processed (pdf category always included in DEFAULT_CATEGORIES)."""
|
||||
msg = EmailMessage()
|
||||
msg["Subject"] = "Invoice"
|
||||
msg.add_attachment(b"%PDF-1.4", maintype="application", subtype="pdf", filename="invoice.pdf")
|
||||
|
||||
doc_mime, doc_ext = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
|
||||
with patch("app.tasks.imap_tasks.settings") as mock_settings:
|
||||
mock_settings.workdir = str(tmp_path)
|
||||
result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
|
||||
result = fetch_attachments_and_enqueue(msg, effective_mime_types=doc_mime, effective_extensions=doc_ext)
|
||||
|
||||
assert result is True
|
||||
mock_process.delay.assert_called_once()
|
||||
|
||||
@patch("app.tasks.imap_tasks.process_document")
|
||||
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
||||
def test_uses_global_setting_when_no_filter(self, mock_convert, mock_process):
|
||||
"""Test that the global settings.imap_attachment_filter is used when no filter is passed."""
|
||||
def test_uses_default_categories_when_no_types_provided(self, mock_convert, mock_process):
|
||||
"""Test that images are skipped when no effective_mime_types / extensions are passed (defaults to DEFAULT_CATEGORIES)."""
|
||||
msg = EmailMessage()
|
||||
msg["Subject"] = "Photo"
|
||||
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
|
||||
|
||||
with patch("app.tasks.imap_tasks.settings") as mock_settings:
|
||||
mock_settings.imap_attachment_filter = "documents_only"
|
||||
mock_settings.workdir = "/tmp"
|
||||
result = fetch_attachments_and_enqueue(msg)
|
||||
# No effective_mime_types passed → function defaults to DEFAULT_CATEGORIES (no images)
|
||||
result = fetch_attachments_and_enqueue(msg)
|
||||
|
||||
assert result is False
|
||||
mock_process.delay.assert_not_called()
|
||||
|
||||
Reference in New Issue
Block a user