From c9f554465d5bf027a92bb4aa56db244aad4a6ff6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 12 Mar 2026 01:14:19 +0000 Subject: [PATCH] feat(imap): add ImapIngestionProfile model, API, migration and UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the simple binary attachment_filter string with a full ingestion profiles system: - Add FILE_TYPE_CATEGORIES dict to allowed_types.py (6 categories: pdf, office, opendocument, text, web, images) + DEFAULT_CATEGORIES / ALL_CATEGORIES + get_allowed_types_for_categories() helper - Add ImapIngestionProfile model (id, name, description, owner_id, allowed_categories JSON, is_builtin) - Update UserImapAccount: replace attachment_filter string with profile_id FK to imap_ingestion_profiles - Migration 033: creates profiles table, seeds 2 built-in profiles (Documents Only, All Files), migrates attachment_filter → profile_id - New /api/imap-profiles/ CRUD endpoints (list, create, get, update, delete) with category validation - Register imap_profiles router in app/api/__init__.py - Update imap_tasks.py: replace attachment_filter string param with profile-based allowed_categories; add _resolve_categories_for_profile() - Update imap_accounts.py API to use profile_id instead of attachment_filter - Update imap_accounts view to pass profiles + categories to template - Full UI overhaul: profiles panel + profile create/edit modal with category checkboxes; profile selector in account modal - 17 new tests (141 total), all passing Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- app/api/__init__.py | 2 + app/api/imap_accounts.py | 28 +- app/api/imap_profiles.py | 256 +++++++++ app/models.py | 49 +- app/tasks/imap_tasks.py | 150 +++--- app/utils/allowed_types.py | 154 ++++++ app/views/imap_accounts.py | 48 +- frontend/templates/imap_accounts.html | 493 +++++++++++++++++- migrations/env.py | 1 + .../033_add_imap_ingestion_profiles.py | 146 ++++++ tests/test_imap_profiles.py | 160 ++++++ tests/test_imap_tasks.py | 35 +- 12 files changed, 1387 insertions(+), 135 deletions(-) create mode 100644 app/api/imap_profiles.py create mode 100644 migrations/versions/033_add_imap_ingestion_profiles.py create mode 100644 tests/test_imap_profiles.py diff --git a/app/api/__init__.py b/app/api/__init__.py index ca331229..28b9fbba 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -21,6 +21,7 @@ from app.api.files import router as files_router from app.api.google_drive import router as google_drive_router from app.api.i18n import router as i18n_router from app.api.imap_accounts import router as imap_accounts_router +from app.api.imap_profiles import router as imap_profiles_router from app.api.integrations import router as integrations_router from app.api.logs import router as logs_router from app.api.mobile import router as mobile_router @@ -83,6 +84,7 @@ router.include_router(onboarding_router) router.include_router(billing_router) router.include_router(pipelines_router) router.include_router(imap_accounts_router) +router.include_router(imap_profiles_router) router.include_router(integrations_router) router.include_router(notifications_router) router.include_router(scheduled_jobs_router) diff --git a/app/api/imap_accounts.py b/app/api/imap_accounts.py index 169fdef7..2aa9b039 100644 --- a/app/api/imap_accounts.py +++ b/app/api/imap_accounts.py @@ -110,12 +110,10 @@ class ImapAccountCreate(BaseModel): use_ssl: bool = Field(default=True, description="Use SSL/TLS connection") delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing") is_active: bool = Field(default=True, description="Whether to poll this mailbox") - attachment_filter: str | None = Field( + profile_id: int | None = Field( default=None, description=( - "Controls which attachment types to ingest. " - "'documents_only' – PDFs and office files only (default when None). " - "'all' – all supported types including images. " + "ID of the ImapIngestionProfile that controls which attachment types to ingest. " "Null inherits the global imap_attachment_filter setting." ), ) @@ -132,13 +130,11 @@ class ImapAccountUpdate(BaseModel): use_ssl: bool | None = None delete_after_process: bool | None = None is_active: bool | None = None - attachment_filter: str | None = Field( + profile_id: int | None = Field( default=None, description=( - "Controls which attachment types to ingest. " - "'documents_only' – PDFs and office files only. " - "'all' – all supported types including images. " - "Null or empty string clears the override (inherits global setting)." + "ID of the ImapIngestionProfile to use. " + "Explicitly sending null clears the override (falls back to global setting)." ), ) @@ -173,7 +169,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]: "use_ssl": acct.use_ssl, "delete_after_process": acct.delete_after_process, "is_active": acct.is_active, - "attachment_filter": acct.attachment_filter, + "profile_id": acct.profile_id, "last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None, "last_error": acct.last_error, "created_at": acct.created_at.isoformat() if acct.created_at else None, @@ -241,7 +237,7 @@ def create_imap_account( use_ssl=body.use_ssl, delete_after_process=body.delete_after_process, is_active=body.is_active, - attachment_filter=body.attachment_filter or None, + profile_id=body.profile_id, ) try: db.add(acct) @@ -297,12 +293,10 @@ def update_imap_account( acct.delete_after_process = body.delete_after_process if body.is_active is not None: acct.is_active = body.is_active - # attachment_filter uses a sentinel check: the field is always present in the - # model (defaulting to None in Pydantic) so we update it unconditionally when - # the caller sends any value (including explicit null to clear the override). - # An empty string is normalised to None to avoid storing a non-meaningful value. - if "attachment_filter" in body.model_fields_set: - acct.attachment_filter = body.attachment_filter or None + # profile_id: update whenever the field is explicitly present in the request payload + # (including sending null to clear the override). + if "profile_id" in body.model_fields_set: + acct.profile_id = body.profile_id # Reset last_error so the next poll gives a fresh result acct.last_error = None diff --git a/app/api/imap_profiles.py b/app/api/imap_profiles.py new file mode 100644 index 00000000..8f267f63 --- /dev/null +++ b/app/api/imap_profiles.py @@ -0,0 +1,256 @@ +"""API endpoints for managing IMAP ingestion profiles. + +Ingestion profiles allow fine-grained control over which attachment types are +accepted when ingesting emails via IMAP. Each profile carries a list of enabled +file-type categories (e.g. ``["pdf", "office", "images"]``) drawn from the +canonical set defined in :mod:`app.utils.allowed_types`. + +Built-in system profiles (``is_builtin=True``) are read-only and cannot be +deleted or modified. Users may create their own profiles which are private to +their ``owner_id``. System-level global profiles (``owner_id=None``) are visible +to all users but can only be created by administrators. +""" + +import json +import logging +from datetime import datetime, timezone +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from app.database import get_db +from app.models import ImapIngestionProfile +from app.utils.allowed_types import FILE_TYPE_CATEGORIES +from app.utils.user_scope import get_current_owner_id + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/imap-profiles", tags=["imap-profiles"]) + +DbSession = Annotated[Session, Depends(get_db)] + +# --------------------------------------------------------------------------- +# Auth helpers +# --------------------------------------------------------------------------- + + +def _get_owner_id(request: Request) -> str: + """Return the current user's owner ID, raising 401 if unauthenticated.""" + owner_id = get_current_owner_id(request) + if owner_id is None: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") + return owner_id + + +CurrentOwner = Annotated[str, Depends(_get_owner_id)] + +# --------------------------------------------------------------------------- +# Pydantic schemas +# --------------------------------------------------------------------------- + +_VALID_CATEGORIES = set(FILE_TYPE_CATEGORIES.keys()) + + +class ImapProfileCreate(BaseModel): + """Schema for creating a new ingestion profile.""" + + name: str = Field(..., min_length=1, max_length=255, description="Human-readable profile name") + description: str | None = Field(default=None, description="Optional description") + allowed_categories: list[str] = Field( + ..., + min_length=1, + description=(f"List of enabled file-type category keys. Valid values: {sorted(_VALID_CATEGORIES)}"), + ) + + +class ImapProfileUpdate(BaseModel): + """Schema for updating an existing profile (all fields optional).""" + + name: str | None = Field(default=None, min_length=1, max_length=255) + description: str | None = None + allowed_categories: list[str] | None = Field(default=None, min_length=1) + + +# --------------------------------------------------------------------------- +# Validation helpers +# --------------------------------------------------------------------------- + + +def _validate_categories(categories: list[str]) -> list[str]: + """Raise 422 if any category key is unknown; return the cleaned list.""" + unknown = [c for c in categories if c not in _VALID_CATEGORIES] + if unknown: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Unknown category key(s): {unknown}. Valid keys: {sorted(_VALID_CATEGORIES)}", + ) + # Deduplicate while preserving order + seen: set[str] = set() + result: list[str] = [] + for cat in categories: + if cat not in seen: + seen.add(cat) + result.append(cat) + return result + + +# --------------------------------------------------------------------------- +# Serialisation +# --------------------------------------------------------------------------- + + +def _to_response(profile: ImapIngestionProfile) -> dict[str, Any]: + """Serialize a profile row to a response dict.""" + try: + categories = json.loads(profile.allowed_categories) + except (ValueError, TypeError): + categories = [] + + # Enrich categories with display metadata + categories_detail = [ + { + "key": cat, + "label": FILE_TYPE_CATEGORIES[cat]["label"] if cat in FILE_TYPE_CATEGORIES else cat, + "description": FILE_TYPE_CATEGORIES[cat]["description"] if cat in FILE_TYPE_CATEGORIES else "", + } + for cat in categories + ] + + return { + "id": profile.id, + "name": profile.name, + "description": profile.description, + "owner_id": profile.owner_id, + "allowed_categories": categories, + "categories_detail": categories_detail, + "is_builtin": profile.is_builtin, + "created_at": profile.created_at.isoformat() if profile.created_at else None, + "updated_at": profile.updated_at.isoformat() if profile.updated_at else None, + } + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +@router.get("/categories", summary="List available file-type categories") +def list_categories(request: Request, owner_id: CurrentOwner) -> list[dict[str, Any]]: + """Return the full list of file-type categories that can be used in profiles.""" + return [ + { + "key": key, + "label": info["label"], + "description": info["description"], + } + for key, info in FILE_TYPE_CATEGORIES.items() + ] + + +@router.get("/", summary="List ingestion profiles visible to the current user") +def list_profiles(request: Request, db: DbSession, owner_id: CurrentOwner) -> list[dict[str, Any]]: + """Return all profiles: system-global (owner_id=NULL) and the user's own profiles.""" + profiles = ( + db.query(ImapIngestionProfile) + .filter( + (ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711 + ) + .order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id) + .all() + ) + return [_to_response(p) for p in profiles] + + +@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new ingestion profile") +def create_profile(request: Request, body: ImapProfileCreate, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]: + """Create a new ingestion profile owned by the current user.""" + categories = _validate_categories(body.allowed_categories) + + profile = ImapIngestionProfile( + name=body.name, + description=body.description, + owner_id=owner_id, + allowed_categories=json.dumps(categories), + is_builtin=False, + ) + try: + db.add(profile) + db.commit() + db.refresh(profile) + except Exception: + db.rollback() + raise + + logger.info("User %s created IMAP ingestion profile %d ('%s')", owner_id, profile.id, body.name) + return _to_response(profile) + + +@router.get("/{profile_id}", summary="Get a single ingestion profile") +def get_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]: + """Return a single profile by ID. Only the owner or system profiles are accessible.""" + profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first() + if not profile or (profile.owner_id is not None and profile.owner_id != owner_id): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found") + return _to_response(profile) + + +@router.put("/{profile_id}", summary="Update an ingestion profile") +def update_profile( + profile_id: int, + request: Request, + body: ImapProfileUpdate, + db: DbSession, + owner_id: CurrentOwner, +) -> dict[str, Any]: + """Update an existing ingestion profile. Built-in profiles cannot be modified.""" + profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first() + if not profile or (profile.owner_id is not None and profile.owner_id != owner_id): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found") + if profile.is_builtin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Built-in profiles cannot be modified.", + ) + + if body.name is not None: + profile.name = body.name + if "description" in body.model_fields_set: + profile.description = body.description + if body.allowed_categories is not None: + categories = _validate_categories(body.allowed_categories) + profile.allowed_categories = json.dumps(categories) + + profile.updated_at = datetime.now(timezone.utc) + + try: + db.commit() + db.refresh(profile) + except Exception: + db.rollback() + raise + + logger.info("User %s updated IMAP ingestion profile %d", owner_id, profile_id) + return _to_response(profile) + + +@router.delete("/{profile_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an ingestion profile") +def delete_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> None: + """Delete an ingestion profile. Built-in profiles cannot be deleted.""" + profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first() + if not profile or (profile.owner_id is not None and profile.owner_id != owner_id): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found") + if profile.is_builtin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Built-in profiles cannot be deleted.", + ) + + try: + db.delete(profile) + db.commit() + except Exception: + db.rollback() + raise + + logger.info("User %s deleted IMAP ingestion profile %d", owner_id, profile_id) diff --git a/app/models.py b/app/models.py index 94cb4e62..5ba06e81 100644 --- a/app/models.py +++ b/app/models.py @@ -401,6 +401,48 @@ class PipelineStep(Base): updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) +class ImapIngestionProfile(Base): + """Named ingestion profile controlling which attachment types are accepted from IMAP emails. + + Profiles group file-type categories (e.g. "pdf", "office", "images") so users + can precisely control what gets ingested from each mailbox. + + System-provided built-in profiles (``is_builtin=True``) are seeded by the + migration and cannot be deleted or renamed. Users may create their own profiles + (``owner_id`` set to their identifier) or rely on the global system profiles + (``owner_id=None``). + + ``allowed_categories`` stores a JSON list of category strings, e.g.:: + + '["pdf", "office", "opendocument", "text", "web"]' + + Valid category names are defined in ``app.utils.allowed_types.FILE_TYPE_CATEGORIES``. + """ + + __tablename__ = "imap_ingestion_profiles" + + id = Column(Integer, primary_key=True, index=True) + + # Human-readable profile name (e.g. "Documents Only", "Documents + Images") + name = Column(String(255), nullable=False) + + # Optional description shown in the UI + description = Column(Text, nullable=True) + + # Owner of this profile. NULL = global/system profile available to all users. + owner_id = Column(String, nullable=True, index=True) + + # JSON-encoded list of enabled category keys. Example: '["pdf","office","text"]' + # See FILE_TYPE_CATEGORIES in app/utils/allowed_types.py for valid values. + allowed_categories = Column(Text, nullable=False, default='["pdf","office","opendocument","text","web"]') + + # Built-in system profiles that cannot be deleted or modified via the API. + is_builtin = Column(Boolean, nullable=False, default=False) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + class UserImapAccount(Base): """Per-user IMAP ingestion account. @@ -439,10 +481,9 @@ class UserImapAccount(Base): # When True, emails are deleted from the mailbox after their attachments are processed delete_after_process = Column(Boolean, nullable=False, default=False) - # Override for which attachment types to ingest. - # NULL means "inherit the global imap_attachment_filter setting". - # Allowed values: 'documents_only', 'all' - attachment_filter = Column(String(50), nullable=True, default=None) + # Optional reference to an ImapIngestionProfile. + # NULL means "use the global imap_attachment_filter setting" (system default). + profile_id = Column(Integer, ForeignKey("imap_ingestion_profiles.id"), nullable=True) # When False the account is not polled by the periodic task (but not deleted) is_active = Column(Boolean, nullable=False, default=True) diff --git a/app/tasks/imap_tasks.py b/app/tasks/imap_tasks.py index be9f3bd4..affd7a9d 100644 --- a/app/tasks/imap_tasks.py +++ b/app/tasks/imap_tasks.py @@ -13,7 +13,11 @@ from celery import shared_task from app.config import settings from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task from app.tasks.process_document import process_document # Updated import -from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, DOCUMENT_MIME_TYPES, IMAGE_MIME_TYPES +from app.utils.allowed_types import ( + ALL_CATEGORIES, + DEFAULT_CATEGORIES, + get_allowed_types_for_categories, +) # Database session for per-user IMAP accounts (imported lazily to avoid circular imports) _db_session_factory = None @@ -50,6 +54,33 @@ def _decrypt_imap_password(password: str | None) -> str | None: return decrypt_value(password) +def _resolve_categories_for_profile(profile_id: int | None) -> list[str]: + """Return the list of allowed categories for a profile ID. + + Loads the profile from the database. If ``profile_id`` is ``None`` or the + profile is not found, falls back to the global ``settings.imap_attachment_filter`` + string (``'documents_only'`` → default categories; ``'all'`` → all categories). + """ + if profile_id is not None: + try: + from app.models import ImapIngestionProfile + + db = _get_db_session() + try: + profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first() + if profile: + return json.loads(profile.allowed_categories) + finally: + db.close() + except Exception as exc: # noqa: BLE001 + logger.warning("Could not load IMAP ingestion profile %d: %s — using global default", profile_id, exc) + + # Fall back to global setting + if settings.imap_attachment_filter == "all": + return ALL_CATEGORIES + return DEFAULT_CATEGORIES + + LOCK_KEY = "imap_lock" # Unique key for locking LOCK_EXPIRE = 300 # Lock expires in 5 minutes @@ -180,7 +211,7 @@ def _pull_user_imap_accounts() -> None: use_ssl=acct.use_ssl, delete_after_process=acct.delete_after_process, owner_id=acct.owner_id, - attachment_filter=acct.attachment_filter or settings.imap_attachment_filter, + allowed_categories=_resolve_categories_for_profile(acct.profile_id), ) # Record successful poll acct.last_checked_at = datetime.now(timezone.utc) @@ -251,7 +282,9 @@ def _pull_user_integration_imap() -> None: use_ssl = cfg.get("use_ssl", True) delete_after = cfg.get("delete_after_process", False) gmail_labels = cfg.get("gmail_apply_labels", True) - attachment_filter = cfg.get("attachment_filter") or settings.imap_attachment_filter + # Integrations can store a profile_id in config; fall back to global default + profile_id = cfg.get("profile_id") + allowed_categories = _resolve_categories_for_profile(profile_id) if not (host and username and password): logger.warning( @@ -271,7 +304,7 @@ def _pull_user_integration_imap() -> None: delete_after_process=delete_after, owner_id=integ.owner_id, gmail_apply_labels=gmail_labels, - attachment_filter=attachment_filter, + allowed_categories=allowed_categories, ) integ.last_used_at = datetime.now(timezone.utc) integ.last_error = None @@ -332,7 +365,7 @@ def pull_inbox( delete_after_process, owner_id=None, gmail_apply_labels=True, - attachment_filter=None, + allowed_categories=None, ): """ Connects to the IMAP inbox, fetches new unread emails from the last 3 days, @@ -349,13 +382,22 @@ def pull_inbox( attributed to this user via ``process_document`` / ``convert_to_pdf``. gmail_apply_labels: Whether to apply Gmail-specific labels and stars to processed emails. Only relevant for Gmail hosts. Defaults to True. - attachment_filter: Controls which attachment types to ingest. - ``'documents_only'`` (default) – PDFs and office files only. - ``'all'`` – all supported types including images. - ``None`` falls back to the global ``settings.imap_attachment_filter``. + allowed_categories: List of file-type category keys to ingest (e.g. + ``["pdf", "office", "images"]``). ``None`` falls back to the + global ``settings.imap_attachment_filter`` mapping. """ - resolved_filter = attachment_filter or settings.imap_attachment_filter - logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl) + if allowed_categories is None: + allowed_categories = _resolve_categories_for_profile(None) + + effective_mime_types, effective_extensions = get_allowed_types_for_categories(allowed_categories) + logger.info( + "Connecting to %s at %s:%s (SSL=%s) — categories: %s", + mailbox_key, + host, + port, + use_ssl, + allowed_categories, + ) processed_emails = load_processed_emails() try: @@ -414,9 +456,13 @@ def pull_inbox( logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key) continue - # Process attachments (and convert non-PDF files). - # We call the function without assigning its return value since it is not used. - fetch_attachments_and_enqueue(email_message, owner_id=owner_id, attachment_filter=resolved_filter) + # Process attachments using the resolved mime types / extensions. + fetch_attachments_and_enqueue( + email_message, + owner_id=owner_id, + effective_mime_types=effective_mime_types, + effective_extensions=effective_extensions, + ) if settings.imap_readonly_mode: logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key) @@ -448,19 +494,20 @@ def pull_inbox( def fetch_attachments_and_enqueue( email_message, owner_id: str | None = None, - attachment_filter: str | None = None, + effective_mime_types: frozenset[str] | None = None, + effective_extensions: frozenset[str] | None = None, ): """ Extracts attachments from the email and processes only allowed file types. - Files are first checked against the ``attachment_filter`` to determine which - broad categories are permitted, then validated against known MIME types / - extensions. + The caller is responsible for computing ``effective_mime_types`` and + ``effective_extensions`` from the relevant :class:`ImapIngestionProfile` (or + the global default) via :func:`app.utils.allowed_types.get_allowed_types_for_categories` + before calling this function. ``pull_inbox`` does this automatically. - Attachment filter values: - - ``'documents_only'`` (default): PDFs, office files (Word, Excel, PowerPoint, - OpenDocument, RTF), plain text, CSV, HTML, and Markdown. Images are skipped. - - ``'all'``: All supported file types, including images (JPEG, PNG, GIF, etc.). + If either set is ``None`` the function falls back to the default category list + so the function still works correctly when called directly in tests or from + other contexts. If the attachment is a PDF (by extension or MIME type), it is enqueued for upload; any other allowed file is enqueued for conversion to PDF. @@ -469,36 +516,13 @@ def fetch_attachments_and_enqueue( email_message: The parsed email message to extract attachments from. owner_id: Optional user identifier forwarded to ``process_document`` / ``convert_to_pdf`` for multi-tenant attribution. - attachment_filter: Override for the filter level. Defaults to - ``settings.imap_attachment_filter`` when not provided. + effective_mime_types: Pre-computed frozenset of allowed MIME type strings. + effective_extensions: Pre-computed frozenset of allowed file extension strings. Returns True if at least one allowed attachment was processed. """ - resolved_filter = attachment_filter or settings.imap_attachment_filter - - # Build the effective allowed MIME type set based on the filter - if resolved_filter == "all": - effective_mime_types = ALLOWED_MIME_TYPES - else: - # 'documents_only' (and any unrecognised value): exclude images - effective_mime_types = DOCUMENT_MIME_TYPES - - # Build the effective allowed extensions set (images excluded for documents_only) - if resolved_filter == "all": - effective_extensions = ALLOWED_EXTENSIONS - else: - image_extensions = { - ".jpg", - ".jpeg", - ".png", - ".gif", - ".bmp", - ".tiff", - ".tif", - ".webp", - ".svg", - } - effective_extensions = ALLOWED_EXTENSIONS - image_extensions + if effective_mime_types is None or effective_extensions is None: + effective_mime_types, effective_extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES) has_attachment = False for part in email_message.walk(): @@ -515,30 +539,14 @@ def fetch_attachments_and_enqueue( mime_type = part.get_content_type() file_ext = os.path.splitext(filename)[1].lower() - # Skip images when filter is documents_only - is_image = mime_type in IMAGE_MIME_TYPES or file_ext in { - ".jpg", - ".jpeg", - ".png", - ".gif", - ".bmp", - ".tiff", - ".tif", - ".webp", - ".svg", - } - if is_image and resolved_filter != "all": - logger.info( - "Skipping image attachment %s (MIME: %s) — attachment_filter=%s", - filename, - mime_type, - resolved_filter, - ) - continue - # Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension: - logger.info("Skipping attachment %s with MIME type %s", filename, mime_type) + logger.info( + "Skipping attachment %s (MIME: %s, ext: %s) — not in effective allowed set", + filename, + mime_type, + file_ext, + ) continue file_path = os.path.join(settings.workdir, filename) diff --git a/app/utils/allowed_types.py b/app/utils/allowed_types.py index 733ff10e..eaa37e6e 100644 --- a/app/utils/allowed_types.py +++ b/app/utils/allowed_types.py @@ -131,3 +131,157 @@ ALLOWED_EXTENSIONS: set[str] = { ".md", ".markdown", } + +# --------------------------------------------------------------------------- +# Fine-grained file-type categories used by IMAP ingestion profiles. +# Each category groups related MIME types and extensions so that users can +# enable/disable a logical collection of formats (e.g. "images") rather than +# having to manage individual MIME strings. +# --------------------------------------------------------------------------- + +FILE_TYPE_CATEGORIES: dict[str, dict] = { + "pdf": { + "label": "PDF", + "description": "PDF documents (.pdf)", + "mime_types": frozenset({"application/pdf"}), + "extensions": frozenset({".pdf"}), + }, + "office": { + "label": "Microsoft Office", + "description": "Word, Excel and PowerPoint files (.doc, .docx, .xls, .xlsx, .ppt, .pptx, …)", + "mime_types": frozenset( + { + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "application/vnd.ms-word.document.macroEnabled.12", + "application/vnd.ms-word.template.macroEnabled.12", + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.openxmlformats-officedocument.spreadsheetml.template", + "application/vnd.ms-excel.sheet.macroEnabled.12", + "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "application/vnd.ms-powerpoint", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.openxmlformats-officedocument.presentationml.template", + "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + } + ), + "extensions": frozenset( + { + ".doc", + ".docx", + ".docm", + ".dot", + ".dotx", + ".dotm", + ".xls", + ".xlsx", + ".xlsm", + ".xlsb", + ".xlt", + ".xltx", + ".xlw", + ".ppt", + ".pptx", + ".pptm", + ".pps", + ".ppsx", + ".pot", + ".potx", + } + ), + }, + "opendocument": { + "label": "OpenDocument (LibreOffice)", + "description": "LibreOffice / OpenOffice files (.odt, .ods, .odp, …)", + "mime_types": frozenset( + { + "application/vnd.oasis.opendocument.text", + "application/vnd.oasis.opendocument.spreadsheet", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.oasis.opendocument.graphics", + "application/vnd.oasis.opendocument.formula", + } + ), + "extensions": frozenset({".odt", ".ods", ".odp", ".odg", ".odf"}), + }, + "text": { + "label": "Text & Data", + "description": "Plain text, CSV and RTF files (.txt, .csv, .rtf)", + "mime_types": frozenset( + { + "text/plain", + "text/csv", + "application/rtf", + "text/rtf", + } + ), + "extensions": frozenset({".txt", ".csv", ".rtf"}), + }, + "web": { + "label": "Web & Markup", + "description": "HTML and Markdown files (.html, .htm, .md, .markdown)", + "mime_types": frozenset( + { + "text/html", + "text/markdown", + "text/x-markdown", + } + ), + "extensions": frozenset({".html", ".htm", ".md", ".markdown"}), + }, + "images": { + "label": "Images", + "description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)", + "mime_types": frozenset( + { + "image/jpeg", + "image/jpg", + "image/png", + "image/gif", + "image/bmp", + "image/tiff", + "image/webp", + "image/svg+xml", + } + ), + "extensions": frozenset( + { + ".jpg", + ".jpeg", + ".png", + ".gif", + ".bmp", + ".tiff", + ".tif", + ".webp", + ".svg", + } + ), + }, +} + +# Default categories for the "documents only" built-in profile (no images) +DEFAULT_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web"] +# All categories including images +ALL_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web", "images"] + + +def get_allowed_types_for_categories( + categories: list[str], +) -> tuple[frozenset[str], frozenset[str]]: + """Return ``(mime_types, extensions)`` for the given category list. + + Unknown category names are silently ignored so that future categories + don't break existing profiles. + """ + mime_types: set[str] = set() + extensions: set[str] = set() + for cat in categories: + info = FILE_TYPE_CATEGORIES.get(cat) + if info: + mime_types |= info["mime_types"] + extensions |= info["extensions"] + return frozenset(mime_types), frozenset(extensions) diff --git a/app/views/imap_accounts.py b/app/views/imap_accounts.py index 460c83ba..a8a2dcbd 100644 --- a/app/views/imap_accounts.py +++ b/app/views/imap_accounts.py @@ -1,11 +1,13 @@ """User-facing view for the per-user IMAP ingestion dashboard.""" +import json import logging from fastapi import Request from sqlalchemy.orm import Session -from app.models import UserImapAccount +from app.models import ImapIngestionProfile, UserImapAccount +from app.utils.allowed_types import FILE_TYPE_CATEGORIES from app.utils.subscription import get_tier, get_user_tier_id from app.utils.user_scope import get_current_owner_id from app.views.base import APIRouter, Depends, get_db, require_login, templates @@ -25,6 +27,22 @@ def _get_max_mailboxes(tier: dict) -> int | None: return max_mb +def _serialize_profile(profile: ImapIngestionProfile) -> dict: + """Serialize a profile for JSON embedding in the template.""" + try: + categories = json.loads(profile.allowed_categories) + except (ValueError, TypeError): + categories = [] + return { + "id": profile.id, + "name": profile.name, + "description": profile.description, + "owner_id": profile.owner_id, + "allowed_categories": categories, + "is_builtin": profile.is_builtin, + } + + @router.get("/imap-accounts") @require_login async def imap_accounts_page(request: Request, db: Session = Depends(get_db)): @@ -49,11 +67,39 @@ async def imap_accounts_page(request: Request, db: Session = Depends(get_db)): max_mailboxes = _get_max_mailboxes(tier) can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes) + # Load ingestion profiles: system-global + user's own + profiles: list[ImapIngestionProfile] = ( + db.query(ImapIngestionProfile) + .filter( + (ImapIngestionProfile.owner_id == None) # noqa: E711 + | (ImapIngestionProfile.owner_id == owner_id) + ) + .order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id) + .all() + if owner_id + else db.query(ImapIngestionProfile) + .filter(ImapIngestionProfile.owner_id == None) # noqa: E711 + .order_by(ImapIngestionProfile.id) + .all() + ) + + # Category definitions for the UI checkbox builder + categories = [ + { + "key": key, + "label": info["label"], + "description": info["description"], + } + for key, info in FILE_TYPE_CATEGORIES.items() + ] + return templates.TemplateResponse( "imap_accounts.html", { "request": request, "accounts": accounts, + "profiles": [_serialize_profile(p) for p in profiles], + "categories": categories, "current_count": current_count, "max_mailboxes": max_mailboxes, "can_add": can_add, diff --git a/frontend/templates/imap_accounts.html b/frontend/templates/imap_accounts.html index 16a6cdaa..ecbe7eaa 100644 --- a/frontend/templates/imap_accounts.html +++ b/frontend/templates/imap_accounts.html @@ -172,14 +172,9 @@ Delete after process - + - All attachments - - - - - Documents only + @@ -398,22 +393,45 @@ - - Attachment Types to Ingest + + Ingestion Profile - - Use global default - Documents only (PDF, Office, ODT, RTF, TXT, CSV) — no images - All supported types (including images) - - - Override the system-wide attachment filter for this mailbox. Leave blank to use the global default (documents only). + + + Use global default + + + + + + + + + + Controls which attachment types are ingested. Leave blank to use the global default (documents only). + + Manage profiles + + + + + : + + + @@ -524,10 +542,276 @@ + + + + + + + Ingestion Profiles + + + + + + + + Profiles define which attachment types are ingested from emails. Assign a profile to each IMAP account for fine-grained control. + + + + + + + + + + + built-in + + + + + + + + + + + + + + + + + + + + + + + + + + New Profile + + + + + + + + + + + + + + + + + + + + + Profile Name * + + + + + + + + Description + + + + + + + + + File Type Categories * + + + + + + + + + + + + + + Select at least one category. + + + + + + + + + + + + + Cancel + + + + + + + + + + + + + + + + + Delete Profile? + + + Are you sure you want to delete the profile + ? + IMAP accounts using this profile will fall back to the global default. + + + + Cancel + + + + + + Delete + + + + +
- Override the system-wide attachment filter for this mailbox. Leave blank to use the global default (documents only). +
+ Controls which attachment types are ingested. Leave blank to use the global default (documents only). + + Manage profiles +
+ Profiles define which attachment types are ingested from emails. Assign a profile to each IMAP account for fine-grained control. +
+ Select at least one category. +
+ Are you sure you want to delete the profile + ? + IMAP accounts using this profile will fall back to the global default. +