diff --git a/.env.demo b/.env.demo index a4a2b2f1..20c6b926 100644 --- a/.env.demo +++ b/.env.demo @@ -339,6 +339,12 @@ IMAP2_DELETE_AFTER_PROCESS=false # Use for pre-production instances that share a mailbox with production. IMAP_READONLY_MODE=false +# Controls which attachment types are ingested from IMAP emails. +# 'documents_only' (default) – PDFs and office files only; images are skipped. +# 'all' – all supported file types including images. +# Per-user IMAP accounts can override this global default. +IMAP_ATTACHMENT_FILTER=documents_only + # **Storage/Document Services** # Amazon S3 # S3_ENABLED=true # Set to false to disable S3 uploads without removing credentials diff --git a/app/api/__init__.py b/app/api/__init__.py index ca331229..28b9fbba 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -21,6 +21,7 @@ from app.api.files import router as files_router from app.api.google_drive import router as google_drive_router from app.api.i18n import router as i18n_router from app.api.imap_accounts import router as imap_accounts_router +from app.api.imap_profiles import router as imap_profiles_router from app.api.integrations import router as integrations_router from app.api.logs import router as logs_router from app.api.mobile import router as mobile_router @@ -83,6 +84,7 @@ router.include_router(onboarding_router) router.include_router(billing_router) router.include_router(pipelines_router) router.include_router(imap_accounts_router) +router.include_router(imap_profiles_router) router.include_router(integrations_router) router.include_router(notifications_router) router.include_router(scheduled_jobs_router) diff --git a/app/api/imap_accounts.py b/app/api/imap_accounts.py index 69b26c72..2aa9b039 100644 --- a/app/api/imap_accounts.py +++ b/app/api/imap_accounts.py @@ -110,6 +110,13 @@ class ImapAccountCreate(BaseModel): use_ssl: bool = Field(default=True, description="Use SSL/TLS connection") delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing") is_active: bool = Field(default=True, description="Whether to poll this mailbox") + profile_id: int | None = Field( + default=None, + description=( + "ID of the ImapIngestionProfile that controls which attachment types to ingest. " + "Null inherits the global imap_attachment_filter setting." + ), + ) class ImapAccountUpdate(BaseModel): @@ -123,6 +130,13 @@ class ImapAccountUpdate(BaseModel): use_ssl: bool | None = None delete_after_process: bool | None = None is_active: bool | None = None + profile_id: int | None = Field( + default=None, + description=( + "ID of the ImapIngestionProfile to use. " + "Explicitly sending null clears the override (falls back to global setting)." + ), + ) class ImapTestRequest(BaseModel): @@ -155,6 +169,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]: "use_ssl": acct.use_ssl, "delete_after_process": acct.delete_after_process, "is_active": acct.is_active, + "profile_id": acct.profile_id, "last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None, "last_error": acct.last_error, "created_at": acct.created_at.isoformat() if acct.created_at else None, @@ -222,6 +237,7 @@ def create_imap_account( use_ssl=body.use_ssl, delete_after_process=body.delete_after_process, is_active=body.is_active, + profile_id=body.profile_id, ) try: db.add(acct) @@ -277,6 +293,10 @@ def update_imap_account( acct.delete_after_process = body.delete_after_process if body.is_active is not None: acct.is_active = body.is_active + # profile_id: update whenever the field is explicitly present in the request payload + # (including sending null to clear the override). + if "profile_id" in body.model_fields_set: + acct.profile_id = body.profile_id # Reset last_error so the next poll gives a fresh result acct.last_error = None diff --git a/app/api/imap_profiles.py b/app/api/imap_profiles.py new file mode 100644 index 00000000..90502686 --- /dev/null +++ b/app/api/imap_profiles.py @@ -0,0 +1,257 @@ +"""API endpoints for managing IMAP ingestion profiles. + +Ingestion profiles allow fine-grained control over which attachment types are +accepted when ingesting emails via IMAP. Each profile carries a list of enabled +file-type categories (e.g. ``["pdf", "office", "images"]``) drawn from the +canonical set defined in :mod:`app.utils.allowed_types`. + +Built-in system profiles (``is_builtin=True``) are read-only and cannot be +deleted or modified. Users may create their own profiles which are private to +their ``owner_id``. System-level global profiles (``owner_id=None``) are visible +to all users but can only be created by administrators. +""" + +import json +import logging +from datetime import datetime, timezone +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from app.database import get_db +from app.models import ImapIngestionProfile +from app.utils.allowed_types import FILE_TYPE_CATEGORIES +from app.utils.user_scope import get_current_owner_id + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/imap-profiles", tags=["imap-profiles"]) + +DbSession = Annotated[Session, Depends(get_db)] + +# --------------------------------------------------------------------------- +# Auth helpers +# --------------------------------------------------------------------------- + + +def _get_owner_id(request: Request) -> str: + """Return the current user's owner ID, raising 401 if unauthenticated.""" + owner_id = get_current_owner_id(request) + if owner_id is None: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") + return owner_id + + +CurrentOwner = Annotated[str, Depends(_get_owner_id)] + +# --------------------------------------------------------------------------- +# Pydantic schemas +# --------------------------------------------------------------------------- + +_VALID_CATEGORIES = set(FILE_TYPE_CATEGORIES.keys()) + + +class ImapProfileCreate(BaseModel): + """Schema for creating a new ingestion profile.""" + + name: str = Field(..., min_length=1, max_length=255, description="Human-readable profile name") + description: str | None = Field(default=None, description="Optional description") + allowed_categories: list[str] = Field( + ..., + min_length=1, + description=(f"List of enabled file-type category keys. Valid values: {sorted(_VALID_CATEGORIES)}"), + ) + + +class ImapProfileUpdate(BaseModel): + """Schema for updating an existing profile (all fields optional).""" + + name: str | None = Field(default=None, min_length=1, max_length=255) + description: str | None = None + allowed_categories: list[str] | None = Field(default=None, min_length=1) + + +# --------------------------------------------------------------------------- +# Validation helpers +# --------------------------------------------------------------------------- + + +def _validate_categories(categories: list[str]) -> list[str]: + """Raise 422 if any category key is unknown; return the cleaned list.""" + unknown = [c for c in categories if c not in _VALID_CATEGORIES] + if unknown: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Unknown category key(s): {unknown}. Valid keys: {sorted(_VALID_CATEGORIES)}", + ) + # Deduplicate while preserving order + seen: set[str] = set() + result: list[str] = [] + for cat in categories: + if cat not in seen: + seen.add(cat) + result.append(cat) + return result + + +# --------------------------------------------------------------------------- +# Serialisation +# --------------------------------------------------------------------------- + + +def _to_response(profile: ImapIngestionProfile) -> dict[str, Any]: + """Serialize a profile row to a response dict.""" + try: + categories = json.loads(profile.allowed_categories) + except (ValueError, TypeError): + categories = [] + + # Enrich categories with display metadata + categories_detail = [ + { + "key": cat, + "label": FILE_TYPE_CATEGORIES[cat]["label"] if cat in FILE_TYPE_CATEGORIES else cat, + "description": FILE_TYPE_CATEGORIES[cat]["description"] if cat in FILE_TYPE_CATEGORIES else "", + } + for cat in categories + ] + + return { + "id": profile.id, + "name": profile.name, + "description": profile.description, + "owner_id": profile.owner_id, + "allowed_categories": categories, + "categories_detail": categories_detail, + "is_builtin": profile.is_builtin, + "created_at": profile.created_at.isoformat() if profile.created_at else None, + "updated_at": profile.updated_at.isoformat() if profile.updated_at else None, + } + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +@router.get("/categories", summary="List available file-type categories") +def list_categories(request: Request, owner_id: CurrentOwner) -> list[dict[str, Any]]: + """Return the full list of file-type categories that can be used in profiles.""" + return [ + { + "key": key, + "label": info["label"], + "description": info["description"], + } + for key, info in FILE_TYPE_CATEGORIES.items() + ] + + +@router.get("/", summary="List ingestion profiles visible to the current user") +def list_profiles(request: Request, db: DbSession, owner_id: CurrentOwner) -> list[dict[str, Any]]: + """Return all profiles: system-global (owner_id=NULL) and the user's own profiles.""" + profiles = ( + db.query(ImapIngestionProfile) + .filter( + # SQLAlchemy requires `== None` for IS NULL comparison in ORM filters + (ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711 + ) + .order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id) + .all() + ) + return [_to_response(p) for p in profiles] + + +@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new ingestion profile") +def create_profile(request: Request, body: ImapProfileCreate, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]: + """Create a new ingestion profile owned by the current user.""" + categories = _validate_categories(body.allowed_categories) + + profile = ImapIngestionProfile( + name=body.name, + description=body.description, + owner_id=owner_id, + allowed_categories=json.dumps(categories), + is_builtin=False, + ) + try: + db.add(profile) + db.commit() + db.refresh(profile) + except Exception: + db.rollback() + raise + + logger.info("User %s created IMAP ingestion profile %d ('%s')", owner_id, profile.id, body.name) + return _to_response(profile) + + +@router.get("/{profile_id}", summary="Get a single ingestion profile") +def get_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]: + """Return a single profile by ID. Only the owner or system profiles are accessible.""" + profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first() + if not profile or (profile.owner_id is not None and profile.owner_id != owner_id): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found") + return _to_response(profile) + + +@router.put("/{profile_id}", summary="Update an ingestion profile") +def update_profile( + profile_id: int, + request: Request, + body: ImapProfileUpdate, + db: DbSession, + owner_id: CurrentOwner, +) -> dict[str, Any]: + """Update an existing ingestion profile. Built-in profiles cannot be modified.""" + profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first() + if not profile or (profile.owner_id is not None and profile.owner_id != owner_id): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found") + if profile.is_builtin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Built-in profiles cannot be modified.", + ) + + if body.name is not None: + profile.name = body.name + if "description" in body.model_fields_set: + profile.description = body.description + if body.allowed_categories is not None: + categories = _validate_categories(body.allowed_categories) + profile.allowed_categories = json.dumps(categories) + + profile.updated_at = datetime.now(timezone.utc) + + try: + db.commit() + db.refresh(profile) + except Exception: + db.rollback() + raise + + logger.info("User %s updated IMAP ingestion profile %d", owner_id, profile_id) + return _to_response(profile) + + +@router.delete("/{profile_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an ingestion profile") +def delete_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> None: + """Delete an ingestion profile. Built-in profiles cannot be deleted.""" + profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first() + if not profile or (profile.owner_id is not None and profile.owner_id != owner_id): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found") + if profile.is_builtin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Built-in profiles cannot be deleted.", + ) + + try: + db.delete(profile) + db.commit() + except Exception: + db.rollback() + raise + + logger.info("User %s deleted IMAP ingestion profile %d", owner_id, profile_id) diff --git a/app/config.py b/app/config.py index 1275dbbc..d3a2080c 100644 --- a/app/config.py +++ b/app/config.py @@ -647,6 +647,17 @@ class Settings(BaseSettings): ), ) + imap_attachment_filter: str = Field( + default="documents_only", + description=( + "Controls which attachment types are ingested from IMAP emails. " + "Accepted values: " + "'documents_only' – ingest only PDFs and office files (Word, Excel, PowerPoint, ODT, etc.); " + "'all' – ingest all supported file types including images. " + "This is the global default; individual user IMAP accounts can override it." + ), + ) + # Batch processing settings processall_throttle_threshold: int = Field( default=20, diff --git a/app/models.py b/app/models.py index 9b3ea4d1..5ba06e81 100644 --- a/app/models.py +++ b/app/models.py @@ -401,6 +401,48 @@ class PipelineStep(Base): updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) +class ImapIngestionProfile(Base): + """Named ingestion profile controlling which attachment types are accepted from IMAP emails. + + Profiles group file-type categories (e.g. "pdf", "office", "images") so users + can precisely control what gets ingested from each mailbox. + + System-provided built-in profiles (``is_builtin=True``) are seeded by the + migration and cannot be deleted or renamed. Users may create their own profiles + (``owner_id`` set to their identifier) or rely on the global system profiles + (``owner_id=None``). + + ``allowed_categories`` stores a JSON list of category strings, e.g.:: + + '["pdf", "office", "opendocument", "text", "web"]' + + Valid category names are defined in ``app.utils.allowed_types.FILE_TYPE_CATEGORIES``. + """ + + __tablename__ = "imap_ingestion_profiles" + + id = Column(Integer, primary_key=True, index=True) + + # Human-readable profile name (e.g. "Documents Only", "Documents + Images") + name = Column(String(255), nullable=False) + + # Optional description shown in the UI + description = Column(Text, nullable=True) + + # Owner of this profile. NULL = global/system profile available to all users. + owner_id = Column(String, nullable=True, index=True) + + # JSON-encoded list of enabled category keys. Example: '["pdf","office","text"]' + # See FILE_TYPE_CATEGORIES in app/utils/allowed_types.py for valid values. + allowed_categories = Column(Text, nullable=False, default='["pdf","office","opendocument","text","web"]') + + # Built-in system profiles that cannot be deleted or modified via the API. + is_builtin = Column(Boolean, nullable=False, default=False) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + class UserImapAccount(Base): """Per-user IMAP ingestion account. @@ -439,6 +481,10 @@ class UserImapAccount(Base): # When True, emails are deleted from the mailbox after their attachments are processed delete_after_process = Column(Boolean, nullable=False, default=False) + # Optional reference to an ImapIngestionProfile. + # NULL means "use the global imap_attachment_filter setting" (system default). + profile_id = Column(Integer, ForeignKey("imap_ingestion_profiles.id"), nullable=True) + # When False the account is not polled by the periodic task (but not deleted) is_active = Column(Boolean, nullable=False, default=True) diff --git a/app/tasks/imap_tasks.py b/app/tasks/imap_tasks.py index 064ccf06..d47ab6c2 100644 --- a/app/tasks/imap_tasks.py +++ b/app/tasks/imap_tasks.py @@ -13,7 +13,11 @@ from celery import shared_task from app.config import settings from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task from app.tasks.process_document import process_document # Updated import -from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES +from app.utils.allowed_types import ( + ALL_CATEGORIES, + DEFAULT_CATEGORIES, + get_allowed_types_for_categories, +) # Database session for per-user IMAP accounts (imported lazily to avoid circular imports) _db_session_factory = None @@ -50,6 +54,38 @@ def _decrypt_imap_password(password: str | None) -> str | None: return decrypt_value(password) +def _resolve_categories_for_profile(profile_id: int | None) -> list[str]: + """Return the list of allowed categories for a profile ID. + + Loads the profile from the database. If ``profile_id`` is ``None`` or the + profile is not found, falls back to the global ``settings.imap_attachment_filter`` + string (``'documents_only'`` → default categories; ``'all'`` → all categories). + """ + if profile_id is not None: + try: + from app.models import ImapIngestionProfile + + db = _get_db_session() + try: + profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first() + if profile: + return json.loads(profile.allowed_categories) + finally: + db.close() + except Exception as exc: # noqa: BLE001 + logger.warning( + "Could not load IMAP ingestion profile %d (%s: %s) — using global default", + profile_id, + type(exc).__name__, + exc, + ) + + # Fall back to global setting + if settings.imap_attachment_filter == "all": + return ALL_CATEGORIES + return DEFAULT_CATEGORIES + + LOCK_KEY = "imap_lock" # Unique key for locking LOCK_EXPIRE = 300 # Lock expires in 5 minutes @@ -180,6 +216,7 @@ def _pull_user_imap_accounts() -> None: use_ssl=acct.use_ssl, delete_after_process=acct.delete_after_process, owner_id=acct.owner_id, + allowed_categories=_resolve_categories_for_profile(acct.profile_id), ) # Record successful poll acct.last_checked_at = datetime.now(timezone.utc) @@ -250,6 +287,9 @@ def _pull_user_integration_imap() -> None: use_ssl = cfg.get("use_ssl", True) delete_after = cfg.get("delete_after_process", False) gmail_labels = cfg.get("gmail_apply_labels", True) + # Integrations can store a profile_id in config; fall back to global default + profile_id = cfg.get("profile_id") + allowed_categories = _resolve_categories_for_profile(profile_id) if not (host and username and password): logger.warning( @@ -269,6 +309,7 @@ def _pull_user_integration_imap() -> None: delete_after_process=delete_after, owner_id=integ.owner_id, gmail_apply_labels=gmail_labels, + allowed_categories=allowed_categories, ) integ.last_used_at = datetime.now(timezone.utc) integ.last_error = None @@ -329,6 +370,7 @@ def pull_inbox( delete_after_process, owner_id=None, gmail_apply_labels=True, + allowed_categories=None, ): """ Connects to the IMAP inbox, fetches new unread emails from the last 3 days, @@ -345,8 +387,22 @@ def pull_inbox( attributed to this user via ``process_document`` / ``convert_to_pdf``. gmail_apply_labels: Whether to apply Gmail-specific labels and stars to processed emails. Only relevant for Gmail hosts. Defaults to True. + allowed_categories: List of file-type category keys to ingest (e.g. + ``["pdf", "office", "images"]``). ``None`` falls back to the + global ``settings.imap_attachment_filter`` mapping. """ - logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl) + if allowed_categories is None: + allowed_categories = _resolve_categories_for_profile(None) + + effective_mime_types, effective_extensions = get_allowed_types_for_categories(allowed_categories) + logger.info( + "Connecting to %s at %s:%s (SSL=%s) — categories: %s", + mailbox_key, + host, + port, + use_ssl, + allowed_categories, + ) processed_emails = load_processed_emails() try: @@ -405,9 +461,13 @@ def pull_inbox( logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key) continue - # Process attachments (and convert non-PDF files). - # We call the function without assigning its return value since it is not used. - fetch_attachments_and_enqueue(email_message, owner_id=owner_id) + # Process attachments using the resolved mime types / extensions. + fetch_attachments_and_enqueue( + email_message, + owner_id=owner_id, + effective_mime_types=effective_mime_types, + effective_extensions=effective_extensions, + ) if settings.imap_readonly_mode: logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key) @@ -436,27 +496,23 @@ def pull_inbox( logger.exception("Error pulling mailbox %s: %s", mailbox_key, e) -def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None): +def fetch_attachments_and_enqueue( + email_message, + owner_id: str | None = None, + effective_mime_types: frozenset[str] | None = None, + effective_extensions: frozenset[str] | None = None, +): """ Extracts attachments from the email and processes only allowed file types. - Files are accepted if either: - 1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR - 2. They have a '.pdf' file extension (regardless of MIME type) + The caller is responsible for computing ``effective_mime_types`` and + ``effective_extensions`` from the relevant :class:`ImapIngestionProfile` (or + the global default) via :func:`app.utils.allowed_types.get_allowed_types_for_categories` + before calling this function. ``pull_inbox`` does this automatically. - Allowed file types include: - - PDF: application/pdf or *.pdf extension - - Microsoft Office files: - - Word: application/msword, - application/vnd.openxmlformats-officedocument.wordprocessingml.document - - Excel: application/vnd.ms-excel, - application/vnd.openxmlformats-officedocument.spreadsheetml.sheet - - PowerPoint: application/vnd.ms-powerpoint, - application/vnd.openxmlformats-officedocument.presentationml.presentation - - Other meaningful attachments: - - Plain text: text/plain - - CSV: text/csv - - Rich Text Format: application/rtf, text/rtf + If either set is ``None`` the function falls back to the default category list + so the function still works correctly when called directly in tests or from + other contexts. If the attachment is a PDF (by extension or MIME type), it is enqueued for upload; any other allowed file is enqueued for conversion to PDF. @@ -465,9 +521,14 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None): email_message: The parsed email message to extract attachments from. owner_id: Optional user identifier forwarded to ``process_document`` / ``convert_to_pdf`` for multi-tenant attribution. + effective_mime_types: Pre-computed frozenset of allowed MIME type strings. + effective_extensions: Pre-computed frozenset of allowed file extension strings. Returns True if at least one allowed attachment was processed. """ + if effective_mime_types is None or effective_extensions is None: + effective_mime_types, effective_extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES) + has_attachment = False for part in email_message.walk(): if part.get_content_maintype() == "multipart": @@ -482,9 +543,15 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None): mime_type = part.get_content_type() file_ext = os.path.splitext(filename)[1].lower() + # Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension - if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension: - logger.info("Skipping attachment %s with MIME type %s", filename, mime_type) + if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension: + logger.info( + "Skipping attachment %s (MIME: %s, ext: %s) — not in effective allowed set", + filename, + mime_type, + file_ext, + ) continue file_path = os.path.join(settings.workdir, filename) @@ -495,7 +562,7 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None): if mime_type == "application/pdf" or is_pdf_by_extension: process_document.delay(file_path, owner_id=owner_id) logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type) - elif mime_type in ALLOWED_MIME_TYPES: + elif mime_type in effective_mime_types: # Other allowed files are sent for conversion convert_to_pdf.delay(file_path, owner_id=owner_id) logger.info("Enqueued file for conversion to PDF: %s", filename) diff --git a/app/utils/allowed_types.py b/app/utils/allowed_types.py index 733ff10e..eaa37e6e 100644 --- a/app/utils/allowed_types.py +++ b/app/utils/allowed_types.py @@ -131,3 +131,157 @@ ALLOWED_EXTENSIONS: set[str] = { ".md", ".markdown", } + +# --------------------------------------------------------------------------- +# Fine-grained file-type categories used by IMAP ingestion profiles. +# Each category groups related MIME types and extensions so that users can +# enable/disable a logical collection of formats (e.g. "images") rather than +# having to manage individual MIME strings. +# --------------------------------------------------------------------------- + +FILE_TYPE_CATEGORIES: dict[str, dict] = { + "pdf": { + "label": "PDF", + "description": "PDF documents (.pdf)", + "mime_types": frozenset({"application/pdf"}), + "extensions": frozenset({".pdf"}), + }, + "office": { + "label": "Microsoft Office", + "description": "Word, Excel and PowerPoint files (.doc, .docx, .xls, .xlsx, .ppt, .pptx, …)", + "mime_types": frozenset( + { + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "application/vnd.ms-word.document.macroEnabled.12", + "application/vnd.ms-word.template.macroEnabled.12", + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.openxmlformats-officedocument.spreadsheetml.template", + "application/vnd.ms-excel.sheet.macroEnabled.12", + "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "application/vnd.ms-powerpoint", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.openxmlformats-officedocument.presentationml.template", + "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + } + ), + "extensions": frozenset( + { + ".doc", + ".docx", + ".docm", + ".dot", + ".dotx", + ".dotm", + ".xls", + ".xlsx", + ".xlsm", + ".xlsb", + ".xlt", + ".xltx", + ".xlw", + ".ppt", + ".pptx", + ".pptm", + ".pps", + ".ppsx", + ".pot", + ".potx", + } + ), + }, + "opendocument": { + "label": "OpenDocument (LibreOffice)", + "description": "LibreOffice / OpenOffice files (.odt, .ods, .odp, …)", + "mime_types": frozenset( + { + "application/vnd.oasis.opendocument.text", + "application/vnd.oasis.opendocument.spreadsheet", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.oasis.opendocument.graphics", + "application/vnd.oasis.opendocument.formula", + } + ), + "extensions": frozenset({".odt", ".ods", ".odp", ".odg", ".odf"}), + }, + "text": { + "label": "Text & Data", + "description": "Plain text, CSV and RTF files (.txt, .csv, .rtf)", + "mime_types": frozenset( + { + "text/plain", + "text/csv", + "application/rtf", + "text/rtf", + } + ), + "extensions": frozenset({".txt", ".csv", ".rtf"}), + }, + "web": { + "label": "Web & Markup", + "description": "HTML and Markdown files (.html, .htm, .md, .markdown)", + "mime_types": frozenset( + { + "text/html", + "text/markdown", + "text/x-markdown", + } + ), + "extensions": frozenset({".html", ".htm", ".md", ".markdown"}), + }, + "images": { + "label": "Images", + "description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)", + "mime_types": frozenset( + { + "image/jpeg", + "image/jpg", + "image/png", + "image/gif", + "image/bmp", + "image/tiff", + "image/webp", + "image/svg+xml", + } + ), + "extensions": frozenset( + { + ".jpg", + ".jpeg", + ".png", + ".gif", + ".bmp", + ".tiff", + ".tif", + ".webp", + ".svg", + } + ), + }, +} + +# Default categories for the "documents only" built-in profile (no images) +DEFAULT_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web"] +# All categories including images +ALL_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web", "images"] + + +def get_allowed_types_for_categories( + categories: list[str], +) -> tuple[frozenset[str], frozenset[str]]: + """Return ``(mime_types, extensions)`` for the given category list. + + Unknown category names are silently ignored so that future categories + don't break existing profiles. + """ + mime_types: set[str] = set() + extensions: set[str] = set() + for cat in categories: + info = FILE_TYPE_CATEGORIES.get(cat) + if info: + mime_types |= info["mime_types"] + extensions |= info["extensions"] + return frozenset(mime_types), frozenset(extensions) diff --git a/app/utils/settings_service.py b/app/utils/settings_service.py index 727e4861..dceaa943 100644 --- a/app/utils/settings_service.py +++ b/app/utils/settings_service.py @@ -1660,6 +1660,18 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + "imap_attachment_filter": { + "category": "IMAP", + "description": ( + "Controls which attachment types are ingested from IMAP emails. " + "Accepted values: 'documents_only' (PDFs and office files only, default) or 'all' (including images). " + "Per-user IMAP accounts can override this global default." + ), + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, # Monitoring - Uptime Kuma "uptime_kuma_url": { "category": "Monitoring", diff --git a/app/views/imap_accounts.py b/app/views/imap_accounts.py index 460c83ba..c952a8cd 100644 --- a/app/views/imap_accounts.py +++ b/app/views/imap_accounts.py @@ -1,11 +1,13 @@ """User-facing view for the per-user IMAP ingestion dashboard.""" +import json import logging from fastapi import Request from sqlalchemy.orm import Session -from app.models import UserImapAccount +from app.models import ImapIngestionProfile, UserImapAccount +from app.utils.allowed_types import DEFAULT_CATEGORIES, FILE_TYPE_CATEGORIES from app.utils.subscription import get_tier, get_user_tier_id from app.utils.user_scope import get_current_owner_id from app.views.base import APIRouter, Depends, get_db, require_login, templates @@ -25,6 +27,22 @@ def _get_max_mailboxes(tier: dict) -> int | None: return max_mb +def _serialize_profile(profile: ImapIngestionProfile) -> dict: + """Serialize a profile for JSON embedding in the template.""" + try: + categories = json.loads(profile.allowed_categories) + except (ValueError, TypeError): + categories = [] + return { + "id": profile.id, + "name": profile.name, + "description": profile.description, + "owner_id": profile.owner_id, + "allowed_categories": categories, + "is_builtin": profile.is_builtin, + } + + @router.get("/imap-accounts") @require_login async def imap_accounts_page(request: Request, db: Session = Depends(get_db)): @@ -49,11 +67,35 @@ async def imap_accounts_page(request: Request, db: Session = Depends(get_db)): max_mailboxes = _get_max_mailboxes(tier) can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes) + # Load ingestion profiles: system-global + user's own + profiles = ( + db.query(ImapIngestionProfile) + .filter( + # SQLAlchemy requires `== None` for IS NULL comparison in ORM filters + (ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711 + ) + .order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id) + .all() + ) + + # Category definitions for the UI checkbox builder + categories = [ + { + "key": key, + "label": info["label"], + "description": info["description"], + } + for key, info in FILE_TYPE_CATEGORIES.items() + ] + return templates.TemplateResponse( "imap_accounts.html", { "request": request, "accounts": accounts, + "profiles": [_serialize_profile(p) for p in profiles], + "categories": categories, + "default_categories": DEFAULT_CATEGORIES, "current_count": current_count, "max_mailboxes": max_mailboxes, "can_add": can_add, diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index 6d23b197..643ac930 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -304,6 +304,42 @@ DocuElevate can automatically pull document attachments from IMAP mailboxes — | `IMAP1_SSL` | Use SSL (`true`/`false`). | `true` | | `IMAP1_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll for new mail. | `5` | | `IMAP_READONLY_MODE` | When `true`, fetches and processes attachments but does **not** modify the mailbox (no starring, labeling, deleting, or flag changes). Use for pre-production instances sharing a mailbox with production. Default: `false`. | `false` | +| `IMAP_ATTACHMENT_FILTER` | System-wide fallback for which attachment types are ingested when no ingestion profile is assigned to a mailbox. `documents_only` (default) ingests PDFs and office files only — images are skipped. `all` ingests every supported file type including images. Individual IMAP accounts can override this using ingestion profiles. | `documents_only` | + +#### IMAP Ingestion Profiles + +For fine-grained control, DocuElevate supports **Ingestion Profiles** — named configurations that let you choose exactly which file-type categories to accept from each mailbox. + +Each profile contains a list of enabled **categories**: + +| Category | Description | +|----------|-------------| +| `pdf` | PDF documents (`.pdf`) | +| `office` | Microsoft Office files (Word, Excel, PowerPoint — `.docx`, `.xlsx`, `.pptx`, …) | +| `opendocument` | LibreOffice/OpenOffice files (`.odt`, `.ods`, `.odp`, …) | +| `text` | Plain text, CSV and RTF files (`.txt`, `.csv`, `.rtf`) | +| `web` | HTML and Markdown files (`.html`, `.htm`, `.md`, `.markdown`) | +| `images` | Image files (`.jpg`, `.png`, `.gif`, `.bmp`, `.tiff`, `.webp`, `.svg`) | + +Two built-in system profiles are seeded automatically: + +| Profile | Categories | +|---------|------------| +| **Documents Only** | pdf, office, opendocument, text, web (no images) | +| **All Files** | All categories, including images | + +Users can create their own custom profiles via the **Email Ingestion** dashboard (`/imap-accounts`) by clicking the **Manage profiles** link or the **+** button next to the profile dropdown. Custom profiles are private to the creating user and can be freely edited or deleted. + +**API endpoints for ingestion profiles:** + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `GET` | `/api/imap-profiles/` | List all visible profiles (system + user's own) | +| `POST` | `/api/imap-profiles/` | Create a new profile | +| `GET` | `/api/imap-profiles/categories` | List available file-type categories | +| `GET` | `/api/imap-profiles/{id}` | Get a single profile | +| `PUT` | `/api/imap-profiles/{id}` | Update a profile (not built-in) | +| `DELETE` | `/api/imap-profiles/{id}` | Delete a profile (not built-in) | #### Per-User IMAP Integrations diff --git a/docs/howto/EmailIngestion.md b/docs/howto/EmailIngestion.md index 1a911f6f..bfb6b735 100644 --- a/docs/howto/EmailIngestion.md +++ b/docs/howto/EmailIngestion.md @@ -63,6 +63,54 @@ DocuElevate will process the following attachment types from emails: | TIFF | `.tif`, `.tiff` | Common format from older scanners/fax | | Multi-page TIFF | `.tif` | Full multi-page support | +### Controlling Which Attachment Types Are Ingested + +By default, DocuElevate only ingests **document** attachments (PDFs, Word, Excel, PowerPoint, OpenDocument, RTF, TXT, CSV, HTML, Markdown). Images are **not** ingested by default — this prevents cluttering your document archive with inline images or unrelated photo attachments. + +#### Global Default (Admin Setting) + +Set the `IMAP_ATTACHMENT_FILTER` environment variable to control the system-wide fallback when no ingestion profile is assigned to a mailbox: + +| Value | Behaviour | +|-------|-----------| +| `documents_only` | **(Default)** Only PDFs and office/document files. Images are skipped. | +| `all` | All supported file types, including images. | + +```env +IMAP_ATTACHMENT_FILTER=documents_only +``` + +#### Ingestion Profiles (Fine-Grained Per-Mailbox Control) + +For precise control, you can create **Ingestion Profiles** that let you pick exactly which file-type categories to accept from each mailbox. This is more powerful than the binary global toggle and works independently per mailbox. + +**Available categories:** + +| Category | File types included | +|----------|---------------------| +| PDF | `.pdf` | +| Microsoft Office | `.doc`, `.docx`, `.xls`, `.xlsx`, `.ppt`, `.pptx`, and macro-enabled variants | +| OpenDocument | `.odt`, `.ods`, `.odp`, `.odg`, `.odf` (LibreOffice / OpenOffice) | +| Text & Data | `.txt`, `.csv`, `.rtf` | +| Web & Markup | `.html`, `.htm`, `.md`, `.markdown` | +| Images | `.jpg`, `.png`, `.gif`, `.bmp`, `.tiff`, `.webp`, `.svg` | + +**Managing profiles:** + +1. Go to **Email Ingestion** (`/imap-accounts`) +2. Click **Manage profiles** (or the **+** icon next to the profile dropdown) +3. Create a new profile, give it a name, and tick the categories you want +4. When adding or editing a mailbox, select your profile from the dropdown + +Two built-in profiles are always available and cannot be deleted: + +- **Documents Only** — PDF, Office, OpenDocument, Text, Web (no images) +- **All Files** — all categories including images + +Users can also create unlimited **custom profiles** to mix and match exactly the categories they need per mailbox (e.g. a scanner mailbox that only accepts PDFs, or a finance mailbox that accepts Office and CSV but not images). + +Custom profiles are created via the UI or the `/api/imap-profiles/` API. + --- ## Setting Up Your Scanner/Device diff --git a/frontend/templates/imap_accounts.html b/frontend/templates/imap_accounts.html index e22614e6..226bb190 100644 --- a/frontend/templates/imap_accounts.html +++ b/frontend/templates/imap_accounts.html @@ -172,6 +172,11 @@ Delete after process +