Merge pull request #604 from christianlouis/copilot/configure-attachment-ingestion

feat(imap): fine-grained attachment ingestion profiles with per-category selection
This commit is contained in:
Christian Krakau-Louis
2026-03-12 08:53:45 +01:00
committed by GitHub
18 changed files with 1614 additions and 30 deletions
+6
View File
@@ -339,6 +339,12 @@ IMAP2_DELETE_AFTER_PROCESS=false
# Use for pre-production instances that share a mailbox with production. # Use for pre-production instances that share a mailbox with production.
IMAP_READONLY_MODE=false IMAP_READONLY_MODE=false
# Controls which attachment types are ingested from IMAP emails.
# 'documents_only' (default) PDFs and office files only; images are skipped.
# 'all' all supported file types including images.
# Per-user IMAP accounts can override this global default.
IMAP_ATTACHMENT_FILTER=documents_only
# **Storage/Document Services** # **Storage/Document Services**
# Amazon S3 # Amazon S3
# S3_ENABLED=true # Set to false to disable S3 uploads without removing credentials # S3_ENABLED=true # Set to false to disable S3 uploads without removing credentials
+2
View File
@@ -21,6 +21,7 @@ from app.api.files import router as files_router
from app.api.google_drive import router as google_drive_router from app.api.google_drive import router as google_drive_router
from app.api.i18n import router as i18n_router from app.api.i18n import router as i18n_router
from app.api.imap_accounts import router as imap_accounts_router from app.api.imap_accounts import router as imap_accounts_router
from app.api.imap_profiles import router as imap_profiles_router
from app.api.integrations import router as integrations_router from app.api.integrations import router as integrations_router
from app.api.logs import router as logs_router from app.api.logs import router as logs_router
from app.api.mobile import router as mobile_router from app.api.mobile import router as mobile_router
@@ -83,6 +84,7 @@ router.include_router(onboarding_router)
router.include_router(billing_router) router.include_router(billing_router)
router.include_router(pipelines_router) router.include_router(pipelines_router)
router.include_router(imap_accounts_router) router.include_router(imap_accounts_router)
router.include_router(imap_profiles_router)
router.include_router(integrations_router) router.include_router(integrations_router)
router.include_router(notifications_router) router.include_router(notifications_router)
router.include_router(scheduled_jobs_router) router.include_router(scheduled_jobs_router)
+20
View File
@@ -110,6 +110,13 @@ class ImapAccountCreate(BaseModel):
use_ssl: bool = Field(default=True, description="Use SSL/TLS connection") use_ssl: bool = Field(default=True, description="Use SSL/TLS connection")
delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing") delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing")
is_active: bool = Field(default=True, description="Whether to poll this mailbox") is_active: bool = Field(default=True, description="Whether to poll this mailbox")
profile_id: int | None = Field(
default=None,
description=(
"ID of the ImapIngestionProfile that controls which attachment types to ingest. "
"Null inherits the global imap_attachment_filter setting."
),
)
class ImapAccountUpdate(BaseModel): class ImapAccountUpdate(BaseModel):
@@ -123,6 +130,13 @@ class ImapAccountUpdate(BaseModel):
use_ssl: bool | None = None use_ssl: bool | None = None
delete_after_process: bool | None = None delete_after_process: bool | None = None
is_active: bool | None = None is_active: bool | None = None
profile_id: int | None = Field(
default=None,
description=(
"ID of the ImapIngestionProfile to use. "
"Explicitly sending null clears the override (falls back to global setting)."
),
)
class ImapTestRequest(BaseModel): class ImapTestRequest(BaseModel):
@@ -155,6 +169,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]:
"use_ssl": acct.use_ssl, "use_ssl": acct.use_ssl,
"delete_after_process": acct.delete_after_process, "delete_after_process": acct.delete_after_process,
"is_active": acct.is_active, "is_active": acct.is_active,
"profile_id": acct.profile_id,
"last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None, "last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None,
"last_error": acct.last_error, "last_error": acct.last_error,
"created_at": acct.created_at.isoformat() if acct.created_at else None, "created_at": acct.created_at.isoformat() if acct.created_at else None,
@@ -222,6 +237,7 @@ def create_imap_account(
use_ssl=body.use_ssl, use_ssl=body.use_ssl,
delete_after_process=body.delete_after_process, delete_after_process=body.delete_after_process,
is_active=body.is_active, is_active=body.is_active,
profile_id=body.profile_id,
) )
try: try:
db.add(acct) db.add(acct)
@@ -277,6 +293,10 @@ def update_imap_account(
acct.delete_after_process = body.delete_after_process acct.delete_after_process = body.delete_after_process
if body.is_active is not None: if body.is_active is not None:
acct.is_active = body.is_active acct.is_active = body.is_active
# profile_id: update whenever the field is explicitly present in the request payload
# (including sending null to clear the override).
if "profile_id" in body.model_fields_set:
acct.profile_id = body.profile_id
# Reset last_error so the next poll gives a fresh result # Reset last_error so the next poll gives a fresh result
acct.last_error = None acct.last_error = None
+257
View File
@@ -0,0 +1,257 @@
"""API endpoints for managing IMAP ingestion profiles.
Ingestion profiles allow fine-grained control over which attachment types are
accepted when ingesting emails via IMAP. Each profile carries a list of enabled
file-type categories (e.g. ``["pdf", "office", "images"]``) drawn from the
canonical set defined in :mod:`app.utils.allowed_types`.
Built-in system profiles (``is_builtin=True``) are read-only and cannot be
deleted or modified. Users may create their own profiles which are private to
their ``owner_id``. System-level global profiles (``owner_id=None``) are visible
to all users but can only be created by administrators.
"""
import json
import logging
from datetime import datetime, timezone
from typing import Annotated, Any
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from app.database import get_db
from app.models import ImapIngestionProfile
from app.utils.allowed_types import FILE_TYPE_CATEGORIES
from app.utils.user_scope import get_current_owner_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/imap-profiles", tags=["imap-profiles"])
DbSession = Annotated[Session, Depends(get_db)]
# ---------------------------------------------------------------------------
# Auth helpers
# ---------------------------------------------------------------------------
def _get_owner_id(request: Request) -> str:
"""Return the current user's owner ID, raising 401 if unauthenticated."""
owner_id = get_current_owner_id(request)
if owner_id is None:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
return owner_id
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
# ---------------------------------------------------------------------------
# Pydantic schemas
# ---------------------------------------------------------------------------
_VALID_CATEGORIES = set(FILE_TYPE_CATEGORIES.keys())
class ImapProfileCreate(BaseModel):
"""Schema for creating a new ingestion profile."""
name: str = Field(..., min_length=1, max_length=255, description="Human-readable profile name")
description: str | None = Field(default=None, description="Optional description")
allowed_categories: list[str] = Field(
...,
min_length=1,
description=(f"List of enabled file-type category keys. Valid values: {sorted(_VALID_CATEGORIES)}"),
)
class ImapProfileUpdate(BaseModel):
"""Schema for updating an existing profile (all fields optional)."""
name: str | None = Field(default=None, min_length=1, max_length=255)
description: str | None = None
allowed_categories: list[str] | None = Field(default=None, min_length=1)
# ---------------------------------------------------------------------------
# Validation helpers
# ---------------------------------------------------------------------------
def _validate_categories(categories: list[str]) -> list[str]:
"""Raise 422 if any category key is unknown; return the cleaned list."""
unknown = [c for c in categories if c not in _VALID_CATEGORIES]
if unknown:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Unknown category key(s): {unknown}. Valid keys: {sorted(_VALID_CATEGORIES)}",
)
# Deduplicate while preserving order
seen: set[str] = set()
result: list[str] = []
for cat in categories:
if cat not in seen:
seen.add(cat)
result.append(cat)
return result
# ---------------------------------------------------------------------------
# Serialisation
# ---------------------------------------------------------------------------
def _to_response(profile: ImapIngestionProfile) -> dict[str, Any]:
"""Serialize a profile row to a response dict."""
try:
categories = json.loads(profile.allowed_categories)
except (ValueError, TypeError):
categories = []
# Enrich categories with display metadata
categories_detail = [
{
"key": cat,
"label": FILE_TYPE_CATEGORIES[cat]["label"] if cat in FILE_TYPE_CATEGORIES else cat,
"description": FILE_TYPE_CATEGORIES[cat]["description"] if cat in FILE_TYPE_CATEGORIES else "",
}
for cat in categories
]
return {
"id": profile.id,
"name": profile.name,
"description": profile.description,
"owner_id": profile.owner_id,
"allowed_categories": categories,
"categories_detail": categories_detail,
"is_builtin": profile.is_builtin,
"created_at": profile.created_at.isoformat() if profile.created_at else None,
"updated_at": profile.updated_at.isoformat() if profile.updated_at else None,
}
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@router.get("/categories", summary="List available file-type categories")
def list_categories(request: Request, owner_id: CurrentOwner) -> list[dict[str, Any]]:
"""Return the full list of file-type categories that can be used in profiles."""
return [
{
"key": key,
"label": info["label"],
"description": info["description"],
}
for key, info in FILE_TYPE_CATEGORIES.items()
]
@router.get("/", summary="List ingestion profiles visible to the current user")
def list_profiles(request: Request, db: DbSession, owner_id: CurrentOwner) -> list[dict[str, Any]]:
"""Return all profiles: system-global (owner_id=NULL) and the user's own profiles."""
profiles = (
db.query(ImapIngestionProfile)
.filter(
# SQLAlchemy requires `== None` for IS NULL comparison in ORM filters
(ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711
)
.order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
.all()
)
return [_to_response(p) for p in profiles]
@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new ingestion profile")
def create_profile(request: Request, body: ImapProfileCreate, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
"""Create a new ingestion profile owned by the current user."""
categories = _validate_categories(body.allowed_categories)
profile = ImapIngestionProfile(
name=body.name,
description=body.description,
owner_id=owner_id,
allowed_categories=json.dumps(categories),
is_builtin=False,
)
try:
db.add(profile)
db.commit()
db.refresh(profile)
except Exception:
db.rollback()
raise
logger.info("User %s created IMAP ingestion profile %d ('%s')", owner_id, profile.id, body.name)
return _to_response(profile)
@router.get("/{profile_id}", summary="Get a single ingestion profile")
def get_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
"""Return a single profile by ID. Only the owner or system profiles are accessible."""
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
return _to_response(profile)
@router.put("/{profile_id}", summary="Update an ingestion profile")
def update_profile(
profile_id: int,
request: Request,
body: ImapProfileUpdate,
db: DbSession,
owner_id: CurrentOwner,
) -> dict[str, Any]:
"""Update an existing ingestion profile. Built-in profiles cannot be modified."""
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
if profile.is_builtin:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Built-in profiles cannot be modified.",
)
if body.name is not None:
profile.name = body.name
if "description" in body.model_fields_set:
profile.description = body.description
if body.allowed_categories is not None:
categories = _validate_categories(body.allowed_categories)
profile.allowed_categories = json.dumps(categories)
profile.updated_at = datetime.now(timezone.utc)
try:
db.commit()
db.refresh(profile)
except Exception:
db.rollback()
raise
logger.info("User %s updated IMAP ingestion profile %d", owner_id, profile_id)
return _to_response(profile)
@router.delete("/{profile_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an ingestion profile")
def delete_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> None:
"""Delete an ingestion profile. Built-in profiles cannot be deleted."""
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
if profile.is_builtin:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Built-in profiles cannot be deleted.",
)
try:
db.delete(profile)
db.commit()
except Exception:
db.rollback()
raise
logger.info("User %s deleted IMAP ingestion profile %d", owner_id, profile_id)
+11
View File
@@ -647,6 +647,17 @@ class Settings(BaseSettings):
), ),
) )
imap_attachment_filter: str = Field(
default="documents_only",
description=(
"Controls which attachment types are ingested from IMAP emails. "
"Accepted values: "
"'documents_only' ingest only PDFs and office files (Word, Excel, PowerPoint, ODT, etc.); "
"'all' ingest all supported file types including images. "
"This is the global default; individual user IMAP accounts can override it."
),
)
# Batch processing settings # Batch processing settings
processall_throttle_threshold: int = Field( processall_throttle_threshold: int = Field(
default=20, default=20,
+46
View File
@@ -401,6 +401,48 @@ class PipelineStep(Base):
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class ImapIngestionProfile(Base):
"""Named ingestion profile controlling which attachment types are accepted from IMAP emails.
Profiles group file-type categories (e.g. "pdf", "office", "images") so users
can precisely control what gets ingested from each mailbox.
System-provided built-in profiles (``is_builtin=True``) are seeded by the
migration and cannot be deleted or renamed. Users may create their own profiles
(``owner_id`` set to their identifier) or rely on the global system profiles
(``owner_id=None``).
``allowed_categories`` stores a JSON list of category strings, e.g.::
'["pdf", "office", "opendocument", "text", "web"]'
Valid category names are defined in ``app.utils.allowed_types.FILE_TYPE_CATEGORIES``.
"""
__tablename__ = "imap_ingestion_profiles"
id = Column(Integer, primary_key=True, index=True)
# Human-readable profile name (e.g. "Documents Only", "Documents + Images")
name = Column(String(255), nullable=False)
# Optional description shown in the UI
description = Column(Text, nullable=True)
# Owner of this profile. NULL = global/system profile available to all users.
owner_id = Column(String, nullable=True, index=True)
# JSON-encoded list of enabled category keys. Example: '["pdf","office","text"]'
# See FILE_TYPE_CATEGORIES in app/utils/allowed_types.py for valid values.
allowed_categories = Column(Text, nullable=False, default='["pdf","office","opendocument","text","web"]')
# Built-in system profiles that cannot be deleted or modified via the API.
is_builtin = Column(Boolean, nullable=False, default=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class UserImapAccount(Base): class UserImapAccount(Base):
"""Per-user IMAP ingestion account. """Per-user IMAP ingestion account.
@@ -439,6 +481,10 @@ class UserImapAccount(Base):
# When True, emails are deleted from the mailbox after their attachments are processed # When True, emails are deleted from the mailbox after their attachments are processed
delete_after_process = Column(Boolean, nullable=False, default=False) delete_after_process = Column(Boolean, nullable=False, default=False)
# Optional reference to an ImapIngestionProfile.
# NULL means "use the global imap_attachment_filter setting" (system default).
profile_id = Column(Integer, ForeignKey("imap_ingestion_profiles.id"), nullable=True)
# When False the account is not polled by the periodic task (but not deleted) # When False the account is not polled by the periodic task (but not deleted)
is_active = Column(Boolean, nullable=False, default=True) is_active = Column(Boolean, nullable=False, default=True)
+92 -25
View File
@@ -13,7 +13,11 @@ from celery import shared_task
from app.config import settings from app.config import settings
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
from app.tasks.process_document import process_document # Updated import from app.tasks.process_document import process_document # Updated import
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES from app.utils.allowed_types import (
ALL_CATEGORIES,
DEFAULT_CATEGORIES,
get_allowed_types_for_categories,
)
# Database session for per-user IMAP accounts (imported lazily to avoid circular imports) # Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
_db_session_factory = None _db_session_factory = None
@@ -50,6 +54,38 @@ def _decrypt_imap_password(password: str | None) -> str | None:
return decrypt_value(password) return decrypt_value(password)
def _resolve_categories_for_profile(profile_id: int | None) -> list[str]:
"""Return the list of allowed categories for a profile ID.
Loads the profile from the database. If ``profile_id`` is ``None`` or the
profile is not found, falls back to the global ``settings.imap_attachment_filter``
string (``'documents_only'`` → default categories; ``'all'`` → all categories).
"""
if profile_id is not None:
try:
from app.models import ImapIngestionProfile
db = _get_db_session()
try:
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
if profile:
return json.loads(profile.allowed_categories)
finally:
db.close()
except Exception as exc: # noqa: BLE001
logger.warning(
"Could not load IMAP ingestion profile %d (%s: %s) — using global default",
profile_id,
type(exc).__name__,
exc,
)
# Fall back to global setting
if settings.imap_attachment_filter == "all":
return ALL_CATEGORIES
return DEFAULT_CATEGORIES
LOCK_KEY = "imap_lock" # Unique key for locking LOCK_KEY = "imap_lock" # Unique key for locking
LOCK_EXPIRE = 300 # Lock expires in 5 minutes LOCK_EXPIRE = 300 # Lock expires in 5 minutes
@@ -180,6 +216,7 @@ def _pull_user_imap_accounts() -> None:
use_ssl=acct.use_ssl, use_ssl=acct.use_ssl,
delete_after_process=acct.delete_after_process, delete_after_process=acct.delete_after_process,
owner_id=acct.owner_id, owner_id=acct.owner_id,
allowed_categories=_resolve_categories_for_profile(acct.profile_id),
) )
# Record successful poll # Record successful poll
acct.last_checked_at = datetime.now(timezone.utc) acct.last_checked_at = datetime.now(timezone.utc)
@@ -250,6 +287,9 @@ def _pull_user_integration_imap() -> None:
use_ssl = cfg.get("use_ssl", True) use_ssl = cfg.get("use_ssl", True)
delete_after = cfg.get("delete_after_process", False) delete_after = cfg.get("delete_after_process", False)
gmail_labels = cfg.get("gmail_apply_labels", True) gmail_labels = cfg.get("gmail_apply_labels", True)
# Integrations can store a profile_id in config; fall back to global default
profile_id = cfg.get("profile_id")
allowed_categories = _resolve_categories_for_profile(profile_id)
if not (host and username and password): if not (host and username and password):
logger.warning( logger.warning(
@@ -269,6 +309,7 @@ def _pull_user_integration_imap() -> None:
delete_after_process=delete_after, delete_after_process=delete_after,
owner_id=integ.owner_id, owner_id=integ.owner_id,
gmail_apply_labels=gmail_labels, gmail_apply_labels=gmail_labels,
allowed_categories=allowed_categories,
) )
integ.last_used_at = datetime.now(timezone.utc) integ.last_used_at = datetime.now(timezone.utc)
integ.last_error = None integ.last_error = None
@@ -329,6 +370,7 @@ def pull_inbox(
delete_after_process, delete_after_process,
owner_id=None, owner_id=None,
gmail_apply_labels=True, gmail_apply_labels=True,
allowed_categories=None,
): ):
""" """
Connects to the IMAP inbox, fetches new unread emails from the last 3 days, Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
@@ -345,8 +387,22 @@ def pull_inbox(
attributed to this user via ``process_document`` / ``convert_to_pdf``. attributed to this user via ``process_document`` / ``convert_to_pdf``.
gmail_apply_labels: Whether to apply Gmail-specific labels and stars to gmail_apply_labels: Whether to apply Gmail-specific labels and stars to
processed emails. Only relevant for Gmail hosts. Defaults to True. processed emails. Only relevant for Gmail hosts. Defaults to True.
allowed_categories: List of file-type category keys to ingest (e.g.
``["pdf", "office", "images"]``). ``None`` falls back to the
global ``settings.imap_attachment_filter`` mapping.
""" """
logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl) if allowed_categories is None:
allowed_categories = _resolve_categories_for_profile(None)
effective_mime_types, effective_extensions = get_allowed_types_for_categories(allowed_categories)
logger.info(
"Connecting to %s at %s:%s (SSL=%s) — categories: %s",
mailbox_key,
host,
port,
use_ssl,
allowed_categories,
)
processed_emails = load_processed_emails() processed_emails = load_processed_emails()
try: try:
@@ -405,9 +461,13 @@ def pull_inbox(
logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key) logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
continue continue
# Process attachments (and convert non-PDF files). # Process attachments using the resolved mime types / extensions.
# We call the function without assigning its return value since it is not used. fetch_attachments_and_enqueue(
fetch_attachments_and_enqueue(email_message, owner_id=owner_id) email_message,
owner_id=owner_id,
effective_mime_types=effective_mime_types,
effective_extensions=effective_extensions,
)
if settings.imap_readonly_mode: if settings.imap_readonly_mode:
logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key) logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
@@ -436,27 +496,23 @@ def pull_inbox(
logger.exception("Error pulling mailbox %s: %s", mailbox_key, e) logger.exception("Error pulling mailbox %s: %s", mailbox_key, e)
def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None): def fetch_attachments_and_enqueue(
email_message,
owner_id: str | None = None,
effective_mime_types: frozenset[str] | None = None,
effective_extensions: frozenset[str] | None = None,
):
""" """
Extracts attachments from the email and processes only allowed file types. Extracts attachments from the email and processes only allowed file types.
Files are accepted if either: The caller is responsible for computing ``effective_mime_types`` and
1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR ``effective_extensions`` from the relevant :class:`ImapIngestionProfile` (or
2. They have a '.pdf' file extension (regardless of MIME type) the global default) via :func:`app.utils.allowed_types.get_allowed_types_for_categories`
before calling this function. ``pull_inbox`` does this automatically.
Allowed file types include: If either set is ``None`` the function falls back to the default category list
- PDF: application/pdf or *.pdf extension so the function still works correctly when called directly in tests or from
- Microsoft Office files: other contexts.
- Word: application/msword,
application/vnd.openxmlformats-officedocument.wordprocessingml.document
- Excel: application/vnd.ms-excel,
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
- PowerPoint: application/vnd.ms-powerpoint,
application/vnd.openxmlformats-officedocument.presentationml.presentation
- Other meaningful attachments:
- Plain text: text/plain
- CSV: text/csv
- Rich Text Format: application/rtf, text/rtf
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload; If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
any other allowed file is enqueued for conversion to PDF. any other allowed file is enqueued for conversion to PDF.
@@ -465,9 +521,14 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
email_message: The parsed email message to extract attachments from. email_message: The parsed email message to extract attachments from.
owner_id: Optional user identifier forwarded to ``process_document`` / owner_id: Optional user identifier forwarded to ``process_document`` /
``convert_to_pdf`` for multi-tenant attribution. ``convert_to_pdf`` for multi-tenant attribution.
effective_mime_types: Pre-computed frozenset of allowed MIME type strings.
effective_extensions: Pre-computed frozenset of allowed file extension strings.
Returns True if at least one allowed attachment was processed. Returns True if at least one allowed attachment was processed.
""" """
if effective_mime_types is None or effective_extensions is None:
effective_mime_types, effective_extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
has_attachment = False has_attachment = False
for part in email_message.walk(): for part in email_message.walk():
if part.get_content_maintype() == "multipart": if part.get_content_maintype() == "multipart":
@@ -482,9 +543,15 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
mime_type = part.get_content_type() mime_type = part.get_content_type()
file_ext = os.path.splitext(filename)[1].lower() file_ext = os.path.splitext(filename)[1].lower()
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension # Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension: if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension:
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type) logger.info(
"Skipping attachment %s (MIME: %s, ext: %s) — not in effective allowed set",
filename,
mime_type,
file_ext,
)
continue continue
file_path = os.path.join(settings.workdir, filename) file_path = os.path.join(settings.workdir, filename)
@@ -495,7 +562,7 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
if mime_type == "application/pdf" or is_pdf_by_extension: if mime_type == "application/pdf" or is_pdf_by_extension:
process_document.delay(file_path, owner_id=owner_id) process_document.delay(file_path, owner_id=owner_id)
logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type) logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type)
elif mime_type in ALLOWED_MIME_TYPES: elif mime_type in effective_mime_types:
# Other allowed files are sent for conversion # Other allowed files are sent for conversion
convert_to_pdf.delay(file_path, owner_id=owner_id) convert_to_pdf.delay(file_path, owner_id=owner_id)
logger.info("Enqueued file for conversion to PDF: %s", filename) logger.info("Enqueued file for conversion to PDF: %s", filename)
+154
View File
@@ -131,3 +131,157 @@ ALLOWED_EXTENSIONS: set[str] = {
".md", ".md",
".markdown", ".markdown",
} }
# ---------------------------------------------------------------------------
# Fine-grained file-type categories used by IMAP ingestion profiles.
# Each category groups related MIME types and extensions so that users can
# enable/disable a logical collection of formats (e.g. "images") rather than
# having to manage individual MIME strings.
# ---------------------------------------------------------------------------
FILE_TYPE_CATEGORIES: dict[str, dict] = {
"pdf": {
"label": "PDF",
"description": "PDF documents (.pdf)",
"mime_types": frozenset({"application/pdf"}),
"extensions": frozenset({".pdf"}),
},
"office": {
"label": "Microsoft Office",
"description": "Word, Excel and PowerPoint files (.doc, .docx, .xls, .xlsx, .ppt, .pptx, …)",
"mime_types": frozenset(
{
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
"application/vnd.ms-word.document.macroEnabled.12",
"application/vnd.ms-word.template.macroEnabled.12",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.openxmlformats-officedocument.spreadsheetml.template",
"application/vnd.ms-excel.sheet.macroEnabled.12",
"application/vnd.ms-excel.sheet.binary.macroEnabled.12",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.openxmlformats-officedocument.presentationml.template",
"application/vnd.openxmlformats-officedocument.presentationml.slideshow",
"application/vnd.ms-powerpoint.presentation.macroEnabled.12",
}
),
"extensions": frozenset(
{
".doc",
".docx",
".docm",
".dot",
".dotx",
".dotm",
".xls",
".xlsx",
".xlsm",
".xlsb",
".xlt",
".xltx",
".xlw",
".ppt",
".pptx",
".pptm",
".pps",
".ppsx",
".pot",
".potx",
}
),
},
"opendocument": {
"label": "OpenDocument (LibreOffice)",
"description": "LibreOffice / OpenOffice files (.odt, .ods, .odp, …)",
"mime_types": frozenset(
{
"application/vnd.oasis.opendocument.text",
"application/vnd.oasis.opendocument.spreadsheet",
"application/vnd.oasis.opendocument.presentation",
"application/vnd.oasis.opendocument.graphics",
"application/vnd.oasis.opendocument.formula",
}
),
"extensions": frozenset({".odt", ".ods", ".odp", ".odg", ".odf"}),
},
"text": {
"label": "Text & Data",
"description": "Plain text, CSV and RTF files (.txt, .csv, .rtf)",
"mime_types": frozenset(
{
"text/plain",
"text/csv",
"application/rtf",
"text/rtf",
}
),
"extensions": frozenset({".txt", ".csv", ".rtf"}),
},
"web": {
"label": "Web & Markup",
"description": "HTML and Markdown files (.html, .htm, .md, .markdown)",
"mime_types": frozenset(
{
"text/html",
"text/markdown",
"text/x-markdown",
}
),
"extensions": frozenset({".html", ".htm", ".md", ".markdown"}),
},
"images": {
"label": "Images",
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)",
"mime_types": frozenset(
{
"image/jpeg",
"image/jpg",
"image/png",
"image/gif",
"image/bmp",
"image/tiff",
"image/webp",
"image/svg+xml",
}
),
"extensions": frozenset(
{
".jpg",
".jpeg",
".png",
".gif",
".bmp",
".tiff",
".tif",
".webp",
".svg",
}
),
},
}
# Default categories for the "documents only" built-in profile (no images)
DEFAULT_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web"]
# All categories including images
ALL_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web", "images"]
def get_allowed_types_for_categories(
categories: list[str],
) -> tuple[frozenset[str], frozenset[str]]:
"""Return ``(mime_types, extensions)`` for the given category list.
Unknown category names are silently ignored so that future categories
don't break existing profiles.
"""
mime_types: set[str] = set()
extensions: set[str] = set()
for cat in categories:
info = FILE_TYPE_CATEGORIES.get(cat)
if info:
mime_types |= info["mime_types"]
extensions |= info["extensions"]
return frozenset(mime_types), frozenset(extensions)
+12
View File
@@ -1660,6 +1660,18 @@ SETTING_METADATA = {
"required": False, "required": False,
"restart_required": False, "restart_required": False,
}, },
"imap_attachment_filter": {
"category": "IMAP",
"description": (
"Controls which attachment types are ingested from IMAP emails. "
"Accepted values: 'documents_only' (PDFs and office files only, default) or 'all' (including images). "
"Per-user IMAP accounts can override this global default."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Monitoring - Uptime Kuma # Monitoring - Uptime Kuma
"uptime_kuma_url": { "uptime_kuma_url": {
"category": "Monitoring", "category": "Monitoring",
+43 -1
View File
@@ -1,11 +1,13 @@
"""User-facing view for the per-user IMAP ingestion dashboard.""" """User-facing view for the per-user IMAP ingestion dashboard."""
import json
import logging import logging
from fastapi import Request from fastapi import Request
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.models import UserImapAccount from app.models import ImapIngestionProfile, UserImapAccount
from app.utils.allowed_types import DEFAULT_CATEGORIES, FILE_TYPE_CATEGORIES
from app.utils.subscription import get_tier, get_user_tier_id from app.utils.subscription import get_tier, get_user_tier_id
from app.utils.user_scope import get_current_owner_id from app.utils.user_scope import get_current_owner_id
from app.views.base import APIRouter, Depends, get_db, require_login, templates from app.views.base import APIRouter, Depends, get_db, require_login, templates
@@ -25,6 +27,22 @@ def _get_max_mailboxes(tier: dict) -> int | None:
return max_mb return max_mb
def _serialize_profile(profile: ImapIngestionProfile) -> dict:
"""Serialize a profile for JSON embedding in the template."""
try:
categories = json.loads(profile.allowed_categories)
except (ValueError, TypeError):
categories = []
return {
"id": profile.id,
"name": profile.name,
"description": profile.description,
"owner_id": profile.owner_id,
"allowed_categories": categories,
"is_builtin": profile.is_builtin,
}
@router.get("/imap-accounts") @router.get("/imap-accounts")
@require_login @require_login
async def imap_accounts_page(request: Request, db: Session = Depends(get_db)): async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
@@ -49,11 +67,35 @@ async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
max_mailboxes = _get_max_mailboxes(tier) max_mailboxes = _get_max_mailboxes(tier)
can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes) can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes)
# Load ingestion profiles: system-global + user's own
profiles = (
db.query(ImapIngestionProfile)
.filter(
# SQLAlchemy requires `== None` for IS NULL comparison in ORM filters
(ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711
)
.order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
.all()
)
# Category definitions for the UI checkbox builder
categories = [
{
"key": key,
"label": info["label"],
"description": info["description"],
}
for key, info in FILE_TYPE_CATEGORIES.items()
]
return templates.TemplateResponse( return templates.TemplateResponse(
"imap_accounts.html", "imap_accounts.html",
{ {
"request": request, "request": request,
"accounts": accounts, "accounts": accounts,
"profiles": [_serialize_profile(p) for p in profiles],
"categories": categories,
"default_categories": DEFAULT_CATEGORIES,
"current_count": current_count, "current_count": current_count,
"max_mailboxes": max_mailboxes, "max_mailboxes": max_mailboxes,
"can_add": can_add, "can_add": can_add,
+36
View File
@@ -304,6 +304,42 @@ DocuElevate can automatically pull document attachments from IMAP mailboxes —
| `IMAP1_SSL` | Use SSL (`true`/`false`). | `true` | | `IMAP1_SSL` | Use SSL (`true`/`false`). | `true` |
| `IMAP1_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll for new mail. | `5` | | `IMAP1_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll for new mail. | `5` |
| `IMAP_READONLY_MODE` | When `true`, fetches and processes attachments but does **not** modify the mailbox (no starring, labeling, deleting, or flag changes). Use for pre-production instances sharing a mailbox with production. Default: `false`. | `false` | | `IMAP_READONLY_MODE` | When `true`, fetches and processes attachments but does **not** modify the mailbox (no starring, labeling, deleting, or flag changes). Use for pre-production instances sharing a mailbox with production. Default: `false`. | `false` |
| `IMAP_ATTACHMENT_FILTER` | System-wide fallback for which attachment types are ingested when no ingestion profile is assigned to a mailbox. `documents_only` (default) ingests PDFs and office files only — images are skipped. `all` ingests every supported file type including images. Individual IMAP accounts can override this using ingestion profiles. | `documents_only` |
#### IMAP Ingestion Profiles
For fine-grained control, DocuElevate supports **Ingestion Profiles** — named configurations that let you choose exactly which file-type categories to accept from each mailbox.
Each profile contains a list of enabled **categories**:
| Category | Description |
|----------|-------------|
| `pdf` | PDF documents (`.pdf`) |
| `office` | Microsoft Office files (Word, Excel, PowerPoint — `.docx`, `.xlsx`, `.pptx`, …) |
| `opendocument` | LibreOffice/OpenOffice files (`.odt`, `.ods`, `.odp`, …) |
| `text` | Plain text, CSV and RTF files (`.txt`, `.csv`, `.rtf`) |
| `web` | HTML and Markdown files (`.html`, `.htm`, `.md`, `.markdown`) |
| `images` | Image files (`.jpg`, `.png`, `.gif`, `.bmp`, `.tiff`, `.webp`, `.svg`) |
Two built-in system profiles are seeded automatically:
| Profile | Categories |
|---------|------------|
| **Documents Only** | pdf, office, opendocument, text, web (no images) |
| **All Files** | All categories, including images |
Users can create their own custom profiles via the **Email Ingestion** dashboard (`/imap-accounts`) by clicking the **Manage profiles** link or the **+** button next to the profile dropdown. Custom profiles are private to the creating user and can be freely edited or deleted.
**API endpoints for ingestion profiles:**
| Method | Endpoint | Description |
|--------|----------|-------------|
| `GET` | `/api/imap-profiles/` | List all visible profiles (system + user's own) |
| `POST` | `/api/imap-profiles/` | Create a new profile |
| `GET` | `/api/imap-profiles/categories` | List available file-type categories |
| `GET` | `/api/imap-profiles/{id}` | Get a single profile |
| `PUT` | `/api/imap-profiles/{id}` | Update a profile (not built-in) |
| `DELETE` | `/api/imap-profiles/{id}` | Delete a profile (not built-in) |
#### Per-User IMAP Integrations #### Per-User IMAP Integrations
+48
View File
@@ -63,6 +63,54 @@ DocuElevate will process the following attachment types from emails:
| TIFF | `.tif`, `.tiff` | Common format from older scanners/fax | | TIFF | `.tif`, `.tiff` | Common format from older scanners/fax |
| Multi-page TIFF | `.tif` | Full multi-page support | | Multi-page TIFF | `.tif` | Full multi-page support |
### Controlling Which Attachment Types Are Ingested
By default, DocuElevate only ingests **document** attachments (PDFs, Word, Excel, PowerPoint, OpenDocument, RTF, TXT, CSV, HTML, Markdown). Images are **not** ingested by default — this prevents cluttering your document archive with inline images or unrelated photo attachments.
#### Global Default (Admin Setting)
Set the `IMAP_ATTACHMENT_FILTER` environment variable to control the system-wide fallback when no ingestion profile is assigned to a mailbox:
| Value | Behaviour |
|-------|-----------|
| `documents_only` | **(Default)** Only PDFs and office/document files. Images are skipped. |
| `all` | All supported file types, including images. |
```env
IMAP_ATTACHMENT_FILTER=documents_only
```
#### Ingestion Profiles (Fine-Grained Per-Mailbox Control)
For precise control, you can create **Ingestion Profiles** that let you pick exactly which file-type categories to accept from each mailbox. This is more powerful than the binary global toggle and works independently per mailbox.
**Available categories:**
| Category | File types included |
|----------|---------------------|
| PDF | `.pdf` |
| Microsoft Office | `.doc`, `.docx`, `.xls`, `.xlsx`, `.ppt`, `.pptx`, and macro-enabled variants |
| OpenDocument | `.odt`, `.ods`, `.odp`, `.odg`, `.odf` (LibreOffice / OpenOffice) |
| Text & Data | `.txt`, `.csv`, `.rtf` |
| Web & Markup | `.html`, `.htm`, `.md`, `.markdown` |
| Images | `.jpg`, `.png`, `.gif`, `.bmp`, `.tiff`, `.webp`, `.svg` |
**Managing profiles:**
1. Go to **Email Ingestion** (`/imap-accounts`)
2. Click **Manage profiles** (or the **+** icon next to the profile dropdown)
3. Create a new profile, give it a name, and tick the categories you want
4. When adding or editing a mailbox, select your profile from the dropdown
Two built-in profiles are always available and cannot be deleted:
- **Documents Only** — PDF, Office, OpenDocument, Text, Web (no images)
- **All Files** — all categories including images
Users can also create unlimited **custom profiles** to mix and match exactly the categories they need per mailbox (e.g. a scanner mailbox that only accepts PDFs, or a finance mailbox that accepts Office and CSV but not images).
Custom profiles are created via the UI or the `/api/imap-profiles/` API.
--- ---
## Setting Up Your Scanner/Device ## Setting Up Your Scanner/Device
+477 -4
View File
@@ -172,6 +172,11 @@
<i class="fas fa-trash-alt mr-1" aria-hidden="true"></i>Delete after process <i class="fas fa-trash-alt mr-1" aria-hidden="true"></i>Delete after process
</span> </span>
</template> </template>
<template x-if="acct.profile_id">
<span class="inline-flex items-center px-2 py-0.5 rounded bg-blue-50 text-blue-700">
<i class="fas fa-filter mr-1" aria-hidden="true"></i><span x-text="profileName(acct.profile_id)"></span>
</span>
</template>
<template x-if="acct.last_checked_at"> <template x-if="acct.last_checked_at">
<span class="text-gray-400" :title="acct.last_checked_at"> <span class="text-gray-400" :title="acct.last_checked_at">
<i class="fas fa-clock mr-1" aria-hidden="true"></i>Last polled: <span x-text="formatDate(acct.last_checked_at)"></span> <i class="fas fa-clock mr-1" aria-hidden="true"></i>Last polled: <span x-text="formatDate(acct.last_checked_at)"></span>
@@ -386,6 +391,49 @@
</label> </label>
</div> </div>
<!-- Attachment filter -->
<div>
<label for="acct-profile" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
Ingestion Profile
</label>
<div class="flex gap-2 items-center">
<select
id="acct-profile"
x-model.number="form.profile_id"
class="flex-1 px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
aria-describedby="acct-profile-hint"
>
<option :value="null">Use global default</option>
<template x-for="profile in profiles" :key="profile.id">
<option :value="profile.id" x-text="profile.name + (profile.is_builtin ? '' : ' (custom)')"></option>
</template>
</select>
<button
type="button"
@click="openProfileModal(null)"
class="inline-flex items-center px-3 py-2 text-sm font-medium rounded border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-blue-500"
style="min-height:40px;"
title="Create new profile"
aria-label="Create new ingestion profile"
>
<i class="fas fa-plus" aria-hidden="true"></i>
</button>
</div>
<p id="acct-profile-hint" class="mt-1 text-xs text-gray-400 dark:text-gray-500">
Controls which attachment types are ingested. Leave blank to use the global default (documents only).
<button type="button" @click="showProfilesSection = !showProfilesSection" class="underline hover:text-blue-600 focus:outline-none">
Manage profiles
</button>
</p>
<!-- Inline selected profile summary -->
<template x-if="form.profile_id">
<div class="mt-2 text-xs bg-blue-50 dark:bg-blue-900/20 rounded px-2 py-1.5 text-blue-800 dark:text-blue-200">
<strong x-text="profileName(form.profile_id)"></strong>:
<span x-text="profileCategorySummary(form.profile_id)"></span>
</div>
</template>
</div>
<!-- Test connection result --> <!-- Test connection result -->
<template x-if="testResult"> <template x-if="testResult">
<div <div
@@ -494,10 +542,277 @@
</div> </div>
<!-- ── Ingestion Profiles panel ─────────────────────────────────────────────── -->
<div
x-show="showProfilesSection"
x-transition
class="fixed inset-0 z-50 flex items-center justify-center bg-black/50 px-4"
role="dialog"
aria-modal="true"
aria-labelledby="profiles-panel-title"
@keydown.escape.window="showProfilesSection = false"
style="display:none;"
>
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-2xl overflow-y-auto max-h-[90vh]" @click.stop>
<div class="px-6 pt-6 pb-2 flex items-center justify-between">
<h2 id="profiles-panel-title" class="text-lg font-semibold text-gray-900 dark:text-white flex items-center gap-2">
<i class="fas fa-filter text-blue-500" aria-hidden="true"></i>
Ingestion Profiles
</h2>
<button
type="button"
@click="showProfilesSection = false"
class="text-gray-400 hover:text-gray-600 dark:hover:text-gray-200 focus:outline-none focus:ring-2 focus:ring-blue-500 rounded"
aria-label="Close profiles panel"
>
<i class="fas fa-times text-xl" aria-hidden="true"></i>
</button>
</div>
<div class="px-6 pb-6 pt-2">
<p class="text-sm text-gray-500 dark:text-gray-400 mb-4">
Profiles define which attachment types are ingested from emails. Assign a profile to each IMAP account for fine-grained control.
</p>
<!-- Profile list -->
<div class="space-y-3 mb-4">
<template x-for="profile in profiles" :key="profile.id">
<div class="border border-gray-100 dark:border-gray-700 rounded-lg p-4 flex items-start gap-3">
<div class="flex-1 min-w-0">
<div class="flex items-center gap-2 flex-wrap">
<span class="font-medium text-sm text-gray-900 dark:text-white" x-text="profile.name"></span>
<template x-if="profile.is_builtin">
<span class="text-xs px-1.5 py-0.5 rounded bg-gray-100 dark:bg-gray-700 text-gray-500">built-in</span>
</template>
</div>
<p class="text-xs text-gray-500 dark:text-gray-400 mt-0.5" x-text="profile.description || ''"></p>
<!-- Category badges -->
<div class="flex flex-wrap gap-1 mt-2">
<template x-for="cat in profile.allowed_categories" :key="cat">
<span class="inline-flex items-center px-2 py-0.5 rounded text-xs font-medium bg-blue-50 dark:bg-blue-900/30 text-blue-700 dark:text-blue-300">
<span x-text="categoryLabel(cat)"></span>
</span>
</template>
</div>
</div>
<div class="flex gap-1 shrink-0">
<button
type="button"
@click="openProfileModal(profile)"
:disabled="profile.is_builtin"
:class="profile.is_builtin ? 'opacity-40 cursor-not-allowed' : ''"
class="p-1.5 text-gray-400 hover:text-blue-600 focus:outline-none focus:ring-2 focus:ring-blue-500 rounded"
:aria-label="`Edit profile ${profile.name}`"
:title="profile.is_builtin ? 'Built-in profiles cannot be edited' : 'Edit profile'"
>
<i class="fas fa-edit text-sm" aria-hidden="true"></i>
</button>
<button
type="button"
@click="confirmDeleteProfile(profile)"
:disabled="profile.is_builtin"
:class="profile.is_builtin ? 'opacity-40 cursor-not-allowed' : ''"
class="p-1.5 text-gray-400 hover:text-red-600 focus:outline-none focus:ring-2 focus:ring-red-500 rounded"
:aria-label="`Delete profile ${profile.name}`"
:title="profile.is_builtin ? 'Built-in profiles cannot be deleted' : 'Delete profile'"
>
<i class="fas fa-trash-alt text-sm" aria-hidden="true"></i>
</button>
</div>
</div>
</template>
</div>
<button
type="button"
@click="openProfileModal(null)"
class="inline-flex items-center px-4 py-2 text-sm font-medium rounded bg-blue-600 hover:bg-blue-700 text-white focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
style="min-height:40px;"
>
<i class="fas fa-plus mr-2" aria-hidden="true"></i> New Profile
</button>
</div>
</div>
</div>
<!-- ── Profile create / edit modal ────────────────────────────────────────────── -->
<div
x-show="profileModalOpen"
x-transition:enter="ease-out duration-200"
x-transition:enter-start="opacity-0"
x-transition:enter-end="opacity-100"
x-transition:leave="ease-in duration-150"
x-transition:leave-start="opacity-100"
x-transition:leave-end="opacity-0"
class="fixed inset-0 z-[60] flex items-center justify-center bg-black/50 px-4"
role="dialog"
aria-modal="true"
:aria-labelledby="editingProfile ? 'profile-modal-title-edit' : 'profile-modal-title-create'"
@keydown.escape.window="closeProfileModal()"
style="display:none;"
>
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-lg overflow-y-auto max-h-[90vh]" @click.stop>
<div class="px-6 pt-6 pb-2 flex items-center justify-between">
<h2
:id="editingProfile ? 'profile-modal-title-edit' : 'profile-modal-title-create'"
class="text-lg font-semibold text-gray-900 dark:text-white"
x-text="editingProfile ? 'Edit Profile' : 'New Ingestion Profile'"
></h2>
<button
type="button"
@click="closeProfileModal()"
class="text-gray-400 hover:text-gray-600 dark:hover:text-gray-200 focus:outline-none focus:ring-2 focus:ring-blue-500 rounded"
aria-label="Close modal"
>
<i class="fas fa-times text-xl" aria-hidden="true"></i>
</button>
</div>
<form @submit.prevent="saveProfile()" class="px-6 pb-6 pt-4 space-y-4">
<!-- Profile name -->
<div>
<label for="profile-name" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
Profile Name <span class="text-red-500" aria-hidden="true">*</span>
</label>
<input
id="profile-name"
type="text"
x-model="profileForm.name"
placeholder="e.g. Scanner Inbox, Finance Documents"
required
maxlength="255"
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
aria-required="true"
/>
</div>
<!-- Description -->
<div>
<label for="profile-description" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
Description
</label>
<input
id="profile-description"
type="text"
x-model="profileForm.description"
placeholder="Optional description"
maxlength="500"
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
/>
</div>
<!-- Category checkboxes -->
<div>
<fieldset>
<legend class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-2">
File Type Categories <span class="text-red-500" aria-hidden="true">*</span>
</legend>
<div class="space-y-2">
<template x-for="cat in categories" :key="cat.key">
<label class="flex items-start gap-3 cursor-pointer group">
<input
type="checkbox"
:value="cat.key"
:checked="profileForm.allowed_categories.includes(cat.key)"
@change="toggleCategory(cat.key)"
class="mt-0.5 h-4 w-4 rounded border-gray-300 text-blue-600 focus:ring-blue-500"
/>
<span class="flex-1">
<span class="text-sm font-medium text-gray-800 dark:text-gray-100" x-text="cat.label"></span>
<span class="block text-xs text-gray-500 dark:text-gray-400" x-text="cat.description"></span>
</span>
</label>
</template>
</div>
<p x-show="profileForm.allowed_categories.length === 0" class="mt-2 text-xs text-red-600" role="alert">
Select at least one category.
</p>
</fieldset>
</div>
<!-- Form error -->
<template x-if="profileFormError">
<p class="text-sm text-red-600" role="alert" x-text="profileFormError"></p>
</template>
<!-- Actions -->
<div class="flex justify-end gap-2 pt-2 border-t border-gray-100 dark:border-gray-700">
<button
type="button"
@click="closeProfileModal()"
class="px-4 py-2 text-sm font-medium rounded border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-blue-500"
style="min-height:40px;"
>
Cancel
</button>
<button
type="submit"
:disabled="savingProfile || profileForm.allowed_categories.length === 0"
class="inline-flex items-center px-4 py-2 text-sm font-medium rounded bg-blue-600 hover:bg-blue-700 text-white focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50"
style="min-height:40px;"
>
<template x-if="savingProfile">
<i class="fas fa-spinner fa-spin mr-2" aria-hidden="true"></i>
</template>
<span x-text="editingProfile ? 'Save Changes' : 'Create Profile'"></span>
</button>
</div>
</form>
</div>
</div>
<!-- ── Delete profile confirmation modal ────────────────────────────────────── -->
<div
x-show="deleteProfileModalOpen"
x-transition
class="fixed inset-0 z-[60] flex items-center justify-center bg-black/50 px-4"
role="dialog"
aria-modal="true"
aria-labelledby="delete-profile-modal-title"
@keydown.escape.window="deleteProfileModalOpen = false"
style="display:none;"
>
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-sm p-6" @click.stop>
<h2 id="delete-profile-modal-title" class="text-lg font-semibold text-gray-900 dark:text-white mb-2">
Delete Profile?
</h2>
<p class="text-sm text-gray-600 dark:text-gray-300 mb-4">
Are you sure you want to delete the profile
<strong x-text="profileToDelete ? profileToDelete.name : ''"></strong>?
IMAP accounts using this profile will fall back to the global default.
</p>
<div class="flex justify-end gap-2">
<button
type="button"
@click="deleteProfileModalOpen = false; profileToDelete = null"
class="px-4 py-2 text-sm font-medium rounded border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-gray-500"
style="min-height:40px;"
>
Cancel
</button>
<button
type="button"
@click="deleteProfile()"
:disabled="deletingProfile"
class="inline-flex items-center px-4 py-2 text-sm font-medium rounded bg-red-600 hover:bg-red-700 text-white focus:outline-none focus:ring-2 focus:ring-red-500 disabled:opacity-50"
style="min-height:40px;"
>
<template x-if="deletingProfile">
<i class="fas fa-spinner fa-spin mr-2" aria-hidden="true"></i>
</template>
Delete
</button>
</div>
</div>
</div>
<script> <script>
function imapAccountsApp() { function imapAccountsApp() {
return { return {
accounts: {{ accounts | tojson }}, accounts: {{ accounts | tojson }},
profiles: {{ profiles | tojson }},
categories: {{ categories | tojson }},
defaultCategories: {{ default_categories | tojson }},
quota: { quota: {
current_count: {{ current_count }}, current_count: {{ current_count }},
max_mailboxes: {{ max_mailboxes | tojson }}, max_mailboxes: {{ max_mailboxes | tojson }},
@@ -508,17 +823,17 @@ function imapAccountsApp() {
loading: false, loading: false,
alert: { show: false, type: '', title: '', message: '' }, alert: { show: false, type: '', title: '', message: '' },
// Modal state // Account modal state
modalOpen: false, modalOpen: false,
editingAccount: null, editingAccount: null,
form: { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true }, form: { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true, profile_id: null },
showPassword: false, showPassword: false,
saving: false, saving: false,
testing: false, testing: false,
testResult: null, testResult: null,
formError: null, formError: null,
// Delete state // Account delete state
deleteModalOpen: false, deleteModalOpen: false,
accountToDelete: null, accountToDelete: null,
deleting: false, deleting: false,
@@ -526,6 +841,17 @@ function imapAccountsApp() {
// Test (saved account) state // Test (saved account) state
testingId: null, testingId: null,
// Profiles panel & modal state
showProfilesSection: false,
profileModalOpen: false,
editingProfile: null,
profileForm: { name: '', description: '', allowed_categories: [] },
savingProfile: false,
profileFormError: null,
deleteProfileModalOpen: false,
profileToDelete: null,
deletingProfile: false,
get canAdd() { get canAdd() {
return this.quota.can_add; return this.quota.can_add;
}, },
@@ -549,9 +875,33 @@ function imapAccountsApp() {
setTimeout(() => { this.alert.show = false; }, 5000); setTimeout(() => { this.alert.show = false; }, 5000);
}, },
// ── Profile helpers ──────────────────────────────────────────────────────
profileById(id) {
return this.profiles.find(p => p.id === id) || null;
},
profileName(id) {
const p = this.profileById(id);
return p ? p.name : 'Unknown profile';
},
profileCategorySummary(id) {
const p = this.profileById(id);
if (!p) return '';
return p.allowed_categories.map(k => this.categoryLabel(k)).join(', ');
},
categoryLabel(key) {
const cat = this.categories.find(c => c.key === key);
return cat ? cat.label : key;
},
// ── Account modal ────────────────────────────────────────────────────────
openCreateModal() { openCreateModal() {
this.editingAccount = null; this.editingAccount = null;
this.form = { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true }; this.form = { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true, profile_id: null };
this.showPassword = false; this.showPassword = false;
this.testResult = null; this.testResult = null;
this.formError = null; this.formError = null;
@@ -569,6 +919,7 @@ function imapAccountsApp() {
use_ssl: acct.use_ssl, use_ssl: acct.use_ssl,
delete_after_process: acct.delete_after_process, delete_after_process: acct.delete_after_process,
is_active: acct.is_active, is_active: acct.is_active,
profile_id: acct.profile_id || null,
}; };
this.showPassword = false; this.showPassword = false;
this.testResult = null; this.testResult = null;
@@ -592,6 +943,8 @@ function imapAccountsApp() {
if (this.editingAccount && payload.password === '') { if (this.editingAccount && payload.password === '') {
delete payload.password; delete payload.password;
} }
// Normalise profile_id: empty string → null
if (payload.profile_id === '' || payload.profile_id === 0) payload.profile_id = null;
let resp; let resp;
if (this.editingAccount) { if (this.editingAccount) {
resp = await fetch(`/api/imap-accounts/${this.editingAccount.id}`, { resp = await fetch(`/api/imap-accounts/${this.editingAccount.id}`, {
@@ -719,6 +1072,126 @@ function imapAccountsApp() {
this.deleting = false; this.deleting = false;
} }
}, },
// ── Profile CRUD ─────────────────────────────────────────────────────────
openProfileModal(profile) {
this.editingProfile = profile;
if (profile) {
this.profileForm = {
name: profile.name,
description: profile.description || '',
allowed_categories: [...profile.allowed_categories],
};
} else {
this.profileForm = { name: '', description: '', allowed_categories: [...this.defaultCategories] };
}
this.profileFormError = null;
this.profileModalOpen = true;
},
closeProfileModal() {
this.profileModalOpen = false;
this.editingProfile = null;
this.profileFormError = null;
},
toggleCategory(key) {
const idx = this.profileForm.allowed_categories.indexOf(key);
if (idx === -1) {
this.profileForm.allowed_categories.push(key);
} else {
this.profileForm.allowed_categories.splice(idx, 1);
}
},
async saveProfile() {
if (this.profileForm.allowed_categories.length === 0) {
this.profileFormError = 'Select at least one category.';
return;
}
this.profileFormError = null;
this.savingProfile = true;
try {
const payload = {
name: this.profileForm.name,
description: this.profileForm.description || null,
allowed_categories: this.profileForm.allowed_categories,
};
let resp;
if (this.editingProfile) {
resp = await fetch(`/api/imap-profiles/${this.editingProfile.id}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': getCsrfToken() },
body: JSON.stringify(payload),
});
} else {
resp = await fetch('/api/imap-profiles/', {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': getCsrfToken() },
body: JSON.stringify(payload),
});
}
const data = await resp.json();
if (!resp.ok) {
this.profileFormError = data.detail || 'Failed to save profile.';
return;
}
// Flatten response into the same shape used locally
const normalized = {
id: data.id,
name: data.name,
description: data.description,
owner_id: data.owner_id,
allowed_categories: data.allowed_categories,
is_builtin: data.is_builtin,
};
if (this.editingProfile) {
const idx = this.profiles.findIndex(p => p.id === normalized.id);
if (idx !== -1) this.profiles.splice(idx, 1, normalized);
} else {
this.profiles.push(normalized);
}
this.closeProfileModal();
this.showAlert('success', 'Saved', this.editingProfile ? 'Profile updated.' : 'Profile created.');
} catch (err) {
this.profileFormError = `Network error: ${err.message || 'Unknown error'}. Please try again.`;
} finally {
this.savingProfile = false;
}
},
confirmDeleteProfile(profile) {
this.profileToDelete = profile;
this.deleteProfileModalOpen = true;
},
async deleteProfile() {
if (!this.profileToDelete) return;
this.deletingProfile = true;
try {
const resp = await fetch(`/api/imap-profiles/${this.profileToDelete.id}`, {
method: 'DELETE',
headers: { 'X-CSRF-Token': getCsrfToken() },
});
if (resp.ok || resp.status === 204) {
const deletedId = this.profileToDelete.id;
this.profiles = this.profiles.filter(p => p.id !== deletedId);
// Clear profile_id on accounts that referenced this profile
this.accounts.forEach(a => { if (a.profile_id === deletedId) a.profile_id = null; });
this.showAlert('success', 'Deleted', `Profile "${this.profileToDelete.name}" removed.`);
this.deleteProfileModalOpen = false;
this.profileToDelete = null;
} else {
const data = await resp.json();
this.showAlert('error', 'Delete Failed', data.detail || 'Could not delete profile.');
}
} catch (err) {
this.showAlert('error', 'Delete Failed', `Network error: ${err.message || 'Unknown error'}`);
} finally {
this.deletingProfile = false;
}
},
}; };
} }
+1
View File
@@ -28,6 +28,7 @@ from app.models import ( # noqa: F401
DocumentMetadata, DocumentMetadata,
FileProcessingStep, FileProcessingStep,
FileRecord, FileRecord,
ImapIngestionProfile,
InAppNotification, InAppNotification,
LocalUser, LocalUser,
MobileDevice, MobileDevice,
@@ -0,0 +1,28 @@
"""Add attachment_filter column to user_imap_accounts table.
Revision ID: 032_add_imap_attachment_filter
Revises: 031_add_compliance_templates
Create Date: 2026-03-12
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "032_add_imap_attachment_filter"
down_revision: Union[str, None] = "031_add_compliance_templates"
depends_on: Union[str, None] = None
def upgrade() -> None:
"""Add attachment_filter column to user_imap_accounts."""
op.add_column(
"user_imap_accounts",
sa.Column("attachment_filter", sa.String(50), nullable=True),
)
def downgrade() -> None:
"""Remove attachment_filter column from user_imap_accounts."""
op.drop_column("user_imap_accounts", "attachment_filter")
@@ -0,0 +1,146 @@
"""Add imap_ingestion_profiles table and migrate user_imap_accounts.
Creates the ``imap_ingestion_profiles`` table, seeds the two built-in profiles
("Documents Only" and "All Files"), and replaces the ``attachment_filter``
string column on ``user_imap_accounts`` with a ``profile_id`` FK that references
the new table.
Revision ID: 033_add_imap_ingestion_profiles
Revises: 032_add_imap_attachment_filter
Create Date: 2026-03-12
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "033_add_imap_ingestion_profiles"
down_revision: Union[str, None] = "032_add_imap_attachment_filter"
depends_on: Union[str, None] = None
# Fixed IDs for the built-in profiles so that the FK migration is reproducible.
_BUILTIN_DOCUMENTS_ONLY_ID = 1
_BUILTIN_ALL_FILES_ID = 2
def upgrade() -> None:
"""Create profiles table, seed built-ins, migrate accounts column."""
# 1 — Create the imap_ingestion_profiles table
op.create_table(
"imap_ingestion_profiles",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("name", sa.String(255), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("owner_id", sa.String(), nullable=True),
sa.Column(
"allowed_categories",
sa.Text(),
nullable=False,
server_default='["pdf","office","opendocument","text","web"]',
),
sa.Column("is_builtin", sa.Boolean(), nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_imap_ingestion_profiles_id", "imap_ingestion_profiles", ["id"])
op.create_index("ix_imap_ingestion_profiles_owner_id", "imap_ingestion_profiles", ["owner_id"])
# 2 — Seed the two built-in profiles
bind = op.get_bind()
bind.execute(
sa.text(
"INSERT INTO imap_ingestion_profiles "
"(id, name, description, owner_id, allowed_categories, is_builtin) "
"VALUES (:id, :name, :desc, NULL, :cats, 1)"
),
[
{
"id": _BUILTIN_DOCUMENTS_ONLY_ID,
"name": "Documents Only",
"desc": (
"Ingest PDFs, Microsoft Office files, OpenDocument files, "
"plain text, CSV, RTF, HTML and Markdown. Images are excluded."
),
"cats": '["pdf","office","opendocument","text","web"]',
},
{
"id": _BUILTIN_ALL_FILES_ID,
"name": "All Files",
"desc": "Ingest all supported file types, including images.",
"cats": '["pdf","office","opendocument","text","web","images"]',
},
],
)
# 3 — Add profile_id column to user_imap_accounts
op.add_column(
"user_imap_accounts",
sa.Column("profile_id", sa.Integer(), nullable=True),
)
# 4 — Migrate existing attachment_filter values to profile_id
bind.execute(
sa.text(
"UPDATE user_imap_accounts SET profile_id = :pid "
"WHERE attachment_filter = 'all'"
),
{"pid": _BUILTIN_ALL_FILES_ID},
)
bind.execute(
sa.text(
"UPDATE user_imap_accounts SET profile_id = :pid "
"WHERE attachment_filter = 'documents_only'"
),
{"pid": _BUILTIN_DOCUMENTS_ONLY_ID},
)
# Rows with NULL attachment_filter keep profile_id = NULL (use global default)
# 5 — Add FK constraint (skip for SQLite which does not enforce FKs at DDL time)
# We use batch_alter_table so this works across SQLite and PostgreSQL
with op.batch_alter_table("user_imap_accounts") as batch_op:
batch_op.create_foreign_key(
"fk_user_imap_accounts_profile_id",
"imap_ingestion_profiles",
["profile_id"],
["id"],
)
# 6 — Drop the now-redundant attachment_filter column
with op.batch_alter_table("user_imap_accounts") as batch_op:
batch_op.drop_column("attachment_filter")
def downgrade() -> None:
"""Reverse the migration: restore attachment_filter, drop profiles table."""
# 1 — Re-add attachment_filter column
with op.batch_alter_table("user_imap_accounts") as batch_op:
batch_op.add_column(sa.Column("attachment_filter", sa.String(50), nullable=True))
# 2 — Restore string values from profile_id
bind = op.get_bind()
bind.execute(
sa.text(
"UPDATE user_imap_accounts SET attachment_filter = 'all' "
"WHERE profile_id = :pid"
),
{"pid": _BUILTIN_ALL_FILES_ID},
)
bind.execute(
sa.text(
"UPDATE user_imap_accounts SET attachment_filter = 'documents_only' "
"WHERE profile_id = :pid"
),
{"pid": _BUILTIN_DOCUMENTS_ONLY_ID},
)
# 3 — Drop the FK and profile_id column
with op.batch_alter_table("user_imap_accounts") as batch_op:
batch_op.drop_constraint("fk_user_imap_accounts_profile_id", type_="foreignkey")
batch_op.drop_column("profile_id")
# 4 — Drop the profiles table
op.drop_index("ix_imap_ingestion_profiles_owner_id", "imap_ingestion_profiles")
op.drop_index("ix_imap_ingestion_profiles_id", "imap_ingestion_profiles")
op.drop_table("imap_ingestion_profiles")
+159
View File
@@ -0,0 +1,159 @@
"""Tests for app/api/imap_profiles.py and app/utils/allowed_types category helpers."""
import pytest
from app.utils.allowed_types import (
ALL_CATEGORIES,
DEFAULT_CATEGORIES,
FILE_TYPE_CATEGORIES,
get_allowed_types_for_categories,
)
@pytest.mark.unit
class TestFileTypeCategories:
"""Tests for FILE_TYPE_CATEGORIES and get_allowed_types_for_categories."""
def test_all_category_keys_present(self):
"""Test that the six expected categories exist."""
assert set(FILE_TYPE_CATEGORIES.keys()) == {"pdf", "office", "opendocument", "text", "web", "images"}
def test_each_category_has_required_fields(self):
"""Test that every category entry has label, description, mime_types, extensions."""
for key, info in FILE_TYPE_CATEGORIES.items():
assert "label" in info, f"Category '{key}' missing 'label'"
assert "description" in info, f"Category '{key}' missing 'description'"
assert "mime_types" in info, f"Category '{key}' missing 'mime_types'"
assert "extensions" in info, f"Category '{key}' missing 'extensions'"
def test_pdf_category_contains_pdf_mime(self):
"""Test that the pdf category includes application/pdf."""
assert "application/pdf" in FILE_TYPE_CATEGORIES["pdf"]["mime_types"]
assert ".pdf" in FILE_TYPE_CATEGORIES["pdf"]["extensions"]
def test_images_category_contains_jpeg(self):
"""Test that the images category includes image/jpeg."""
assert "image/jpeg" in FILE_TYPE_CATEGORIES["images"]["mime_types"]
assert ".jpg" in FILE_TYPE_CATEGORIES["images"]["extensions"]
assert ".png" in FILE_TYPE_CATEGORIES["images"]["extensions"]
def test_get_allowed_types_for_default_categories(self):
"""Test that DEFAULT_CATEGORIES excludes image MIME types."""
mime_types, extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
assert "application/pdf" in mime_types
assert "application/msword" in mime_types
# images should NOT be in the default
assert "image/jpeg" not in mime_types
assert ".jpg" not in extensions
def test_get_allowed_types_for_all_categories(self):
"""Test that ALL_CATEGORIES includes image MIME types."""
mime_types, extensions = get_allowed_types_for_categories(ALL_CATEGORIES)
assert "image/jpeg" in mime_types
assert ".jpg" in extensions
assert "application/pdf" in mime_types
def test_get_allowed_types_returns_frozensets(self):
"""Test that returned sets are frozensets."""
mime_types, extensions = get_allowed_types_for_categories(["pdf"])
assert isinstance(mime_types, frozenset)
assert isinstance(extensions, frozenset)
def test_get_allowed_types_unknown_category_ignored(self):
"""Test that unknown category keys are silently ignored."""
mime_types, extensions = get_allowed_types_for_categories(["pdf", "nonexistent_category"])
assert "application/pdf" in mime_types # 'pdf' still works
# No crash for unknown key
def test_get_allowed_types_empty_list(self):
"""Test empty category list returns empty sets."""
mime_types, extensions = get_allowed_types_for_categories([])
assert mime_types == frozenset()
assert extensions == frozenset()
def test_default_categories_excludes_images(self):
"""Test that DEFAULT_CATEGORIES does not include 'images'."""
assert "images" not in DEFAULT_CATEGORIES
def test_all_categories_includes_images(self):
"""Test that ALL_CATEGORIES includes 'images'."""
assert "images" in ALL_CATEGORIES
def test_all_categories_is_superset_of_default(self):
"""Test that ALL_CATEGORIES contains all DEFAULT_CATEGORIES."""
for cat in DEFAULT_CATEGORIES:
assert cat in ALL_CATEGORIES
@pytest.mark.unit
class TestImapProfilesApiLogic:
"""Tests for ingestion profile validation helpers."""
def test_validate_categories_accepts_valid_keys(self):
"""Test that valid category keys pass validation."""
from app.api.imap_profiles import _validate_categories
result = _validate_categories(["pdf", "office", "images"])
assert set(result) == {"pdf", "office", "images"}
def test_validate_categories_rejects_unknown_key(self):
"""Test that unknown category keys raise 422."""
from fastapi import HTTPException
from app.api.imap_profiles import _validate_categories
with pytest.raises(HTTPException) as exc_info:
_validate_categories(["pdf", "nonexistent"])
assert exc_info.value.status_code == 422
assert "nonexistent" in str(exc_info.value.detail)
def test_validate_categories_deduplicates(self):
"""Test that duplicate category keys are de-duplicated while preserving order."""
from app.api.imap_profiles import _validate_categories
result = _validate_categories(["pdf", "pdf", "office", "pdf"])
assert result == ["pdf", "office"]
def test_to_response_serializes_profile(self, tmp_path):
"""Test _to_response produces expected dict shape."""
from unittest.mock import MagicMock
from app.api.imap_profiles import _to_response
profile = MagicMock()
profile.id = 42
profile.name = "My Profile"
profile.description = "Test description"
profile.owner_id = "user@example.com"
profile.allowed_categories = '["pdf","office"]'
profile.is_builtin = False
profile.created_at = None
profile.updated_at = None
result = _to_response(profile)
assert result["id"] == 42
assert result["name"] == "My Profile"
assert result["allowed_categories"] == ["pdf", "office"]
assert len(result["categories_detail"]) == 2
assert result["categories_detail"][0]["key"] == "pdf"
assert result["is_builtin"] is False
def test_to_response_handles_invalid_categories_json(self):
"""Test _to_response gracefully handles invalid JSON in allowed_categories."""
from unittest.mock import MagicMock
from app.api.imap_profiles import _to_response
profile = MagicMock()
profile.id = 1
profile.name = "Broken"
profile.description = None
profile.owner_id = None
profile.allowed_categories = "this is not valid json {"
profile.is_builtin = True
profile.created_at = None
profile.updated_at = None
result = _to_response(profile)
assert result["allowed_categories"] == []
+76
View File
@@ -24,6 +24,7 @@ from app.tasks.imap_tasks import (
release_lock, release_lock,
save_processed_emails, save_processed_emails,
) )
from app.utils.allowed_types import ALL_CATEGORIES, DEFAULT_CATEGORIES, get_allowed_types_for_categories
_TEST_CREDENTIAL = "pass" # noqa: S105 _TEST_CREDENTIAL = "pass" # noqa: S105
@@ -179,6 +180,81 @@ class TestFetchAttachmentsAndEnqueue:
assert result is True assert result is True
mock_convert.delay.assert_called_once() mock_convert.delay.assert_called_once()
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_skips_image_when_documents_only(self, mock_convert, mock_process):
"""Test that image attachments are skipped with the default (documents-only) categories."""
msg = EmailMessage()
msg["Subject"] = "Photo"
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
doc_mime, doc_ext = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
result = fetch_attachments_and_enqueue(msg, effective_mime_types=doc_mime, effective_extensions=doc_ext)
assert result is False
mock_process.delay.assert_not_called()
mock_convert.delay.assert_not_called()
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_processes_image_when_all_categories(self, mock_convert, mock_process, tmp_path):
"""Test that image attachments are processed when images category is included."""
msg = EmailMessage()
msg["Subject"] = "Photo"
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
all_mime, all_ext = get_allowed_types_for_categories(ALL_CATEGORIES)
with patch("app.tasks.imap_tasks.settings") as mock_settings:
mock_settings.workdir = str(tmp_path)
result = fetch_attachments_and_enqueue(msg, effective_mime_types=all_mime, effective_extensions=all_ext)
assert result is True
mock_convert.delay.assert_called_once()
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_skips_image_with_image_extension_documents_only(self, mock_convert, mock_process):
"""Test image files identified by extension are skipped with documents-only categories."""
msg = EmailMessage()
msg["Subject"] = "Screenshot"
msg.add_attachment(b"\x89PNG", maintype="application", subtype="octet-stream", filename="screenshot.png")
doc_mime, doc_ext = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
result = fetch_attachments_and_enqueue(msg, effective_mime_types=doc_mime, effective_extensions=doc_ext)
assert result is False
mock_process.delay.assert_not_called()
mock_convert.delay.assert_not_called()
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_processes_pdf_regardless_of_categories(self, mock_convert, mock_process, tmp_path):
"""Test that PDFs are always processed (pdf category always included in DEFAULT_CATEGORIES)."""
msg = EmailMessage()
msg["Subject"] = "Invoice"
msg.add_attachment(b"%PDF-1.4", maintype="application", subtype="pdf", filename="invoice.pdf")
doc_mime, doc_ext = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
with patch("app.tasks.imap_tasks.settings") as mock_settings:
mock_settings.workdir = str(tmp_path)
result = fetch_attachments_and_enqueue(msg, effective_mime_types=doc_mime, effective_extensions=doc_ext)
assert result is True
mock_process.delay.assert_called_once()
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_uses_default_categories_when_no_types_provided(self, mock_convert, mock_process):
"""Test that images are skipped when no effective_mime_types / extensions are passed (defaults to DEFAULT_CATEGORIES)."""
msg = EmailMessage()
msg["Subject"] = "Photo"
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
# No effective_mime_types passed → function defaults to DEFAULT_CATEGORIES (no images)
result = fetch_attachments_and_enqueue(msg)
assert result is False
mock_process.delay.assert_not_called()
mock_convert.delay.assert_not_called()
@pytest.mark.unit @pytest.mark.unit
class TestEmailAlreadyHasLabel: class TestEmailAlreadyHasLabel: