feat(imap): add ImapIngestionProfile model, API, migration and UI

Replaces the simple binary attachment_filter string with a full
ingestion profiles system:

- Add FILE_TYPE_CATEGORIES dict to allowed_types.py (6 categories:
  pdf, office, opendocument, text, web, images) + DEFAULT_CATEGORIES
  / ALL_CATEGORIES + get_allowed_types_for_categories() helper
- Add ImapIngestionProfile model (id, name, description, owner_id,
  allowed_categories JSON, is_builtin)
- Update UserImapAccount: replace attachment_filter string with
  profile_id FK to imap_ingestion_profiles
- Migration 033: creates profiles table, seeds 2 built-in profiles
  (Documents Only, All Files), migrates attachment_filter → profile_id
- New /api/imap-profiles/ CRUD endpoints (list, create, get, update,
  delete) with category validation
- Register imap_profiles router in app/api/__init__.py
- Update imap_tasks.py: replace attachment_filter string param with
  profile-based allowed_categories; add _resolve_categories_for_profile()
- Update imap_accounts.py API to use profile_id instead of attachment_filter
- Update imap_accounts view to pass profiles + categories to template
- Full UI overhaul: profiles panel + profile create/edit modal with
  category checkboxes; profile selector in account modal
- 17 new tests (141 total), all passing

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-12 01:14:19 +00:00
parent 554bb21d32
commit c9f554465d
12 changed files with 1387 additions and 135 deletions
+2
View File
@@ -21,6 +21,7 @@ from app.api.files import router as files_router
from app.api.google_drive import router as google_drive_router
from app.api.i18n import router as i18n_router
from app.api.imap_accounts import router as imap_accounts_router
from app.api.imap_profiles import router as imap_profiles_router
from app.api.integrations import router as integrations_router
from app.api.logs import router as logs_router
from app.api.mobile import router as mobile_router
@@ -83,6 +84,7 @@ router.include_router(onboarding_router)
router.include_router(billing_router)
router.include_router(pipelines_router)
router.include_router(imap_accounts_router)
router.include_router(imap_profiles_router)
router.include_router(integrations_router)
router.include_router(notifications_router)
router.include_router(scheduled_jobs_router)
+11 -17
View File
@@ -110,12 +110,10 @@ class ImapAccountCreate(BaseModel):
use_ssl: bool = Field(default=True, description="Use SSL/TLS connection")
delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing")
is_active: bool = Field(default=True, description="Whether to poll this mailbox")
attachment_filter: str | None = Field(
profile_id: int | None = Field(
default=None,
description=(
"Controls which attachment types to ingest. "
"'documents_only' PDFs and office files only (default when None). "
"'all' all supported types including images. "
"ID of the ImapIngestionProfile that controls which attachment types to ingest. "
"Null inherits the global imap_attachment_filter setting."
),
)
@@ -132,13 +130,11 @@ class ImapAccountUpdate(BaseModel):
use_ssl: bool | None = None
delete_after_process: bool | None = None
is_active: bool | None = None
attachment_filter: str | None = Field(
profile_id: int | None = Field(
default=None,
description=(
"Controls which attachment types to ingest. "
"'documents_only' PDFs and office files only. "
"'all' all supported types including images. "
"Null or empty string clears the override (inherits global setting)."
"ID of the ImapIngestionProfile to use. "
"Explicitly sending null clears the override (falls back to global setting)."
),
)
@@ -173,7 +169,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]:
"use_ssl": acct.use_ssl,
"delete_after_process": acct.delete_after_process,
"is_active": acct.is_active,
"attachment_filter": acct.attachment_filter,
"profile_id": acct.profile_id,
"last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None,
"last_error": acct.last_error,
"created_at": acct.created_at.isoformat() if acct.created_at else None,
@@ -241,7 +237,7 @@ def create_imap_account(
use_ssl=body.use_ssl,
delete_after_process=body.delete_after_process,
is_active=body.is_active,
attachment_filter=body.attachment_filter or None,
profile_id=body.profile_id,
)
try:
db.add(acct)
@@ -297,12 +293,10 @@ def update_imap_account(
acct.delete_after_process = body.delete_after_process
if body.is_active is not None:
acct.is_active = body.is_active
# attachment_filter uses a sentinel check: the field is always present in the
# model (defaulting to None in Pydantic) so we update it unconditionally when
# the caller sends any value (including explicit null to clear the override).
# An empty string is normalised to None to avoid storing a non-meaningful value.
if "attachment_filter" in body.model_fields_set:
acct.attachment_filter = body.attachment_filter or None
# profile_id: update whenever the field is explicitly present in the request payload
# (including sending null to clear the override).
if "profile_id" in body.model_fields_set:
acct.profile_id = body.profile_id
# Reset last_error so the next poll gives a fresh result
acct.last_error = None
+256
View File
@@ -0,0 +1,256 @@
"""API endpoints for managing IMAP ingestion profiles.
Ingestion profiles allow fine-grained control over which attachment types are
accepted when ingesting emails via IMAP. Each profile carries a list of enabled
file-type categories (e.g. ``["pdf", "office", "images"]``) drawn from the
canonical set defined in :mod:`app.utils.allowed_types`.
Built-in system profiles (``is_builtin=True``) are read-only and cannot be
deleted or modified. Users may create their own profiles which are private to
their ``owner_id``. System-level global profiles (``owner_id=None``) are visible
to all users but can only be created by administrators.
"""
import json
import logging
from datetime import datetime, timezone
from typing import Annotated, Any
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from app.database import get_db
from app.models import ImapIngestionProfile
from app.utils.allowed_types import FILE_TYPE_CATEGORIES
from app.utils.user_scope import get_current_owner_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/imap-profiles", tags=["imap-profiles"])
DbSession = Annotated[Session, Depends(get_db)]
# ---------------------------------------------------------------------------
# Auth helpers
# ---------------------------------------------------------------------------
def _get_owner_id(request: Request) -> str:
"""Return the current user's owner ID, raising 401 if unauthenticated."""
owner_id = get_current_owner_id(request)
if owner_id is None:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
return owner_id
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
# ---------------------------------------------------------------------------
# Pydantic schemas
# ---------------------------------------------------------------------------
_VALID_CATEGORIES = set(FILE_TYPE_CATEGORIES.keys())
class ImapProfileCreate(BaseModel):
"""Schema for creating a new ingestion profile."""
name: str = Field(..., min_length=1, max_length=255, description="Human-readable profile name")
description: str | None = Field(default=None, description="Optional description")
allowed_categories: list[str] = Field(
...,
min_length=1,
description=(f"List of enabled file-type category keys. Valid values: {sorted(_VALID_CATEGORIES)}"),
)
class ImapProfileUpdate(BaseModel):
"""Schema for updating an existing profile (all fields optional)."""
name: str | None = Field(default=None, min_length=1, max_length=255)
description: str | None = None
allowed_categories: list[str] | None = Field(default=None, min_length=1)
# ---------------------------------------------------------------------------
# Validation helpers
# ---------------------------------------------------------------------------
def _validate_categories(categories: list[str]) -> list[str]:
"""Raise 422 if any category key is unknown; return the cleaned list."""
unknown = [c for c in categories if c not in _VALID_CATEGORIES]
if unknown:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Unknown category key(s): {unknown}. Valid keys: {sorted(_VALID_CATEGORIES)}",
)
# Deduplicate while preserving order
seen: set[str] = set()
result: list[str] = []
for cat in categories:
if cat not in seen:
seen.add(cat)
result.append(cat)
return result
# ---------------------------------------------------------------------------
# Serialisation
# ---------------------------------------------------------------------------
def _to_response(profile: ImapIngestionProfile) -> dict[str, Any]:
"""Serialize a profile row to a response dict."""
try:
categories = json.loads(profile.allowed_categories)
except (ValueError, TypeError):
categories = []
# Enrich categories with display metadata
categories_detail = [
{
"key": cat,
"label": FILE_TYPE_CATEGORIES[cat]["label"] if cat in FILE_TYPE_CATEGORIES else cat,
"description": FILE_TYPE_CATEGORIES[cat]["description"] if cat in FILE_TYPE_CATEGORIES else "",
}
for cat in categories
]
return {
"id": profile.id,
"name": profile.name,
"description": profile.description,
"owner_id": profile.owner_id,
"allowed_categories": categories,
"categories_detail": categories_detail,
"is_builtin": profile.is_builtin,
"created_at": profile.created_at.isoformat() if profile.created_at else None,
"updated_at": profile.updated_at.isoformat() if profile.updated_at else None,
}
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@router.get("/categories", summary="List available file-type categories")
def list_categories(request: Request, owner_id: CurrentOwner) -> list[dict[str, Any]]:
"""Return the full list of file-type categories that can be used in profiles."""
return [
{
"key": key,
"label": info["label"],
"description": info["description"],
}
for key, info in FILE_TYPE_CATEGORIES.items()
]
@router.get("/", summary="List ingestion profiles visible to the current user")
def list_profiles(request: Request, db: DbSession, owner_id: CurrentOwner) -> list[dict[str, Any]]:
"""Return all profiles: system-global (owner_id=NULL) and the user's own profiles."""
profiles = (
db.query(ImapIngestionProfile)
.filter(
(ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711
)
.order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
.all()
)
return [_to_response(p) for p in profiles]
@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new ingestion profile")
def create_profile(request: Request, body: ImapProfileCreate, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
"""Create a new ingestion profile owned by the current user."""
categories = _validate_categories(body.allowed_categories)
profile = ImapIngestionProfile(
name=body.name,
description=body.description,
owner_id=owner_id,
allowed_categories=json.dumps(categories),
is_builtin=False,
)
try:
db.add(profile)
db.commit()
db.refresh(profile)
except Exception:
db.rollback()
raise
logger.info("User %s created IMAP ingestion profile %d ('%s')", owner_id, profile.id, body.name)
return _to_response(profile)
@router.get("/{profile_id}", summary="Get a single ingestion profile")
def get_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
"""Return a single profile by ID. Only the owner or system profiles are accessible."""
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
return _to_response(profile)
@router.put("/{profile_id}", summary="Update an ingestion profile")
def update_profile(
profile_id: int,
request: Request,
body: ImapProfileUpdate,
db: DbSession,
owner_id: CurrentOwner,
) -> dict[str, Any]:
"""Update an existing ingestion profile. Built-in profiles cannot be modified."""
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
if profile.is_builtin:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Built-in profiles cannot be modified.",
)
if body.name is not None:
profile.name = body.name
if "description" in body.model_fields_set:
profile.description = body.description
if body.allowed_categories is not None:
categories = _validate_categories(body.allowed_categories)
profile.allowed_categories = json.dumps(categories)
profile.updated_at = datetime.now(timezone.utc)
try:
db.commit()
db.refresh(profile)
except Exception:
db.rollback()
raise
logger.info("User %s updated IMAP ingestion profile %d", owner_id, profile_id)
return _to_response(profile)
@router.delete("/{profile_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an ingestion profile")
def delete_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> None:
"""Delete an ingestion profile. Built-in profiles cannot be deleted."""
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
if profile.is_builtin:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Built-in profiles cannot be deleted.",
)
try:
db.delete(profile)
db.commit()
except Exception:
db.rollback()
raise
logger.info("User %s deleted IMAP ingestion profile %d", owner_id, profile_id)
+45 -4
View File
@@ -401,6 +401,48 @@ class PipelineStep(Base):
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class ImapIngestionProfile(Base):
"""Named ingestion profile controlling which attachment types are accepted from IMAP emails.
Profiles group file-type categories (e.g. "pdf", "office", "images") so users
can precisely control what gets ingested from each mailbox.
System-provided built-in profiles (``is_builtin=True``) are seeded by the
migration and cannot be deleted or renamed. Users may create their own profiles
(``owner_id`` set to their identifier) or rely on the global system profiles
(``owner_id=None``).
``allowed_categories`` stores a JSON list of category strings, e.g.::
'["pdf", "office", "opendocument", "text", "web"]'
Valid category names are defined in ``app.utils.allowed_types.FILE_TYPE_CATEGORIES``.
"""
__tablename__ = "imap_ingestion_profiles"
id = Column(Integer, primary_key=True, index=True)
# Human-readable profile name (e.g. "Documents Only", "Documents + Images")
name = Column(String(255), nullable=False)
# Optional description shown in the UI
description = Column(Text, nullable=True)
# Owner of this profile. NULL = global/system profile available to all users.
owner_id = Column(String, nullable=True, index=True)
# JSON-encoded list of enabled category keys. Example: '["pdf","office","text"]'
# See FILE_TYPE_CATEGORIES in app/utils/allowed_types.py for valid values.
allowed_categories = Column(Text, nullable=False, default='["pdf","office","opendocument","text","web"]')
# Built-in system profiles that cannot be deleted or modified via the API.
is_builtin = Column(Boolean, nullable=False, default=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class UserImapAccount(Base):
"""Per-user IMAP ingestion account.
@@ -439,10 +481,9 @@ class UserImapAccount(Base):
# When True, emails are deleted from the mailbox after their attachments are processed
delete_after_process = Column(Boolean, nullable=False, default=False)
# Override for which attachment types to ingest.
# NULL means "inherit the global imap_attachment_filter setting".
# Allowed values: 'documents_only', 'all'
attachment_filter = Column(String(50), nullable=True, default=None)
# Optional reference to an ImapIngestionProfile.
# NULL means "use the global imap_attachment_filter setting" (system default).
profile_id = Column(Integer, ForeignKey("imap_ingestion_profiles.id"), nullable=True)
# When False the account is not polled by the periodic task (but not deleted)
is_active = Column(Boolean, nullable=False, default=True)
+79 -71
View File
@@ -13,7 +13,11 @@ from celery import shared_task
from app.config import settings
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
from app.tasks.process_document import process_document # Updated import
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, DOCUMENT_MIME_TYPES, IMAGE_MIME_TYPES
from app.utils.allowed_types import (
ALL_CATEGORIES,
DEFAULT_CATEGORIES,
get_allowed_types_for_categories,
)
# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
_db_session_factory = None
@@ -50,6 +54,33 @@ def _decrypt_imap_password(password: str | None) -> str | None:
return decrypt_value(password)
def _resolve_categories_for_profile(profile_id: int | None) -> list[str]:
"""Return the list of allowed categories for a profile ID.
Loads the profile from the database. If ``profile_id`` is ``None`` or the
profile is not found, falls back to the global ``settings.imap_attachment_filter``
string (``'documents_only'`` → default categories; ``'all'`` → all categories).
"""
if profile_id is not None:
try:
from app.models import ImapIngestionProfile
db = _get_db_session()
try:
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
if profile:
return json.loads(profile.allowed_categories)
finally:
db.close()
except Exception as exc: # noqa: BLE001
logger.warning("Could not load IMAP ingestion profile %d: %s — using global default", profile_id, exc)
# Fall back to global setting
if settings.imap_attachment_filter == "all":
return ALL_CATEGORIES
return DEFAULT_CATEGORIES
LOCK_KEY = "imap_lock" # Unique key for locking
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
@@ -180,7 +211,7 @@ def _pull_user_imap_accounts() -> None:
use_ssl=acct.use_ssl,
delete_after_process=acct.delete_after_process,
owner_id=acct.owner_id,
attachment_filter=acct.attachment_filter or settings.imap_attachment_filter,
allowed_categories=_resolve_categories_for_profile(acct.profile_id),
)
# Record successful poll
acct.last_checked_at = datetime.now(timezone.utc)
@@ -251,7 +282,9 @@ def _pull_user_integration_imap() -> None:
use_ssl = cfg.get("use_ssl", True)
delete_after = cfg.get("delete_after_process", False)
gmail_labels = cfg.get("gmail_apply_labels", True)
attachment_filter = cfg.get("attachment_filter") or settings.imap_attachment_filter
# Integrations can store a profile_id in config; fall back to global default
profile_id = cfg.get("profile_id")
allowed_categories = _resolve_categories_for_profile(profile_id)
if not (host and username and password):
logger.warning(
@@ -271,7 +304,7 @@ def _pull_user_integration_imap() -> None:
delete_after_process=delete_after,
owner_id=integ.owner_id,
gmail_apply_labels=gmail_labels,
attachment_filter=attachment_filter,
allowed_categories=allowed_categories,
)
integ.last_used_at = datetime.now(timezone.utc)
integ.last_error = None
@@ -332,7 +365,7 @@ def pull_inbox(
delete_after_process,
owner_id=None,
gmail_apply_labels=True,
attachment_filter=None,
allowed_categories=None,
):
"""
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
@@ -349,13 +382,22 @@ def pull_inbox(
attributed to this user via ``process_document`` / ``convert_to_pdf``.
gmail_apply_labels: Whether to apply Gmail-specific labels and stars to
processed emails. Only relevant for Gmail hosts. Defaults to True.
attachment_filter: Controls which attachment types to ingest.
``'documents_only'`` (default) PDFs and office files only.
``'all'`` all supported types including images.
``None`` falls back to the global ``settings.imap_attachment_filter``.
allowed_categories: List of file-type category keys to ingest (e.g.
``["pdf", "office", "images"]``). ``None`` falls back to the
global ``settings.imap_attachment_filter`` mapping.
"""
resolved_filter = attachment_filter or settings.imap_attachment_filter
logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
if allowed_categories is None:
allowed_categories = _resolve_categories_for_profile(None)
effective_mime_types, effective_extensions = get_allowed_types_for_categories(allowed_categories)
logger.info(
"Connecting to %s at %s:%s (SSL=%s) — categories: %s",
mailbox_key,
host,
port,
use_ssl,
allowed_categories,
)
processed_emails = load_processed_emails()
try:
@@ -414,9 +456,13 @@ def pull_inbox(
logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
continue
# Process attachments (and convert non-PDF files).
# We call the function without assigning its return value since it is not used.
fetch_attachments_and_enqueue(email_message, owner_id=owner_id, attachment_filter=resolved_filter)
# Process attachments using the resolved mime types / extensions.
fetch_attachments_and_enqueue(
email_message,
owner_id=owner_id,
effective_mime_types=effective_mime_types,
effective_extensions=effective_extensions,
)
if settings.imap_readonly_mode:
logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
@@ -448,19 +494,20 @@ def pull_inbox(
def fetch_attachments_and_enqueue(
email_message,
owner_id: str | None = None,
attachment_filter: str | None = None,
effective_mime_types: frozenset[str] | None = None,
effective_extensions: frozenset[str] | None = None,
):
"""
Extracts attachments from the email and processes only allowed file types.
Files are first checked against the ``attachment_filter`` to determine which
broad categories are permitted, then validated against known MIME types /
extensions.
The caller is responsible for computing ``effective_mime_types`` and
``effective_extensions`` from the relevant :class:`ImapIngestionProfile` (or
the global default) via :func:`app.utils.allowed_types.get_allowed_types_for_categories`
before calling this function. ``pull_inbox`` does this automatically.
Attachment filter values:
- ``'documents_only'`` (default): PDFs, office files (Word, Excel, PowerPoint,
OpenDocument, RTF), plain text, CSV, HTML, and Markdown. Images are skipped.
- ``'all'``: All supported file types, including images (JPEG, PNG, GIF, etc.).
If either set is ``None`` the function falls back to the default category list
so the function still works correctly when called directly in tests or from
other contexts.
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
any other allowed file is enqueued for conversion to PDF.
@@ -469,36 +516,13 @@ def fetch_attachments_and_enqueue(
email_message: The parsed email message to extract attachments from.
owner_id: Optional user identifier forwarded to ``process_document`` /
``convert_to_pdf`` for multi-tenant attribution.
attachment_filter: Override for the filter level. Defaults to
``settings.imap_attachment_filter`` when not provided.
effective_mime_types: Pre-computed frozenset of allowed MIME type strings.
effective_extensions: Pre-computed frozenset of allowed file extension strings.
Returns True if at least one allowed attachment was processed.
"""
resolved_filter = attachment_filter or settings.imap_attachment_filter
# Build the effective allowed MIME type set based on the filter
if resolved_filter == "all":
effective_mime_types = ALLOWED_MIME_TYPES
else:
# 'documents_only' (and any unrecognised value): exclude images
effective_mime_types = DOCUMENT_MIME_TYPES
# Build the effective allowed extensions set (images excluded for documents_only)
if resolved_filter == "all":
effective_extensions = ALLOWED_EXTENSIONS
else:
image_extensions = {
".jpg",
".jpeg",
".png",
".gif",
".bmp",
".tiff",
".tif",
".webp",
".svg",
}
effective_extensions = ALLOWED_EXTENSIONS - image_extensions
if effective_mime_types is None or effective_extensions is None:
effective_mime_types, effective_extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
has_attachment = False
for part in email_message.walk():
@@ -515,30 +539,14 @@ def fetch_attachments_and_enqueue(
mime_type = part.get_content_type()
file_ext = os.path.splitext(filename)[1].lower()
# Skip images when filter is documents_only
is_image = mime_type in IMAGE_MIME_TYPES or file_ext in {
".jpg",
".jpeg",
".png",
".gif",
".bmp",
".tiff",
".tif",
".webp",
".svg",
}
if is_image and resolved_filter != "all":
logger.info(
"Skipping image attachment %s (MIME: %s) — attachment_filter=%s",
filename,
mime_type,
resolved_filter,
)
continue
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension:
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
logger.info(
"Skipping attachment %s (MIME: %s, ext: %s) — not in effective allowed set",
filename,
mime_type,
file_ext,
)
continue
file_path = os.path.join(settings.workdir, filename)
+154
View File
@@ -131,3 +131,157 @@ ALLOWED_EXTENSIONS: set[str] = {
".md",
".markdown",
}
# ---------------------------------------------------------------------------
# Fine-grained file-type categories used by IMAP ingestion profiles.
# Each category groups related MIME types and extensions so that users can
# enable/disable a logical collection of formats (e.g. "images") rather than
# having to manage individual MIME strings.
# ---------------------------------------------------------------------------
FILE_TYPE_CATEGORIES: dict[str, dict] = {
"pdf": {
"label": "PDF",
"description": "PDF documents (.pdf)",
"mime_types": frozenset({"application/pdf"}),
"extensions": frozenset({".pdf"}),
},
"office": {
"label": "Microsoft Office",
"description": "Word, Excel and PowerPoint files (.doc, .docx, .xls, .xlsx, .ppt, .pptx, …)",
"mime_types": frozenset(
{
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
"application/vnd.ms-word.document.macroEnabled.12",
"application/vnd.ms-word.template.macroEnabled.12",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.openxmlformats-officedocument.spreadsheetml.template",
"application/vnd.ms-excel.sheet.macroEnabled.12",
"application/vnd.ms-excel.sheet.binary.macroEnabled.12",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.openxmlformats-officedocument.presentationml.template",
"application/vnd.openxmlformats-officedocument.presentationml.slideshow",
"application/vnd.ms-powerpoint.presentation.macroEnabled.12",
}
),
"extensions": frozenset(
{
".doc",
".docx",
".docm",
".dot",
".dotx",
".dotm",
".xls",
".xlsx",
".xlsm",
".xlsb",
".xlt",
".xltx",
".xlw",
".ppt",
".pptx",
".pptm",
".pps",
".ppsx",
".pot",
".potx",
}
),
},
"opendocument": {
"label": "OpenDocument (LibreOffice)",
"description": "LibreOffice / OpenOffice files (.odt, .ods, .odp, …)",
"mime_types": frozenset(
{
"application/vnd.oasis.opendocument.text",
"application/vnd.oasis.opendocument.spreadsheet",
"application/vnd.oasis.opendocument.presentation",
"application/vnd.oasis.opendocument.graphics",
"application/vnd.oasis.opendocument.formula",
}
),
"extensions": frozenset({".odt", ".ods", ".odp", ".odg", ".odf"}),
},
"text": {
"label": "Text & Data",
"description": "Plain text, CSV and RTF files (.txt, .csv, .rtf)",
"mime_types": frozenset(
{
"text/plain",
"text/csv",
"application/rtf",
"text/rtf",
}
),
"extensions": frozenset({".txt", ".csv", ".rtf"}),
},
"web": {
"label": "Web & Markup",
"description": "HTML and Markdown files (.html, .htm, .md, .markdown)",
"mime_types": frozenset(
{
"text/html",
"text/markdown",
"text/x-markdown",
}
),
"extensions": frozenset({".html", ".htm", ".md", ".markdown"}),
},
"images": {
"label": "Images",
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)",
"mime_types": frozenset(
{
"image/jpeg",
"image/jpg",
"image/png",
"image/gif",
"image/bmp",
"image/tiff",
"image/webp",
"image/svg+xml",
}
),
"extensions": frozenset(
{
".jpg",
".jpeg",
".png",
".gif",
".bmp",
".tiff",
".tif",
".webp",
".svg",
}
),
},
}
# Default categories for the "documents only" built-in profile (no images)
DEFAULT_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web"]
# All categories including images
ALL_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web", "images"]
def get_allowed_types_for_categories(
categories: list[str],
) -> tuple[frozenset[str], frozenset[str]]:
"""Return ``(mime_types, extensions)`` for the given category list.
Unknown category names are silently ignored so that future categories
don't break existing profiles.
"""
mime_types: set[str] = set()
extensions: set[str] = set()
for cat in categories:
info = FILE_TYPE_CATEGORIES.get(cat)
if info:
mime_types |= info["mime_types"]
extensions |= info["extensions"]
return frozenset(mime_types), frozenset(extensions)
+47 -1
View File
@@ -1,11 +1,13 @@
"""User-facing view for the per-user IMAP ingestion dashboard."""
import json
import logging
from fastapi import Request
from sqlalchemy.orm import Session
from app.models import UserImapAccount
from app.models import ImapIngestionProfile, UserImapAccount
from app.utils.allowed_types import FILE_TYPE_CATEGORIES
from app.utils.subscription import get_tier, get_user_tier_id
from app.utils.user_scope import get_current_owner_id
from app.views.base import APIRouter, Depends, get_db, require_login, templates
@@ -25,6 +27,22 @@ def _get_max_mailboxes(tier: dict) -> int | None:
return max_mb
def _serialize_profile(profile: ImapIngestionProfile) -> dict:
"""Serialize a profile for JSON embedding in the template."""
try:
categories = json.loads(profile.allowed_categories)
except (ValueError, TypeError):
categories = []
return {
"id": profile.id,
"name": profile.name,
"description": profile.description,
"owner_id": profile.owner_id,
"allowed_categories": categories,
"is_builtin": profile.is_builtin,
}
@router.get("/imap-accounts")
@require_login
async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
@@ -49,11 +67,39 @@ async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
max_mailboxes = _get_max_mailboxes(tier)
can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes)
# Load ingestion profiles: system-global + user's own
profiles: list[ImapIngestionProfile] = (
db.query(ImapIngestionProfile)
.filter(
(ImapIngestionProfile.owner_id == None) # noqa: E711
| (ImapIngestionProfile.owner_id == owner_id)
)
.order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
.all()
if owner_id
else db.query(ImapIngestionProfile)
.filter(ImapIngestionProfile.owner_id == None) # noqa: E711
.order_by(ImapIngestionProfile.id)
.all()
)
# Category definitions for the UI checkbox builder
categories = [
{
"key": key,
"label": info["label"],
"description": info["description"],
}
for key, info in FILE_TYPE_CATEGORIES.items()
]
return templates.TemplateResponse(
"imap_accounts.html",
{
"request": request,
"accounts": accounts,
"profiles": [_serialize_profile(p) for p in profiles],
"categories": categories,
"current_count": current_count,
"max_mailboxes": max_mailboxes,
"can_add": can_add,
+467 -26
View File
@@ -172,14 +172,9 @@
<i class="fas fa-trash-alt mr-1" aria-hidden="true"></i>Delete after process
</span>
</template>
<template x-if="acct.attachment_filter === 'all'">
<template x-if="acct.profile_id">
<span class="inline-flex items-center px-2 py-0.5 rounded bg-blue-50 text-blue-700">
<i class="fas fa-paperclip mr-1" aria-hidden="true"></i>All attachments
</span>
</template>
<template x-if="acct.attachment_filter === 'documents_only'">
<span class="inline-flex items-center px-2 py-0.5 rounded bg-gray-50 text-gray-600">
<i class="fas fa-file-alt mr-1" aria-hidden="true"></i>Documents only
<i class="fas fa-filter mr-1" aria-hidden="true"></i><span x-text="profileName(acct.profile_id)"></span>
</span>
</template>
<template x-if="acct.last_checked_at">
@@ -398,22 +393,45 @@
<!-- Attachment filter -->
<div>
<label for="acct-attachment-filter" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
Attachment Types to Ingest
<label for="acct-profile" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
Ingestion Profile
</label>
<select
id="acct-attachment-filter"
x-model="form.attachment_filter"
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
aria-describedby="acct-attachment-filter-hint"
>
<option value="">Use global default</option>
<option value="documents_only">Documents only (PDF, Office, ODT, RTF, TXT, CSV) — no images</option>
<option value="all">All supported types (including images)</option>
</select>
<p id="acct-attachment-filter-hint" class="mt-1 text-xs text-gray-400 dark:text-gray-500">
Override the system-wide attachment filter for this mailbox. Leave blank to use the global default (documents only).
<div class="flex gap-2 items-center">
<select
id="acct-profile"
x-model.number="form.profile_id"
class="flex-1 px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
aria-describedby="acct-profile-hint"
>
<option :value="null">Use global default</option>
<template x-for="profile in profiles" :key="profile.id">
<option :value="profile.id" x-text="profile.name + (profile.is_builtin ? '' : ' (custom)')"></option>
</template>
</select>
<button
type="button"
@click="openProfileModal(null)"
class="inline-flex items-center px-3 py-2 text-sm font-medium rounded border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-blue-500"
style="min-height:40px;"
title="Create new profile"
aria-label="Create new ingestion profile"
>
<i class="fas fa-plus" aria-hidden="true"></i>
</button>
</div>
<p id="acct-profile-hint" class="mt-1 text-xs text-gray-400 dark:text-gray-500">
Controls which attachment types are ingested. Leave blank to use the global default (documents only).
<button type="button" @click="showProfilesSection = !showProfilesSection" class="underline hover:text-blue-600 focus:outline-none">
Manage profiles
</button>
</p>
<!-- Inline selected profile summary -->
<template x-if="form.profile_id">
<div class="mt-2 text-xs bg-blue-50 dark:bg-blue-900/20 rounded px-2 py-1.5 text-blue-800 dark:text-blue-200">
<strong x-text="profileName(form.profile_id)"></strong>:
<span x-text="profileCategorySummary(form.profile_id)"></span>
</div>
</template>
</div>
<!-- Test connection result -->
@@ -524,10 +542,276 @@
</div>
<!-- ── Ingestion Profiles panel ─────────────────────────────────────────────── -->
<div
x-show="showProfilesSection"
x-transition
class="fixed inset-0 z-50 flex items-center justify-center bg-black/50 px-4"
role="dialog"
aria-modal="true"
aria-labelledby="profiles-panel-title"
@keydown.escape.window="showProfilesSection = false"
style="display:none;"
>
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-2xl overflow-y-auto max-h-[90vh]" @click.stop>
<div class="px-6 pt-6 pb-2 flex items-center justify-between">
<h2 id="profiles-panel-title" class="text-lg font-semibold text-gray-900 dark:text-white flex items-center gap-2">
<i class="fas fa-filter text-blue-500" aria-hidden="true"></i>
Ingestion Profiles
</h2>
<button
type="button"
@click="showProfilesSection = false"
class="text-gray-400 hover:text-gray-600 dark:hover:text-gray-200 focus:outline-none focus:ring-2 focus:ring-blue-500 rounded"
aria-label="Close profiles panel"
>
<i class="fas fa-times text-xl" aria-hidden="true"></i>
</button>
</div>
<div class="px-6 pb-6 pt-2">
<p class="text-sm text-gray-500 dark:text-gray-400 mb-4">
Profiles define which attachment types are ingested from emails. Assign a profile to each IMAP account for fine-grained control.
</p>
<!-- Profile list -->
<div class="space-y-3 mb-4">
<template x-for="profile in profiles" :key="profile.id">
<div class="border border-gray-100 dark:border-gray-700 rounded-lg p-4 flex items-start gap-3">
<div class="flex-1 min-w-0">
<div class="flex items-center gap-2 flex-wrap">
<span class="font-medium text-sm text-gray-900 dark:text-white" x-text="profile.name"></span>
<template x-if="profile.is_builtin">
<span class="text-xs px-1.5 py-0.5 rounded bg-gray-100 dark:bg-gray-700 text-gray-500">built-in</span>
</template>
</div>
<p class="text-xs text-gray-500 dark:text-gray-400 mt-0.5" x-text="profile.description || ''"></p>
<!-- Category badges -->
<div class="flex flex-wrap gap-1 mt-2">
<template x-for="cat in profile.allowed_categories" :key="cat">
<span class="inline-flex items-center px-2 py-0.5 rounded text-xs font-medium bg-blue-50 dark:bg-blue-900/30 text-blue-700 dark:text-blue-300">
<span x-text="categoryLabel(cat)"></span>
</span>
</template>
</div>
</div>
<div class="flex gap-1 shrink-0">
<button
type="button"
@click="openProfileModal(profile)"
:disabled="profile.is_builtin"
:class="profile.is_builtin ? 'opacity-40 cursor-not-allowed' : ''"
class="p-1.5 text-gray-400 hover:text-blue-600 focus:outline-none focus:ring-2 focus:ring-blue-500 rounded"
:aria-label="`Edit profile ${profile.name}`"
:title="profile.is_builtin ? 'Built-in profiles cannot be edited' : 'Edit profile'"
>
<i class="fas fa-edit text-sm" aria-hidden="true"></i>
</button>
<button
type="button"
@click="confirmDeleteProfile(profile)"
:disabled="profile.is_builtin"
:class="profile.is_builtin ? 'opacity-40 cursor-not-allowed' : ''"
class="p-1.5 text-gray-400 hover:text-red-600 focus:outline-none focus:ring-2 focus:ring-red-500 rounded"
:aria-label="`Delete profile ${profile.name}`"
:title="profile.is_builtin ? 'Built-in profiles cannot be deleted' : 'Delete profile'"
>
<i class="fas fa-trash-alt text-sm" aria-hidden="true"></i>
</button>
</div>
</div>
</template>
</div>
<button
type="button"
@click="openProfileModal(null)"
class="inline-flex items-center px-4 py-2 text-sm font-medium rounded bg-blue-600 hover:bg-blue-700 text-white focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
style="min-height:40px;"
>
<i class="fas fa-plus mr-2" aria-hidden="true"></i> New Profile
</button>
</div>
</div>
</div>
<!-- ── Profile create / edit modal ────────────────────────────────────────────── -->
<div
x-show="profileModalOpen"
x-transition:enter="ease-out duration-200"
x-transition:enter-start="opacity-0"
x-transition:enter-end="opacity-100"
x-transition:leave="ease-in duration-150"
x-transition:leave-start="opacity-100"
x-transition:leave-end="opacity-0"
class="fixed inset-0 z-[60] flex items-center justify-center bg-black/50 px-4"
role="dialog"
aria-modal="true"
:aria-labelledby="editingProfile ? 'profile-modal-title-edit' : 'profile-modal-title-create'"
@keydown.escape.window="closeProfileModal()"
style="display:none;"
>
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-lg overflow-y-auto max-h-[90vh]" @click.stop>
<div class="px-6 pt-6 pb-2 flex items-center justify-between">
<h2
:id="editingProfile ? 'profile-modal-title-edit' : 'profile-modal-title-create'"
class="text-lg font-semibold text-gray-900 dark:text-white"
x-text="editingProfile ? 'Edit Profile' : 'New Ingestion Profile'"
></h2>
<button
type="button"
@click="closeProfileModal()"
class="text-gray-400 hover:text-gray-600 dark:hover:text-gray-200 focus:outline-none focus:ring-2 focus:ring-blue-500 rounded"
aria-label="Close modal"
>
<i class="fas fa-times text-xl" aria-hidden="true"></i>
</button>
</div>
<form @submit.prevent="saveProfile()" class="px-6 pb-6 pt-4 space-y-4">
<!-- Profile name -->
<div>
<label for="profile-name" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
Profile Name <span class="text-red-500" aria-hidden="true">*</span>
</label>
<input
id="profile-name"
type="text"
x-model="profileForm.name"
placeholder="e.g. Scanner Inbox, Finance Documents"
required
maxlength="255"
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
aria-required="true"
/>
</div>
<!-- Description -->
<div>
<label for="profile-description" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
Description
</label>
<input
id="profile-description"
type="text"
x-model="profileForm.description"
placeholder="Optional description"
maxlength="500"
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
/>
</div>
<!-- Category checkboxes -->
<div>
<fieldset>
<legend class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-2">
File Type Categories <span class="text-red-500" aria-hidden="true">*</span>
</legend>
<div class="space-y-2">
<template x-for="cat in categories" :key="cat.key">
<label class="flex items-start gap-3 cursor-pointer group">
<input
type="checkbox"
:value="cat.key"
:checked="profileForm.allowed_categories.includes(cat.key)"
@change="toggleCategory(cat.key)"
class="mt-0.5 h-4 w-4 rounded border-gray-300 text-blue-600 focus:ring-blue-500"
/>
<span class="flex-1">
<span class="text-sm font-medium text-gray-800 dark:text-gray-100" x-text="cat.label"></span>
<span class="block text-xs text-gray-500 dark:text-gray-400" x-text="cat.description"></span>
</span>
</label>
</template>
</div>
<p x-show="profileForm.allowed_categories.length === 0" class="mt-2 text-xs text-red-600" role="alert">
Select at least one category.
</p>
</fieldset>
</div>
<!-- Form error -->
<template x-if="profileFormError">
<p class="text-sm text-red-600" role="alert" x-text="profileFormError"></p>
</template>
<!-- Actions -->
<div class="flex justify-end gap-2 pt-2 border-t border-gray-100 dark:border-gray-700">
<button
type="button"
@click="closeProfileModal()"
class="px-4 py-2 text-sm font-medium rounded border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-blue-500"
style="min-height:40px;"
>
Cancel
</button>
<button
type="submit"
:disabled="savingProfile || profileForm.allowed_categories.length === 0"
class="inline-flex items-center px-4 py-2 text-sm font-medium rounded bg-blue-600 hover:bg-blue-700 text-white focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50"
style="min-height:40px;"
>
<template x-if="savingProfile">
<i class="fas fa-spinner fa-spin mr-2" aria-hidden="true"></i>
</template>
<span x-text="editingProfile ? 'Save Changes' : 'Create Profile'"></span>
</button>
</div>
</form>
</div>
</div>
<!-- ── Delete profile confirmation modal ────────────────────────────────────── -->
<div
x-show="deleteProfileModalOpen"
x-transition
class="fixed inset-0 z-[60] flex items-center justify-center bg-black/50 px-4"
role="dialog"
aria-modal="true"
aria-labelledby="delete-profile-modal-title"
@keydown.escape.window="deleteProfileModalOpen = false"
style="display:none;"
>
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-sm p-6" @click.stop>
<h2 id="delete-profile-modal-title" class="text-lg font-semibold text-gray-900 dark:text-white mb-2">
Delete Profile?
</h2>
<p class="text-sm text-gray-600 dark:text-gray-300 mb-4">
Are you sure you want to delete the profile
<strong x-text="profileToDelete ? profileToDelete.name : ''"></strong>?
IMAP accounts using this profile will fall back to the global default.
</p>
<div class="flex justify-end gap-2">
<button
type="button"
@click="deleteProfileModalOpen = false; profileToDelete = null"
class="px-4 py-2 text-sm font-medium rounded border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-gray-500"
style="min-height:40px;"
>
Cancel
</button>
<button
type="button"
@click="deleteProfile()"
:disabled="deletingProfile"
class="inline-flex items-center px-4 py-2 text-sm font-medium rounded bg-red-600 hover:bg-red-700 text-white focus:outline-none focus:ring-2 focus:ring-red-500 disabled:opacity-50"
style="min-height:40px;"
>
<template x-if="deletingProfile">
<i class="fas fa-spinner fa-spin mr-2" aria-hidden="true"></i>
</template>
Delete
</button>
</div>
</div>
</div>
<script>
function imapAccountsApp() {
return {
accounts: {{ accounts | tojson }},
profiles: {{ profiles | tojson }},
categories: {{ categories | tojson }},
quota: {
current_count: {{ current_count }},
max_mailboxes: {{ max_mailboxes | tojson }},
@@ -538,17 +822,17 @@ function imapAccountsApp() {
loading: false,
alert: { show: false, type: '', title: '', message: '' },
// Modal state
// Account modal state
modalOpen: false,
editingAccount: null,
form: { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true },
form: { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true, profile_id: null },
showPassword: false,
saving: false,
testing: false,
testResult: null,
formError: null,
// Delete state
// Account delete state
deleteModalOpen: false,
accountToDelete: null,
deleting: false,
@@ -556,6 +840,17 @@ function imapAccountsApp() {
// Test (saved account) state
testingId: null,
// Profiles panel & modal state
showProfilesSection: false,
profileModalOpen: false,
editingProfile: null,
profileForm: { name: '', description: '', allowed_categories: [] },
savingProfile: false,
profileFormError: null,
deleteProfileModalOpen: false,
profileToDelete: null,
deletingProfile: false,
get canAdd() {
return this.quota.can_add;
},
@@ -579,9 +874,33 @@ function imapAccountsApp() {
setTimeout(() => { this.alert.show = false; }, 5000);
},
// ── Profile helpers ──────────────────────────────────────────────────────
profileById(id) {
return this.profiles.find(p => p.id === id) || null;
},
profileName(id) {
const p = this.profileById(id);
return p ? p.name : 'Unknown profile';
},
profileCategorySummary(id) {
const p = this.profileById(id);
if (!p) return '';
return p.allowed_categories.map(k => this.categoryLabel(k)).join(', ');
},
categoryLabel(key) {
const cat = this.categories.find(c => c.key === key);
return cat ? cat.label : key;
},
// ── Account modal ────────────────────────────────────────────────────────
openCreateModal() {
this.editingAccount = null;
this.form = { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true, attachment_filter: '' };
this.form = { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true, profile_id: null };
this.showPassword = false;
this.testResult = null;
this.formError = null;
@@ -599,7 +918,7 @@ function imapAccountsApp() {
use_ssl: acct.use_ssl,
delete_after_process: acct.delete_after_process,
is_active: acct.is_active,
attachment_filter: acct.attachment_filter || '',
profile_id: acct.profile_id || null,
};
this.showPassword = false;
this.testResult = null;
@@ -623,6 +942,8 @@ function imapAccountsApp() {
if (this.editingAccount && payload.password === '') {
delete payload.password;
}
// Normalise profile_id: empty string → null
if (payload.profile_id === '' || payload.profile_id === 0) payload.profile_id = null;
let resp;
if (this.editingAccount) {
resp = await fetch(`/api/imap-accounts/${this.editingAccount.id}`, {
@@ -750,6 +1071,126 @@ function imapAccountsApp() {
this.deleting = false;
}
},
// ── Profile CRUD ─────────────────────────────────────────────────────────
openProfileModal(profile) {
this.editingProfile = profile;
if (profile) {
this.profileForm = {
name: profile.name,
description: profile.description || '',
allowed_categories: [...profile.allowed_categories],
};
} else {
this.profileForm = { name: '', description: '', allowed_categories: ['pdf', 'office', 'opendocument', 'text', 'web'] };
}
this.profileFormError = null;
this.profileModalOpen = true;
},
closeProfileModal() {
this.profileModalOpen = false;
this.editingProfile = null;
this.profileFormError = null;
},
toggleCategory(key) {
const idx = this.profileForm.allowed_categories.indexOf(key);
if (idx === -1) {
this.profileForm.allowed_categories.push(key);
} else {
this.profileForm.allowed_categories.splice(idx, 1);
}
},
async saveProfile() {
if (this.profileForm.allowed_categories.length === 0) {
this.profileFormError = 'Select at least one category.';
return;
}
this.profileFormError = null;
this.savingProfile = true;
try {
const payload = {
name: this.profileForm.name,
description: this.profileForm.description || null,
allowed_categories: this.profileForm.allowed_categories,
};
let resp;
if (this.editingProfile) {
resp = await fetch(`/api/imap-profiles/${this.editingProfile.id}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': getCsrfToken() },
body: JSON.stringify(payload),
});
} else {
resp = await fetch('/api/imap-profiles/', {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': getCsrfToken() },
body: JSON.stringify(payload),
});
}
const data = await resp.json();
if (!resp.ok) {
this.profileFormError = data.detail || 'Failed to save profile.';
return;
}
// Flatten response into the same shape used locally
const normalized = {
id: data.id,
name: data.name,
description: data.description,
owner_id: data.owner_id,
allowed_categories: data.allowed_categories,
is_builtin: data.is_builtin,
};
if (this.editingProfile) {
const idx = this.profiles.findIndex(p => p.id === normalized.id);
if (idx !== -1) this.profiles.splice(idx, 1, normalized);
} else {
this.profiles.push(normalized);
}
this.closeProfileModal();
this.showAlert('success', 'Saved', this.editingProfile ? 'Profile updated.' : 'Profile created.');
} catch (err) {
this.profileFormError = `Network error: ${err.message || 'Unknown error'}. Please try again.`;
} finally {
this.savingProfile = false;
}
},
confirmDeleteProfile(profile) {
this.profileToDelete = profile;
this.deleteProfileModalOpen = true;
},
async deleteProfile() {
if (!this.profileToDelete) return;
this.deletingProfile = true;
try {
const resp = await fetch(`/api/imap-profiles/${this.profileToDelete.id}`, {
method: 'DELETE',
headers: { 'X-CSRF-Token': getCsrfToken() },
});
if (resp.ok || resp.status === 204) {
const deletedId = this.profileToDelete.id;
this.profiles = this.profiles.filter(p => p.id !== deletedId);
// Clear profile_id on accounts that referenced this profile
this.accounts.forEach(a => { if (a.profile_id === deletedId) a.profile_id = null; });
this.showAlert('success', 'Deleted', `Profile "${this.profileToDelete.name}" removed.`);
this.deleteProfileModalOpen = false;
this.profileToDelete = null;
} else {
const data = await resp.json();
this.showAlert('error', 'Delete Failed', data.detail || 'Could not delete profile.');
}
} catch (err) {
this.showAlert('error', 'Delete Failed', `Network error: ${err.message || 'Unknown error'}`);
} finally {
this.deletingProfile = false;
}
},
};
}
+1
View File
@@ -28,6 +28,7 @@ from app.models import ( # noqa: F401
DocumentMetadata,
FileProcessingStep,
FileRecord,
ImapIngestionProfile,
InAppNotification,
LocalUser,
MobileDevice,
@@ -0,0 +1,146 @@
"""Add imap_ingestion_profiles table and migrate user_imap_accounts.
Creates the ``imap_ingestion_profiles`` table, seeds the two built-in profiles
("Documents Only" and "All Files"), and replaces the ``attachment_filter``
string column on ``user_imap_accounts`` with a ``profile_id`` FK that references
the new table.
Revision ID: 033_add_imap_ingestion_profiles
Revises: 032_add_imap_attachment_filter
Create Date: 2026-03-12
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "033_add_imap_ingestion_profiles"
down_revision: Union[str, None] = "032_add_imap_attachment_filter"
depends_on: Union[str, None] = None
# Fixed IDs for the built-in profiles so that the FK migration is reproducible.
_BUILTIN_DOCUMENTS_ONLY_ID = 1
_BUILTIN_ALL_FILES_ID = 2
def upgrade() -> None:
"""Create profiles table, seed built-ins, migrate accounts column."""
# 1 — Create the imap_ingestion_profiles table
op.create_table(
"imap_ingestion_profiles",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("name", sa.String(255), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("owner_id", sa.String(), nullable=True),
sa.Column(
"allowed_categories",
sa.Text(),
nullable=False,
server_default='["pdf","office","opendocument","text","web"]',
),
sa.Column("is_builtin", sa.Boolean(), nullable=False, server_default="0"),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_imap_ingestion_profiles_id", "imap_ingestion_profiles", ["id"])
op.create_index("ix_imap_ingestion_profiles_owner_id", "imap_ingestion_profiles", ["owner_id"])
# 2 — Seed the two built-in profiles
bind = op.get_bind()
bind.execute(
sa.text(
"INSERT INTO imap_ingestion_profiles "
"(id, name, description, owner_id, allowed_categories, is_builtin) "
"VALUES (:id, :name, :desc, NULL, :cats, 1)"
),
[
{
"id": _BUILTIN_DOCUMENTS_ONLY_ID,
"name": "Documents Only",
"desc": (
"Ingest PDFs, Microsoft Office files, OpenDocument files, "
"plain text, CSV, RTF, HTML and Markdown. Images are excluded."
),
"cats": '["pdf","office","opendocument","text","web"]',
},
{
"id": _BUILTIN_ALL_FILES_ID,
"name": "All Files",
"desc": "Ingest all supported file types, including images.",
"cats": '["pdf","office","opendocument","text","web","images"]',
},
],
)
# 3 — Add profile_id column to user_imap_accounts
op.add_column(
"user_imap_accounts",
sa.Column("profile_id", sa.Integer(), nullable=True),
)
# 4 — Migrate existing attachment_filter values to profile_id
bind.execute(
sa.text(
"UPDATE user_imap_accounts SET profile_id = :pid "
"WHERE attachment_filter = 'all'"
),
{"pid": _BUILTIN_ALL_FILES_ID},
)
bind.execute(
sa.text(
"UPDATE user_imap_accounts SET profile_id = :pid "
"WHERE attachment_filter = 'documents_only'"
),
{"pid": _BUILTIN_DOCUMENTS_ONLY_ID},
)
# Rows with NULL attachment_filter keep profile_id = NULL (use global default)
# 5 — Add FK constraint (skip for SQLite which does not enforce FKs at DDL time)
# We use batch_alter_table so this works across SQLite and PostgreSQL
with op.batch_alter_table("user_imap_accounts") as batch_op:
batch_op.create_foreign_key(
"fk_user_imap_accounts_profile_id",
"imap_ingestion_profiles",
["profile_id"],
["id"],
)
# 6 — Drop the now-redundant attachment_filter column
with op.batch_alter_table("user_imap_accounts") as batch_op:
batch_op.drop_column("attachment_filter")
def downgrade() -> None:
"""Reverse the migration: restore attachment_filter, drop profiles table."""
# 1 — Re-add attachment_filter column
with op.batch_alter_table("user_imap_accounts") as batch_op:
batch_op.add_column(sa.Column("attachment_filter", sa.String(50), nullable=True))
# 2 — Restore string values from profile_id
bind = op.get_bind()
bind.execute(
sa.text(
"UPDATE user_imap_accounts SET attachment_filter = 'all' "
"WHERE profile_id = :pid"
),
{"pid": _BUILTIN_ALL_FILES_ID},
)
bind.execute(
sa.text(
"UPDATE user_imap_accounts SET attachment_filter = 'documents_only' "
"WHERE profile_id = :pid"
),
{"pid": _BUILTIN_DOCUMENTS_ONLY_ID},
)
# 3 — Drop the FK and profile_id column
with op.batch_alter_table("user_imap_accounts") as batch_op:
batch_op.drop_constraint("fk_user_imap_accounts_profile_id", type_="foreignkey")
batch_op.drop_column("profile_id")
# 4 — Drop the profiles table
op.drop_index("ix_imap_ingestion_profiles_owner_id", "imap_ingestion_profiles")
op.drop_index("ix_imap_ingestion_profiles_id", "imap_ingestion_profiles")
op.drop_table("imap_ingestion_profiles")
+160
View File
@@ -0,0 +1,160 @@
"""Tests for app/api/imap_profiles.py and app/utils/allowed_types category helpers."""
import json
import pytest
from app.utils.allowed_types import (
ALL_CATEGORIES,
DEFAULT_CATEGORIES,
FILE_TYPE_CATEGORIES,
get_allowed_types_for_categories,
)
@pytest.mark.unit
class TestFileTypeCategories:
"""Tests for FILE_TYPE_CATEGORIES and get_allowed_types_for_categories."""
def test_all_category_keys_present(self):
"""Test that the six expected categories exist."""
assert set(FILE_TYPE_CATEGORIES.keys()) == {"pdf", "office", "opendocument", "text", "web", "images"}
def test_each_category_has_required_fields(self):
"""Test that every category entry has label, description, mime_types, extensions."""
for key, info in FILE_TYPE_CATEGORIES.items():
assert "label" in info, f"Category '{key}' missing 'label'"
assert "description" in info, f"Category '{key}' missing 'description'"
assert "mime_types" in info, f"Category '{key}' missing 'mime_types'"
assert "extensions" in info, f"Category '{key}' missing 'extensions'"
def test_pdf_category_contains_pdf_mime(self):
"""Test that the pdf category includes application/pdf."""
assert "application/pdf" in FILE_TYPE_CATEGORIES["pdf"]["mime_types"]
assert ".pdf" in FILE_TYPE_CATEGORIES["pdf"]["extensions"]
def test_images_category_contains_jpeg(self):
"""Test that the images category includes image/jpeg."""
assert "image/jpeg" in FILE_TYPE_CATEGORIES["images"]["mime_types"]
assert ".jpg" in FILE_TYPE_CATEGORIES["images"]["extensions"]
assert ".png" in FILE_TYPE_CATEGORIES["images"]["extensions"]
def test_get_allowed_types_for_default_categories(self):
"""Test that DEFAULT_CATEGORIES excludes image MIME types."""
mime_types, extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
assert "application/pdf" in mime_types
assert "application/msword" in mime_types
# images should NOT be in the default
assert "image/jpeg" not in mime_types
assert ".jpg" not in extensions
def test_get_allowed_types_for_all_categories(self):
"""Test that ALL_CATEGORIES includes image MIME types."""
mime_types, extensions = get_allowed_types_for_categories(ALL_CATEGORIES)
assert "image/jpeg" in mime_types
assert ".jpg" in extensions
assert "application/pdf" in mime_types
def test_get_allowed_types_returns_frozensets(self):
"""Test that returned sets are frozensets."""
mime_types, extensions = get_allowed_types_for_categories(["pdf"])
assert isinstance(mime_types, frozenset)
assert isinstance(extensions, frozenset)
def test_get_allowed_types_unknown_category_ignored(self):
"""Test that unknown category keys are silently ignored."""
mime_types, extensions = get_allowed_types_for_categories(["pdf", "nonexistent_category"])
assert "application/pdf" in mime_types # 'pdf' still works
# No crash for unknown key
def test_get_allowed_types_empty_list(self):
"""Test empty category list returns empty sets."""
mime_types, extensions = get_allowed_types_for_categories([])
assert mime_types == frozenset()
assert extensions == frozenset()
def test_default_categories_excludes_images(self):
"""Test that DEFAULT_CATEGORIES does not include 'images'."""
assert "images" not in DEFAULT_CATEGORIES
def test_all_categories_includes_images(self):
"""Test that ALL_CATEGORIES includes 'images'."""
assert "images" in ALL_CATEGORIES
def test_all_categories_is_superset_of_default(self):
"""Test that ALL_CATEGORIES contains all DEFAULT_CATEGORIES."""
for cat in DEFAULT_CATEGORIES:
assert cat in ALL_CATEGORIES
@pytest.mark.unit
class TestImapProfilesApiLogic:
"""Tests for ingestion profile validation helpers."""
def test_validate_categories_accepts_valid_keys(self):
"""Test that valid category keys pass validation."""
from app.api.imap_profiles import _validate_categories
result = _validate_categories(["pdf", "office", "images"])
assert set(result) == {"pdf", "office", "images"}
def test_validate_categories_rejects_unknown_key(self):
"""Test that unknown category keys raise 422."""
from fastapi import HTTPException
from app.api.imap_profiles import _validate_categories
with pytest.raises(HTTPException) as exc_info:
_validate_categories(["pdf", "nonexistent"])
assert exc_info.value.status_code == 422
assert "nonexistent" in str(exc_info.value.detail)
def test_validate_categories_deduplicates(self):
"""Test that duplicate category keys are de-duplicated while preserving order."""
from app.api.imap_profiles import _validate_categories
result = _validate_categories(["pdf", "pdf", "office", "pdf"])
assert result == ["pdf", "office"]
def test_to_response_serializes_profile(self, tmp_path):
"""Test _to_response produces expected dict shape."""
from unittest.mock import MagicMock
from app.api.imap_profiles import _to_response
profile = MagicMock()
profile.id = 42
profile.name = "My Profile"
profile.description = "Test description"
profile.owner_id = "user@example.com"
profile.allowed_categories = '["pdf","office"]'
profile.is_builtin = False
profile.created_at = None
profile.updated_at = None
result = _to_response(profile)
assert result["id"] == 42
assert result["name"] == "My Profile"
assert result["allowed_categories"] == ["pdf", "office"]
assert len(result["categories_detail"]) == 2
assert result["categories_detail"][0]["key"] == "pdf"
assert result["is_builtin"] is False
def test_to_response_handles_invalid_categories_json(self):
"""Test _to_response gracefully handles invalid JSON in allowed_categories."""
from unittest.mock import MagicMock
from app.api.imap_profiles import _to_response
profile = MagicMock()
profile.id = 1
profile.name = "Broken"
profile.description = None
profile.owner_id = None
profile.allowed_categories = "this is not valid json {"
profile.is_builtin = True
profile.created_at = None
profile.updated_at = None
result = _to_response(profile)
assert result["allowed_categories"] == []
+19 -16
View File
@@ -24,6 +24,7 @@ from app.tasks.imap_tasks import (
release_lock,
save_processed_emails,
)
from app.utils.allowed_types import ALL_CATEGORIES, DEFAULT_CATEGORIES, get_allowed_types_for_categories
_TEST_CREDENTIAL = "pass" # noqa: S105
@@ -182,27 +183,29 @@ class TestFetchAttachmentsAndEnqueue:
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_skips_image_when_documents_only(self, mock_convert, mock_process):
"""Test that image attachments are skipped with documents_only filter."""
"""Test that image attachments are skipped with the default (documents-only) categories."""
msg = EmailMessage()
msg["Subject"] = "Photo"
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
doc_mime, doc_ext = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
result = fetch_attachments_and_enqueue(msg, effective_mime_types=doc_mime, effective_extensions=doc_ext)
assert result is False
mock_process.delay.assert_not_called()
mock_convert.delay.assert_not_called()
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_processes_image_when_all_filter(self, mock_convert, mock_process, tmp_path):
"""Test that image attachments are processed with 'all' filter."""
def test_processes_image_when_all_categories(self, mock_convert, mock_process, tmp_path):
"""Test that image attachments are processed when images category is included."""
msg = EmailMessage()
msg["Subject"] = "Photo"
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
all_mime, all_ext = get_allowed_types_for_categories(ALL_CATEGORIES)
with patch("app.tasks.imap_tasks.settings") as mock_settings:
mock_settings.workdir = str(tmp_path)
result = fetch_attachments_and_enqueue(msg, attachment_filter="all")
result = fetch_attachments_and_enqueue(msg, effective_mime_types=all_mime, effective_extensions=all_ext)
assert result is True
mock_convert.delay.assert_called_once()
@@ -210,43 +213,43 @@ class TestFetchAttachmentsAndEnqueue:
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_skips_image_with_image_extension_documents_only(self, mock_convert, mock_process):
"""Test image files identified by extension are skipped with documents_only."""
"""Test image files identified by extension are skipped with documents-only categories."""
msg = EmailMessage()
msg["Subject"] = "Screenshot"
msg.add_attachment(b"\x89PNG", maintype="application", subtype="octet-stream", filename="screenshot.png")
result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
doc_mime, doc_ext = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
result = fetch_attachments_and_enqueue(msg, effective_mime_types=doc_mime, effective_extensions=doc_ext)
assert result is False
mock_process.delay.assert_not_called()
mock_convert.delay.assert_not_called()
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_processes_pdf_regardless_of_filter(self, mock_convert, mock_process, tmp_path):
"""Test that PDFs are always processed, even with documents_only filter."""
def test_processes_pdf_regardless_of_categories(self, mock_convert, mock_process, tmp_path):
"""Test that PDFs are always processed (pdf category always included in DEFAULT_CATEGORIES)."""
msg = EmailMessage()
msg["Subject"] = "Invoice"
msg.add_attachment(b"%PDF-1.4", maintype="application", subtype="pdf", filename="invoice.pdf")
doc_mime, doc_ext = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
with patch("app.tasks.imap_tasks.settings") as mock_settings:
mock_settings.workdir = str(tmp_path)
result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
result = fetch_attachments_and_enqueue(msg, effective_mime_types=doc_mime, effective_extensions=doc_ext)
assert result is True
mock_process.delay.assert_called_once()
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_uses_global_setting_when_no_filter(self, mock_convert, mock_process):
"""Test that the global settings.imap_attachment_filter is used when no filter is passed."""
def test_uses_default_categories_when_no_types_provided(self, mock_convert, mock_process):
"""Test that images are skipped when no effective_mime_types / extensions are passed (defaults to DEFAULT_CATEGORIES)."""
msg = EmailMessage()
msg["Subject"] = "Photo"
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
with patch("app.tasks.imap_tasks.settings") as mock_settings:
mock_settings.imap_attachment_filter = "documents_only"
mock_settings.workdir = "/tmp"
result = fetch_attachments_and_enqueue(msg)
# No effective_mime_types passed → function defaults to DEFAULT_CATEGORIES (no images)
result = fetch_attachments_and_enqueue(msg)
assert result is False
mock_process.delay.assert_not_called()