feat(imap): add ImapIngestionProfile model, API, migration and UI
Replaces the simple binary attachment_filter string with a full ingestion profiles system: - Add FILE_TYPE_CATEGORIES dict to allowed_types.py (6 categories: pdf, office, opendocument, text, web, images) + DEFAULT_CATEGORIES / ALL_CATEGORIES + get_allowed_types_for_categories() helper - Add ImapIngestionProfile model (id, name, description, owner_id, allowed_categories JSON, is_builtin) - Update UserImapAccount: replace attachment_filter string with profile_id FK to imap_ingestion_profiles - Migration 033: creates profiles table, seeds 2 built-in profiles (Documents Only, All Files), migrates attachment_filter → profile_id - New /api/imap-profiles/ CRUD endpoints (list, create, get, update, delete) with category validation - Register imap_profiles router in app/api/__init__.py - Update imap_tasks.py: replace attachment_filter string param with profile-based allowed_categories; add _resolve_categories_for_profile() - Update imap_accounts.py API to use profile_id instead of attachment_filter - Update imap_accounts view to pass profiles + categories to template - Full UI overhaul: profiles panel + profile create/edit modal with category checkboxes; profile selector in account modal - 17 new tests (141 total), all passing Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -21,6 +21,7 @@ from app.api.files import router as files_router
|
||||
from app.api.google_drive import router as google_drive_router
|
||||
from app.api.i18n import router as i18n_router
|
||||
from app.api.imap_accounts import router as imap_accounts_router
|
||||
from app.api.imap_profiles import router as imap_profiles_router
|
||||
from app.api.integrations import router as integrations_router
|
||||
from app.api.logs import router as logs_router
|
||||
from app.api.mobile import router as mobile_router
|
||||
@@ -83,6 +84,7 @@ router.include_router(onboarding_router)
|
||||
router.include_router(billing_router)
|
||||
router.include_router(pipelines_router)
|
||||
router.include_router(imap_accounts_router)
|
||||
router.include_router(imap_profiles_router)
|
||||
router.include_router(integrations_router)
|
||||
router.include_router(notifications_router)
|
||||
router.include_router(scheduled_jobs_router)
|
||||
|
||||
+11
-17
@@ -110,12 +110,10 @@ class ImapAccountCreate(BaseModel):
|
||||
use_ssl: bool = Field(default=True, description="Use SSL/TLS connection")
|
||||
delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing")
|
||||
is_active: bool = Field(default=True, description="Whether to poll this mailbox")
|
||||
attachment_filter: str | None = Field(
|
||||
profile_id: int | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Controls which attachment types to ingest. "
|
||||
"'documents_only' – PDFs and office files only (default when None). "
|
||||
"'all' – all supported types including images. "
|
||||
"ID of the ImapIngestionProfile that controls which attachment types to ingest. "
|
||||
"Null inherits the global imap_attachment_filter setting."
|
||||
),
|
||||
)
|
||||
@@ -132,13 +130,11 @@ class ImapAccountUpdate(BaseModel):
|
||||
use_ssl: bool | None = None
|
||||
delete_after_process: bool | None = None
|
||||
is_active: bool | None = None
|
||||
attachment_filter: str | None = Field(
|
||||
profile_id: int | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Controls which attachment types to ingest. "
|
||||
"'documents_only' – PDFs and office files only. "
|
||||
"'all' – all supported types including images. "
|
||||
"Null or empty string clears the override (inherits global setting)."
|
||||
"ID of the ImapIngestionProfile to use. "
|
||||
"Explicitly sending null clears the override (falls back to global setting)."
|
||||
),
|
||||
)
|
||||
|
||||
@@ -173,7 +169,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]:
|
||||
"use_ssl": acct.use_ssl,
|
||||
"delete_after_process": acct.delete_after_process,
|
||||
"is_active": acct.is_active,
|
||||
"attachment_filter": acct.attachment_filter,
|
||||
"profile_id": acct.profile_id,
|
||||
"last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None,
|
||||
"last_error": acct.last_error,
|
||||
"created_at": acct.created_at.isoformat() if acct.created_at else None,
|
||||
@@ -241,7 +237,7 @@ def create_imap_account(
|
||||
use_ssl=body.use_ssl,
|
||||
delete_after_process=body.delete_after_process,
|
||||
is_active=body.is_active,
|
||||
attachment_filter=body.attachment_filter or None,
|
||||
profile_id=body.profile_id,
|
||||
)
|
||||
try:
|
||||
db.add(acct)
|
||||
@@ -297,12 +293,10 @@ def update_imap_account(
|
||||
acct.delete_after_process = body.delete_after_process
|
||||
if body.is_active is not None:
|
||||
acct.is_active = body.is_active
|
||||
# attachment_filter uses a sentinel check: the field is always present in the
|
||||
# model (defaulting to None in Pydantic) so we update it unconditionally when
|
||||
# the caller sends any value (including explicit null to clear the override).
|
||||
# An empty string is normalised to None to avoid storing a non-meaningful value.
|
||||
if "attachment_filter" in body.model_fields_set:
|
||||
acct.attachment_filter = body.attachment_filter or None
|
||||
# profile_id: update whenever the field is explicitly present in the request payload
|
||||
# (including sending null to clear the override).
|
||||
if "profile_id" in body.model_fields_set:
|
||||
acct.profile_id = body.profile_id
|
||||
|
||||
# Reset last_error so the next poll gives a fresh result
|
||||
acct.last_error = None
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
"""API endpoints for managing IMAP ingestion profiles.
|
||||
|
||||
Ingestion profiles allow fine-grained control over which attachment types are
|
||||
accepted when ingesting emails via IMAP. Each profile carries a list of enabled
|
||||
file-type categories (e.g. ``["pdf", "office", "images"]``) drawn from the
|
||||
canonical set defined in :mod:`app.utils.allowed_types`.
|
||||
|
||||
Built-in system profiles (``is_builtin=True``) are read-only and cannot be
|
||||
deleted or modified. Users may create their own profiles which are private to
|
||||
their ``owner_id``. System-level global profiles (``owner_id=None``) are visible
|
||||
to all users but can only be created by administrators.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import ImapIngestionProfile
|
||||
from app.utils.allowed_types import FILE_TYPE_CATEGORIES
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/imap-profiles", tags=["imap-profiles"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_owner_id(request: Request) -> str:
|
||||
"""Return the current user's owner ID, raising 401 if unauthenticated."""
|
||||
owner_id = get_current_owner_id(request)
|
||||
if owner_id is None:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
return owner_id
|
||||
|
||||
|
||||
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_VALID_CATEGORIES = set(FILE_TYPE_CATEGORIES.keys())
|
||||
|
||||
|
||||
class ImapProfileCreate(BaseModel):
|
||||
"""Schema for creating a new ingestion profile."""
|
||||
|
||||
name: str = Field(..., min_length=1, max_length=255, description="Human-readable profile name")
|
||||
description: str | None = Field(default=None, description="Optional description")
|
||||
allowed_categories: list[str] = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description=(f"List of enabled file-type category keys. Valid values: {sorted(_VALID_CATEGORIES)}"),
|
||||
)
|
||||
|
||||
|
||||
class ImapProfileUpdate(BaseModel):
|
||||
"""Schema for updating an existing profile (all fields optional)."""
|
||||
|
||||
name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
description: str | None = None
|
||||
allowed_categories: list[str] | None = Field(default=None, min_length=1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _validate_categories(categories: list[str]) -> list[str]:
|
||||
"""Raise 422 if any category key is unknown; return the cleaned list."""
|
||||
unknown = [c for c in categories if c not in _VALID_CATEGORIES]
|
||||
if unknown:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"Unknown category key(s): {unknown}. Valid keys: {sorted(_VALID_CATEGORIES)}",
|
||||
)
|
||||
# Deduplicate while preserving order
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for cat in categories:
|
||||
if cat not in seen:
|
||||
seen.add(cat)
|
||||
result.append(cat)
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Serialisation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _to_response(profile: ImapIngestionProfile) -> dict[str, Any]:
|
||||
"""Serialize a profile row to a response dict."""
|
||||
try:
|
||||
categories = json.loads(profile.allowed_categories)
|
||||
except (ValueError, TypeError):
|
||||
categories = []
|
||||
|
||||
# Enrich categories with display metadata
|
||||
categories_detail = [
|
||||
{
|
||||
"key": cat,
|
||||
"label": FILE_TYPE_CATEGORIES[cat]["label"] if cat in FILE_TYPE_CATEGORIES else cat,
|
||||
"description": FILE_TYPE_CATEGORIES[cat]["description"] if cat in FILE_TYPE_CATEGORIES else "",
|
||||
}
|
||||
for cat in categories
|
||||
]
|
||||
|
||||
return {
|
||||
"id": profile.id,
|
||||
"name": profile.name,
|
||||
"description": profile.description,
|
||||
"owner_id": profile.owner_id,
|
||||
"allowed_categories": categories,
|
||||
"categories_detail": categories_detail,
|
||||
"is_builtin": profile.is_builtin,
|
||||
"created_at": profile.created_at.isoformat() if profile.created_at else None,
|
||||
"updated_at": profile.updated_at.isoformat() if profile.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/categories", summary="List available file-type categories")
|
||||
def list_categories(request: Request, owner_id: CurrentOwner) -> list[dict[str, Any]]:
|
||||
"""Return the full list of file-type categories that can be used in profiles."""
|
||||
return [
|
||||
{
|
||||
"key": key,
|
||||
"label": info["label"],
|
||||
"description": info["description"],
|
||||
}
|
||||
for key, info in FILE_TYPE_CATEGORIES.items()
|
||||
]
|
||||
|
||||
|
||||
@router.get("/", summary="List ingestion profiles visible to the current user")
|
||||
def list_profiles(request: Request, db: DbSession, owner_id: CurrentOwner) -> list[dict[str, Any]]:
|
||||
"""Return all profiles: system-global (owner_id=NULL) and the user's own profiles."""
|
||||
profiles = (
|
||||
db.query(ImapIngestionProfile)
|
||||
.filter(
|
||||
(ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711
|
||||
)
|
||||
.order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
|
||||
.all()
|
||||
)
|
||||
return [_to_response(p) for p in profiles]
|
||||
|
||||
|
||||
@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new ingestion profile")
|
||||
def create_profile(request: Request, body: ImapProfileCreate, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
|
||||
"""Create a new ingestion profile owned by the current user."""
|
||||
categories = _validate_categories(body.allowed_categories)
|
||||
|
||||
profile = ImapIngestionProfile(
|
||||
name=body.name,
|
||||
description=body.description,
|
||||
owner_id=owner_id,
|
||||
allowed_categories=json.dumps(categories),
|
||||
is_builtin=False,
|
||||
)
|
||||
try:
|
||||
db.add(profile)
|
||||
db.commit()
|
||||
db.refresh(profile)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s created IMAP ingestion profile %d ('%s')", owner_id, profile.id, body.name)
|
||||
return _to_response(profile)
|
||||
|
||||
|
||||
@router.get("/{profile_id}", summary="Get a single ingestion profile")
|
||||
def get_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
|
||||
"""Return a single profile by ID. Only the owner or system profiles are accessible."""
|
||||
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
|
||||
return _to_response(profile)
|
||||
|
||||
|
||||
@router.put("/{profile_id}", summary="Update an ingestion profile")
|
||||
def update_profile(
|
||||
profile_id: int,
|
||||
request: Request,
|
||||
body: ImapProfileUpdate,
|
||||
db: DbSession,
|
||||
owner_id: CurrentOwner,
|
||||
) -> dict[str, Any]:
|
||||
"""Update an existing ingestion profile. Built-in profiles cannot be modified."""
|
||||
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
|
||||
if profile.is_builtin:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Built-in profiles cannot be modified.",
|
||||
)
|
||||
|
||||
if body.name is not None:
|
||||
profile.name = body.name
|
||||
if "description" in body.model_fields_set:
|
||||
profile.description = body.description
|
||||
if body.allowed_categories is not None:
|
||||
categories = _validate_categories(body.allowed_categories)
|
||||
profile.allowed_categories = json.dumps(categories)
|
||||
|
||||
profile.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(profile)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s updated IMAP ingestion profile %d", owner_id, profile_id)
|
||||
return _to_response(profile)
|
||||
|
||||
|
||||
@router.delete("/{profile_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an ingestion profile")
|
||||
def delete_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> None:
|
||||
"""Delete an ingestion profile. Built-in profiles cannot be deleted."""
|
||||
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
|
||||
if profile.is_builtin:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Built-in profiles cannot be deleted.",
|
||||
)
|
||||
|
||||
try:
|
||||
db.delete(profile)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
logger.info("User %s deleted IMAP ingestion profile %d", owner_id, profile_id)
|
||||
+45
-4
@@ -401,6 +401,48 @@ class PipelineStep(Base):
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class ImapIngestionProfile(Base):
|
||||
"""Named ingestion profile controlling which attachment types are accepted from IMAP emails.
|
||||
|
||||
Profiles group file-type categories (e.g. "pdf", "office", "images") so users
|
||||
can precisely control what gets ingested from each mailbox.
|
||||
|
||||
System-provided built-in profiles (``is_builtin=True``) are seeded by the
|
||||
migration and cannot be deleted or renamed. Users may create their own profiles
|
||||
(``owner_id`` set to their identifier) or rely on the global system profiles
|
||||
(``owner_id=None``).
|
||||
|
||||
``allowed_categories`` stores a JSON list of category strings, e.g.::
|
||||
|
||||
'["pdf", "office", "opendocument", "text", "web"]'
|
||||
|
||||
Valid category names are defined in ``app.utils.allowed_types.FILE_TYPE_CATEGORIES``.
|
||||
"""
|
||||
|
||||
__tablename__ = "imap_ingestion_profiles"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
|
||||
# Human-readable profile name (e.g. "Documents Only", "Documents + Images")
|
||||
name = Column(String(255), nullable=False)
|
||||
|
||||
# Optional description shown in the UI
|
||||
description = Column(Text, nullable=True)
|
||||
|
||||
# Owner of this profile. NULL = global/system profile available to all users.
|
||||
owner_id = Column(String, nullable=True, index=True)
|
||||
|
||||
# JSON-encoded list of enabled category keys. Example: '["pdf","office","text"]'
|
||||
# See FILE_TYPE_CATEGORIES in app/utils/allowed_types.py for valid values.
|
||||
allowed_categories = Column(Text, nullable=False, default='["pdf","office","opendocument","text","web"]')
|
||||
|
||||
# Built-in system profiles that cannot be deleted or modified via the API.
|
||||
is_builtin = Column(Boolean, nullable=False, default=False)
|
||||
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class UserImapAccount(Base):
|
||||
"""Per-user IMAP ingestion account.
|
||||
|
||||
@@ -439,10 +481,9 @@ class UserImapAccount(Base):
|
||||
# When True, emails are deleted from the mailbox after their attachments are processed
|
||||
delete_after_process = Column(Boolean, nullable=False, default=False)
|
||||
|
||||
# Override for which attachment types to ingest.
|
||||
# NULL means "inherit the global imap_attachment_filter setting".
|
||||
# Allowed values: 'documents_only', 'all'
|
||||
attachment_filter = Column(String(50), nullable=True, default=None)
|
||||
# Optional reference to an ImapIngestionProfile.
|
||||
# NULL means "use the global imap_attachment_filter setting" (system default).
|
||||
profile_id = Column(Integer, ForeignKey("imap_ingestion_profiles.id"), nullable=True)
|
||||
|
||||
# When False the account is not polled by the periodic task (but not deleted)
|
||||
is_active = Column(Boolean, nullable=False, default=True)
|
||||
|
||||
+79
-71
@@ -13,7 +13,11 @@ from celery import shared_task
|
||||
from app.config import settings
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
|
||||
from app.tasks.process_document import process_document # Updated import
|
||||
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, DOCUMENT_MIME_TYPES, IMAGE_MIME_TYPES
|
||||
from app.utils.allowed_types import (
|
||||
ALL_CATEGORIES,
|
||||
DEFAULT_CATEGORIES,
|
||||
get_allowed_types_for_categories,
|
||||
)
|
||||
|
||||
# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
|
||||
_db_session_factory = None
|
||||
@@ -50,6 +54,33 @@ def _decrypt_imap_password(password: str | None) -> str | None:
|
||||
return decrypt_value(password)
|
||||
|
||||
|
||||
def _resolve_categories_for_profile(profile_id: int | None) -> list[str]:
|
||||
"""Return the list of allowed categories for a profile ID.
|
||||
|
||||
Loads the profile from the database. If ``profile_id`` is ``None`` or the
|
||||
profile is not found, falls back to the global ``settings.imap_attachment_filter``
|
||||
string (``'documents_only'`` → default categories; ``'all'`` → all categories).
|
||||
"""
|
||||
if profile_id is not None:
|
||||
try:
|
||||
from app.models import ImapIngestionProfile
|
||||
|
||||
db = _get_db_session()
|
||||
try:
|
||||
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||
if profile:
|
||||
return json.loads(profile.allowed_categories)
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Could not load IMAP ingestion profile %d: %s — using global default", profile_id, exc)
|
||||
|
||||
# Fall back to global setting
|
||||
if settings.imap_attachment_filter == "all":
|
||||
return ALL_CATEGORIES
|
||||
return DEFAULT_CATEGORIES
|
||||
|
||||
|
||||
LOCK_KEY = "imap_lock" # Unique key for locking
|
||||
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
|
||||
|
||||
@@ -180,7 +211,7 @@ def _pull_user_imap_accounts() -> None:
|
||||
use_ssl=acct.use_ssl,
|
||||
delete_after_process=acct.delete_after_process,
|
||||
owner_id=acct.owner_id,
|
||||
attachment_filter=acct.attachment_filter or settings.imap_attachment_filter,
|
||||
allowed_categories=_resolve_categories_for_profile(acct.profile_id),
|
||||
)
|
||||
# Record successful poll
|
||||
acct.last_checked_at = datetime.now(timezone.utc)
|
||||
@@ -251,7 +282,9 @@ def _pull_user_integration_imap() -> None:
|
||||
use_ssl = cfg.get("use_ssl", True)
|
||||
delete_after = cfg.get("delete_after_process", False)
|
||||
gmail_labels = cfg.get("gmail_apply_labels", True)
|
||||
attachment_filter = cfg.get("attachment_filter") or settings.imap_attachment_filter
|
||||
# Integrations can store a profile_id in config; fall back to global default
|
||||
profile_id = cfg.get("profile_id")
|
||||
allowed_categories = _resolve_categories_for_profile(profile_id)
|
||||
|
||||
if not (host and username and password):
|
||||
logger.warning(
|
||||
@@ -271,7 +304,7 @@ def _pull_user_integration_imap() -> None:
|
||||
delete_after_process=delete_after,
|
||||
owner_id=integ.owner_id,
|
||||
gmail_apply_labels=gmail_labels,
|
||||
attachment_filter=attachment_filter,
|
||||
allowed_categories=allowed_categories,
|
||||
)
|
||||
integ.last_used_at = datetime.now(timezone.utc)
|
||||
integ.last_error = None
|
||||
@@ -332,7 +365,7 @@ def pull_inbox(
|
||||
delete_after_process,
|
||||
owner_id=None,
|
||||
gmail_apply_labels=True,
|
||||
attachment_filter=None,
|
||||
allowed_categories=None,
|
||||
):
|
||||
"""
|
||||
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
|
||||
@@ -349,13 +382,22 @@ def pull_inbox(
|
||||
attributed to this user via ``process_document`` / ``convert_to_pdf``.
|
||||
gmail_apply_labels: Whether to apply Gmail-specific labels and stars to
|
||||
processed emails. Only relevant for Gmail hosts. Defaults to True.
|
||||
attachment_filter: Controls which attachment types to ingest.
|
||||
``'documents_only'`` (default) – PDFs and office files only.
|
||||
``'all'`` – all supported types including images.
|
||||
``None`` falls back to the global ``settings.imap_attachment_filter``.
|
||||
allowed_categories: List of file-type category keys to ingest (e.g.
|
||||
``["pdf", "office", "images"]``). ``None`` falls back to the
|
||||
global ``settings.imap_attachment_filter`` mapping.
|
||||
"""
|
||||
resolved_filter = attachment_filter or settings.imap_attachment_filter
|
||||
logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
|
||||
if allowed_categories is None:
|
||||
allowed_categories = _resolve_categories_for_profile(None)
|
||||
|
||||
effective_mime_types, effective_extensions = get_allowed_types_for_categories(allowed_categories)
|
||||
logger.info(
|
||||
"Connecting to %s at %s:%s (SSL=%s) — categories: %s",
|
||||
mailbox_key,
|
||||
host,
|
||||
port,
|
||||
use_ssl,
|
||||
allowed_categories,
|
||||
)
|
||||
processed_emails = load_processed_emails()
|
||||
|
||||
try:
|
||||
@@ -414,9 +456,13 @@ def pull_inbox(
|
||||
logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
|
||||
continue
|
||||
|
||||
# Process attachments (and convert non-PDF files).
|
||||
# We call the function without assigning its return value since it is not used.
|
||||
fetch_attachments_and_enqueue(email_message, owner_id=owner_id, attachment_filter=resolved_filter)
|
||||
# Process attachments using the resolved mime types / extensions.
|
||||
fetch_attachments_and_enqueue(
|
||||
email_message,
|
||||
owner_id=owner_id,
|
||||
effective_mime_types=effective_mime_types,
|
||||
effective_extensions=effective_extensions,
|
||||
)
|
||||
|
||||
if settings.imap_readonly_mode:
|
||||
logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
|
||||
@@ -448,19 +494,20 @@ def pull_inbox(
|
||||
def fetch_attachments_and_enqueue(
|
||||
email_message,
|
||||
owner_id: str | None = None,
|
||||
attachment_filter: str | None = None,
|
||||
effective_mime_types: frozenset[str] | None = None,
|
||||
effective_extensions: frozenset[str] | None = None,
|
||||
):
|
||||
"""
|
||||
Extracts attachments from the email and processes only allowed file types.
|
||||
|
||||
Files are first checked against the ``attachment_filter`` to determine which
|
||||
broad categories are permitted, then validated against known MIME types /
|
||||
extensions.
|
||||
The caller is responsible for computing ``effective_mime_types`` and
|
||||
``effective_extensions`` from the relevant :class:`ImapIngestionProfile` (or
|
||||
the global default) via :func:`app.utils.allowed_types.get_allowed_types_for_categories`
|
||||
before calling this function. ``pull_inbox`` does this automatically.
|
||||
|
||||
Attachment filter values:
|
||||
- ``'documents_only'`` (default): PDFs, office files (Word, Excel, PowerPoint,
|
||||
OpenDocument, RTF), plain text, CSV, HTML, and Markdown. Images are skipped.
|
||||
- ``'all'``: All supported file types, including images (JPEG, PNG, GIF, etc.).
|
||||
If either set is ``None`` the function falls back to the default category list
|
||||
so the function still works correctly when called directly in tests or from
|
||||
other contexts.
|
||||
|
||||
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
|
||||
any other allowed file is enqueued for conversion to PDF.
|
||||
@@ -469,36 +516,13 @@ def fetch_attachments_and_enqueue(
|
||||
email_message: The parsed email message to extract attachments from.
|
||||
owner_id: Optional user identifier forwarded to ``process_document`` /
|
||||
``convert_to_pdf`` for multi-tenant attribution.
|
||||
attachment_filter: Override for the filter level. Defaults to
|
||||
``settings.imap_attachment_filter`` when not provided.
|
||||
effective_mime_types: Pre-computed frozenset of allowed MIME type strings.
|
||||
effective_extensions: Pre-computed frozenset of allowed file extension strings.
|
||||
|
||||
Returns True if at least one allowed attachment was processed.
|
||||
"""
|
||||
resolved_filter = attachment_filter or settings.imap_attachment_filter
|
||||
|
||||
# Build the effective allowed MIME type set based on the filter
|
||||
if resolved_filter == "all":
|
||||
effective_mime_types = ALLOWED_MIME_TYPES
|
||||
else:
|
||||
# 'documents_only' (and any unrecognised value): exclude images
|
||||
effective_mime_types = DOCUMENT_MIME_TYPES
|
||||
|
||||
# Build the effective allowed extensions set (images excluded for documents_only)
|
||||
if resolved_filter == "all":
|
||||
effective_extensions = ALLOWED_EXTENSIONS
|
||||
else:
|
||||
image_extensions = {
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".png",
|
||||
".gif",
|
||||
".bmp",
|
||||
".tiff",
|
||||
".tif",
|
||||
".webp",
|
||||
".svg",
|
||||
}
|
||||
effective_extensions = ALLOWED_EXTENSIONS - image_extensions
|
||||
if effective_mime_types is None or effective_extensions is None:
|
||||
effective_mime_types, effective_extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
|
||||
|
||||
has_attachment = False
|
||||
for part in email_message.walk():
|
||||
@@ -515,30 +539,14 @@ def fetch_attachments_and_enqueue(
|
||||
mime_type = part.get_content_type()
|
||||
file_ext = os.path.splitext(filename)[1].lower()
|
||||
|
||||
# Skip images when filter is documents_only
|
||||
is_image = mime_type in IMAGE_MIME_TYPES or file_ext in {
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".png",
|
||||
".gif",
|
||||
".bmp",
|
||||
".tiff",
|
||||
".tif",
|
||||
".webp",
|
||||
".svg",
|
||||
}
|
||||
if is_image and resolved_filter != "all":
|
||||
logger.info(
|
||||
"Skipping image attachment %s (MIME: %s) — attachment_filter=%s",
|
||||
filename,
|
||||
mime_type,
|
||||
resolved_filter,
|
||||
)
|
||||
continue
|
||||
|
||||
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
|
||||
if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension:
|
||||
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
|
||||
logger.info(
|
||||
"Skipping attachment %s (MIME: %s, ext: %s) — not in effective allowed set",
|
||||
filename,
|
||||
mime_type,
|
||||
file_ext,
|
||||
)
|
||||
continue
|
||||
|
||||
file_path = os.path.join(settings.workdir, filename)
|
||||
|
||||
@@ -131,3 +131,157 @@ ALLOWED_EXTENSIONS: set[str] = {
|
||||
".md",
|
||||
".markdown",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fine-grained file-type categories used by IMAP ingestion profiles.
|
||||
# Each category groups related MIME types and extensions so that users can
|
||||
# enable/disable a logical collection of formats (e.g. "images") rather than
|
||||
# having to manage individual MIME strings.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FILE_TYPE_CATEGORIES: dict[str, dict] = {
|
||||
"pdf": {
|
||||
"label": "PDF",
|
||||
"description": "PDF documents (.pdf)",
|
||||
"mime_types": frozenset({"application/pdf"}),
|
||||
"extensions": frozenset({".pdf"}),
|
||||
},
|
||||
"office": {
|
||||
"label": "Microsoft Office",
|
||||
"description": "Word, Excel and PowerPoint files (.doc, .docx, .xls, .xlsx, .ppt, .pptx, …)",
|
||||
"mime_types": frozenset(
|
||||
{
|
||||
"application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
|
||||
"application/vnd.ms-word.document.macroEnabled.12",
|
||||
"application/vnd.ms-word.template.macroEnabled.12",
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.template",
|
||||
"application/vnd.ms-excel.sheet.macroEnabled.12",
|
||||
"application/vnd.ms-excel.sheet.binary.macroEnabled.12",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.template",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.slideshow",
|
||||
"application/vnd.ms-powerpoint.presentation.macroEnabled.12",
|
||||
}
|
||||
),
|
||||
"extensions": frozenset(
|
||||
{
|
||||
".doc",
|
||||
".docx",
|
||||
".docm",
|
||||
".dot",
|
||||
".dotx",
|
||||
".dotm",
|
||||
".xls",
|
||||
".xlsx",
|
||||
".xlsm",
|
||||
".xlsb",
|
||||
".xlt",
|
||||
".xltx",
|
||||
".xlw",
|
||||
".ppt",
|
||||
".pptx",
|
||||
".pptm",
|
||||
".pps",
|
||||
".ppsx",
|
||||
".pot",
|
||||
".potx",
|
||||
}
|
||||
),
|
||||
},
|
||||
"opendocument": {
|
||||
"label": "OpenDocument (LibreOffice)",
|
||||
"description": "LibreOffice / OpenOffice files (.odt, .ods, .odp, …)",
|
||||
"mime_types": frozenset(
|
||||
{
|
||||
"application/vnd.oasis.opendocument.text",
|
||||
"application/vnd.oasis.opendocument.spreadsheet",
|
||||
"application/vnd.oasis.opendocument.presentation",
|
||||
"application/vnd.oasis.opendocument.graphics",
|
||||
"application/vnd.oasis.opendocument.formula",
|
||||
}
|
||||
),
|
||||
"extensions": frozenset({".odt", ".ods", ".odp", ".odg", ".odf"}),
|
||||
},
|
||||
"text": {
|
||||
"label": "Text & Data",
|
||||
"description": "Plain text, CSV and RTF files (.txt, .csv, .rtf)",
|
||||
"mime_types": frozenset(
|
||||
{
|
||||
"text/plain",
|
||||
"text/csv",
|
||||
"application/rtf",
|
||||
"text/rtf",
|
||||
}
|
||||
),
|
||||
"extensions": frozenset({".txt", ".csv", ".rtf"}),
|
||||
},
|
||||
"web": {
|
||||
"label": "Web & Markup",
|
||||
"description": "HTML and Markdown files (.html, .htm, .md, .markdown)",
|
||||
"mime_types": frozenset(
|
||||
{
|
||||
"text/html",
|
||||
"text/markdown",
|
||||
"text/x-markdown",
|
||||
}
|
||||
),
|
||||
"extensions": frozenset({".html", ".htm", ".md", ".markdown"}),
|
||||
},
|
||||
"images": {
|
||||
"label": "Images",
|
||||
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)",
|
||||
"mime_types": frozenset(
|
||||
{
|
||||
"image/jpeg",
|
||||
"image/jpg",
|
||||
"image/png",
|
||||
"image/gif",
|
||||
"image/bmp",
|
||||
"image/tiff",
|
||||
"image/webp",
|
||||
"image/svg+xml",
|
||||
}
|
||||
),
|
||||
"extensions": frozenset(
|
||||
{
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".png",
|
||||
".gif",
|
||||
".bmp",
|
||||
".tiff",
|
||||
".tif",
|
||||
".webp",
|
||||
".svg",
|
||||
}
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
# Default categories for the "documents only" built-in profile (no images)
|
||||
DEFAULT_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web"]
|
||||
# All categories including images
|
||||
ALL_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web", "images"]
|
||||
|
||||
|
||||
def get_allowed_types_for_categories(
|
||||
categories: list[str],
|
||||
) -> tuple[frozenset[str], frozenset[str]]:
|
||||
"""Return ``(mime_types, extensions)`` for the given category list.
|
||||
|
||||
Unknown category names are silently ignored so that future categories
|
||||
don't break existing profiles.
|
||||
"""
|
||||
mime_types: set[str] = set()
|
||||
extensions: set[str] = set()
|
||||
for cat in categories:
|
||||
info = FILE_TYPE_CATEGORIES.get(cat)
|
||||
if info:
|
||||
mime_types |= info["mime_types"]
|
||||
extensions |= info["extensions"]
|
||||
return frozenset(mime_types), frozenset(extensions)
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
"""User-facing view for the per-user IMAP ingestion dashboard."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from fastapi import Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import UserImapAccount
|
||||
from app.models import ImapIngestionProfile, UserImapAccount
|
||||
from app.utils.allowed_types import FILE_TYPE_CATEGORIES
|
||||
from app.utils.subscription import get_tier, get_user_tier_id
|
||||
from app.utils.user_scope import get_current_owner_id
|
||||
from app.views.base import APIRouter, Depends, get_db, require_login, templates
|
||||
@@ -25,6 +27,22 @@ def _get_max_mailboxes(tier: dict) -> int | None:
|
||||
return max_mb
|
||||
|
||||
|
||||
def _serialize_profile(profile: ImapIngestionProfile) -> dict:
|
||||
"""Serialize a profile for JSON embedding in the template."""
|
||||
try:
|
||||
categories = json.loads(profile.allowed_categories)
|
||||
except (ValueError, TypeError):
|
||||
categories = []
|
||||
return {
|
||||
"id": profile.id,
|
||||
"name": profile.name,
|
||||
"description": profile.description,
|
||||
"owner_id": profile.owner_id,
|
||||
"allowed_categories": categories,
|
||||
"is_builtin": profile.is_builtin,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/imap-accounts")
|
||||
@require_login
|
||||
async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
|
||||
@@ -49,11 +67,39 @@ async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
|
||||
max_mailboxes = _get_max_mailboxes(tier)
|
||||
can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes)
|
||||
|
||||
# Load ingestion profiles: system-global + user's own
|
||||
profiles: list[ImapIngestionProfile] = (
|
||||
db.query(ImapIngestionProfile)
|
||||
.filter(
|
||||
(ImapIngestionProfile.owner_id == None) # noqa: E711
|
||||
| (ImapIngestionProfile.owner_id == owner_id)
|
||||
)
|
||||
.order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
|
||||
.all()
|
||||
if owner_id
|
||||
else db.query(ImapIngestionProfile)
|
||||
.filter(ImapIngestionProfile.owner_id == None) # noqa: E711
|
||||
.order_by(ImapIngestionProfile.id)
|
||||
.all()
|
||||
)
|
||||
|
||||
# Category definitions for the UI checkbox builder
|
||||
categories = [
|
||||
{
|
||||
"key": key,
|
||||
"label": info["label"],
|
||||
"description": info["description"],
|
||||
}
|
||||
for key, info in FILE_TYPE_CATEGORIES.items()
|
||||
]
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"imap_accounts.html",
|
||||
{
|
||||
"request": request,
|
||||
"accounts": accounts,
|
||||
"profiles": [_serialize_profile(p) for p in profiles],
|
||||
"categories": categories,
|
||||
"current_count": current_count,
|
||||
"max_mailboxes": max_mailboxes,
|
||||
"can_add": can_add,
|
||||
|
||||
Reference in New Issue
Block a user