feat(imap): add ImapIngestionProfile model, API, migration and UI
Replaces the simple binary attachment_filter string with a full ingestion profiles system: - Add FILE_TYPE_CATEGORIES dict to allowed_types.py (6 categories: pdf, office, opendocument, text, web, images) + DEFAULT_CATEGORIES / ALL_CATEGORIES + get_allowed_types_for_categories() helper - Add ImapIngestionProfile model (id, name, description, owner_id, allowed_categories JSON, is_builtin) - Update UserImapAccount: replace attachment_filter string with profile_id FK to imap_ingestion_profiles - Migration 033: creates profiles table, seeds 2 built-in profiles (Documents Only, All Files), migrates attachment_filter → profile_id - New /api/imap-profiles/ CRUD endpoints (list, create, get, update, delete) with category validation - Register imap_profiles router in app/api/__init__.py - Update imap_tasks.py: replace attachment_filter string param with profile-based allowed_categories; add _resolve_categories_for_profile() - Update imap_accounts.py API to use profile_id instead of attachment_filter - Update imap_accounts view to pass profiles + categories to template - Full UI overhaul: profiles panel + profile create/edit modal with category checkboxes; profile selector in account modal - 17 new tests (141 total), all passing Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
@@ -21,6 +21,7 @@ from app.api.files import router as files_router
|
|||||||
from app.api.google_drive import router as google_drive_router
|
from app.api.google_drive import router as google_drive_router
|
||||||
from app.api.i18n import router as i18n_router
|
from app.api.i18n import router as i18n_router
|
||||||
from app.api.imap_accounts import router as imap_accounts_router
|
from app.api.imap_accounts import router as imap_accounts_router
|
||||||
|
from app.api.imap_profiles import router as imap_profiles_router
|
||||||
from app.api.integrations import router as integrations_router
|
from app.api.integrations import router as integrations_router
|
||||||
from app.api.logs import router as logs_router
|
from app.api.logs import router as logs_router
|
||||||
from app.api.mobile import router as mobile_router
|
from app.api.mobile import router as mobile_router
|
||||||
@@ -83,6 +84,7 @@ router.include_router(onboarding_router)
|
|||||||
router.include_router(billing_router)
|
router.include_router(billing_router)
|
||||||
router.include_router(pipelines_router)
|
router.include_router(pipelines_router)
|
||||||
router.include_router(imap_accounts_router)
|
router.include_router(imap_accounts_router)
|
||||||
|
router.include_router(imap_profiles_router)
|
||||||
router.include_router(integrations_router)
|
router.include_router(integrations_router)
|
||||||
router.include_router(notifications_router)
|
router.include_router(notifications_router)
|
||||||
router.include_router(scheduled_jobs_router)
|
router.include_router(scheduled_jobs_router)
|
||||||
|
|||||||
+11
-17
@@ -110,12 +110,10 @@ class ImapAccountCreate(BaseModel):
|
|||||||
use_ssl: bool = Field(default=True, description="Use SSL/TLS connection")
|
use_ssl: bool = Field(default=True, description="Use SSL/TLS connection")
|
||||||
delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing")
|
delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing")
|
||||||
is_active: bool = Field(default=True, description="Whether to poll this mailbox")
|
is_active: bool = Field(default=True, description="Whether to poll this mailbox")
|
||||||
attachment_filter: str | None = Field(
|
profile_id: int | None = Field(
|
||||||
default=None,
|
default=None,
|
||||||
description=(
|
description=(
|
||||||
"Controls which attachment types to ingest. "
|
"ID of the ImapIngestionProfile that controls which attachment types to ingest. "
|
||||||
"'documents_only' – PDFs and office files only (default when None). "
|
|
||||||
"'all' – all supported types including images. "
|
|
||||||
"Null inherits the global imap_attachment_filter setting."
|
"Null inherits the global imap_attachment_filter setting."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -132,13 +130,11 @@ class ImapAccountUpdate(BaseModel):
|
|||||||
use_ssl: bool | None = None
|
use_ssl: bool | None = None
|
||||||
delete_after_process: bool | None = None
|
delete_after_process: bool | None = None
|
||||||
is_active: bool | None = None
|
is_active: bool | None = None
|
||||||
attachment_filter: str | None = Field(
|
profile_id: int | None = Field(
|
||||||
default=None,
|
default=None,
|
||||||
description=(
|
description=(
|
||||||
"Controls which attachment types to ingest. "
|
"ID of the ImapIngestionProfile to use. "
|
||||||
"'documents_only' – PDFs and office files only. "
|
"Explicitly sending null clears the override (falls back to global setting)."
|
||||||
"'all' – all supported types including images. "
|
|
||||||
"Null or empty string clears the override (inherits global setting)."
|
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -173,7 +169,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]:
|
|||||||
"use_ssl": acct.use_ssl,
|
"use_ssl": acct.use_ssl,
|
||||||
"delete_after_process": acct.delete_after_process,
|
"delete_after_process": acct.delete_after_process,
|
||||||
"is_active": acct.is_active,
|
"is_active": acct.is_active,
|
||||||
"attachment_filter": acct.attachment_filter,
|
"profile_id": acct.profile_id,
|
||||||
"last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None,
|
"last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None,
|
||||||
"last_error": acct.last_error,
|
"last_error": acct.last_error,
|
||||||
"created_at": acct.created_at.isoformat() if acct.created_at else None,
|
"created_at": acct.created_at.isoformat() if acct.created_at else None,
|
||||||
@@ -241,7 +237,7 @@ def create_imap_account(
|
|||||||
use_ssl=body.use_ssl,
|
use_ssl=body.use_ssl,
|
||||||
delete_after_process=body.delete_after_process,
|
delete_after_process=body.delete_after_process,
|
||||||
is_active=body.is_active,
|
is_active=body.is_active,
|
||||||
attachment_filter=body.attachment_filter or None,
|
profile_id=body.profile_id,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
db.add(acct)
|
db.add(acct)
|
||||||
@@ -297,12 +293,10 @@ def update_imap_account(
|
|||||||
acct.delete_after_process = body.delete_after_process
|
acct.delete_after_process = body.delete_after_process
|
||||||
if body.is_active is not None:
|
if body.is_active is not None:
|
||||||
acct.is_active = body.is_active
|
acct.is_active = body.is_active
|
||||||
# attachment_filter uses a sentinel check: the field is always present in the
|
# profile_id: update whenever the field is explicitly present in the request payload
|
||||||
# model (defaulting to None in Pydantic) so we update it unconditionally when
|
# (including sending null to clear the override).
|
||||||
# the caller sends any value (including explicit null to clear the override).
|
if "profile_id" in body.model_fields_set:
|
||||||
# An empty string is normalised to None to avoid storing a non-meaningful value.
|
acct.profile_id = body.profile_id
|
||||||
if "attachment_filter" in body.model_fields_set:
|
|
||||||
acct.attachment_filter = body.attachment_filter or None
|
|
||||||
|
|
||||||
# Reset last_error so the next poll gives a fresh result
|
# Reset last_error so the next poll gives a fresh result
|
||||||
acct.last_error = None
|
acct.last_error = None
|
||||||
|
|||||||
@@ -0,0 +1,256 @@
|
|||||||
|
"""API endpoints for managing IMAP ingestion profiles.
|
||||||
|
|
||||||
|
Ingestion profiles allow fine-grained control over which attachment types are
|
||||||
|
accepted when ingesting emails via IMAP. Each profile carries a list of enabled
|
||||||
|
file-type categories (e.g. ``["pdf", "office", "images"]``) drawn from the
|
||||||
|
canonical set defined in :mod:`app.utils.allowed_types`.
|
||||||
|
|
||||||
|
Built-in system profiles (``is_builtin=True``) are read-only and cannot be
|
||||||
|
deleted or modified. Users may create their own profiles which are private to
|
||||||
|
their ``owner_id``. System-level global profiles (``owner_id=None``) are visible
|
||||||
|
to all users but can only be created by administrators.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Annotated, Any
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.database import get_db
|
||||||
|
from app.models import ImapIngestionProfile
|
||||||
|
from app.utils.allowed_types import FILE_TYPE_CATEGORIES
|
||||||
|
from app.utils.user_scope import get_current_owner_id
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
router = APIRouter(prefix="/imap-profiles", tags=["imap-profiles"])
|
||||||
|
|
||||||
|
DbSession = Annotated[Session, Depends(get_db)]
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Auth helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _get_owner_id(request: Request) -> str:
|
||||||
|
"""Return the current user's owner ID, raising 401 if unauthenticated."""
|
||||||
|
owner_id = get_current_owner_id(request)
|
||||||
|
if owner_id is None:
|
||||||
|
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||||
|
return owner_id
|
||||||
|
|
||||||
|
|
||||||
|
CurrentOwner = Annotated[str, Depends(_get_owner_id)]
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Pydantic schemas
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_VALID_CATEGORIES = set(FILE_TYPE_CATEGORIES.keys())
|
||||||
|
|
||||||
|
|
||||||
|
class ImapProfileCreate(BaseModel):
|
||||||
|
"""Schema for creating a new ingestion profile."""
|
||||||
|
|
||||||
|
name: str = Field(..., min_length=1, max_length=255, description="Human-readable profile name")
|
||||||
|
description: str | None = Field(default=None, description="Optional description")
|
||||||
|
allowed_categories: list[str] = Field(
|
||||||
|
...,
|
||||||
|
min_length=1,
|
||||||
|
description=(f"List of enabled file-type category keys. Valid values: {sorted(_VALID_CATEGORIES)}"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ImapProfileUpdate(BaseModel):
|
||||||
|
"""Schema for updating an existing profile (all fields optional)."""
|
||||||
|
|
||||||
|
name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||||
|
description: str | None = None
|
||||||
|
allowed_categories: list[str] | None = Field(default=None, min_length=1)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Validation helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_categories(categories: list[str]) -> list[str]:
|
||||||
|
"""Raise 422 if any category key is unknown; return the cleaned list."""
|
||||||
|
unknown = [c for c in categories if c not in _VALID_CATEGORIES]
|
||||||
|
if unknown:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||||
|
detail=f"Unknown category key(s): {unknown}. Valid keys: {sorted(_VALID_CATEGORIES)}",
|
||||||
|
)
|
||||||
|
# Deduplicate while preserving order
|
||||||
|
seen: set[str] = set()
|
||||||
|
result: list[str] = []
|
||||||
|
for cat in categories:
|
||||||
|
if cat not in seen:
|
||||||
|
seen.add(cat)
|
||||||
|
result.append(cat)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Serialisation
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _to_response(profile: ImapIngestionProfile) -> dict[str, Any]:
|
||||||
|
"""Serialize a profile row to a response dict."""
|
||||||
|
try:
|
||||||
|
categories = json.loads(profile.allowed_categories)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
categories = []
|
||||||
|
|
||||||
|
# Enrich categories with display metadata
|
||||||
|
categories_detail = [
|
||||||
|
{
|
||||||
|
"key": cat,
|
||||||
|
"label": FILE_TYPE_CATEGORIES[cat]["label"] if cat in FILE_TYPE_CATEGORIES else cat,
|
||||||
|
"description": FILE_TYPE_CATEGORIES[cat]["description"] if cat in FILE_TYPE_CATEGORIES else "",
|
||||||
|
}
|
||||||
|
for cat in categories
|
||||||
|
]
|
||||||
|
|
||||||
|
return {
|
||||||
|
"id": profile.id,
|
||||||
|
"name": profile.name,
|
||||||
|
"description": profile.description,
|
||||||
|
"owner_id": profile.owner_id,
|
||||||
|
"allowed_categories": categories,
|
||||||
|
"categories_detail": categories_detail,
|
||||||
|
"is_builtin": profile.is_builtin,
|
||||||
|
"created_at": profile.created_at.isoformat() if profile.created_at else None,
|
||||||
|
"updated_at": profile.updated_at.isoformat() if profile.updated_at else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Endpoints
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/categories", summary="List available file-type categories")
|
||||||
|
def list_categories(request: Request, owner_id: CurrentOwner) -> list[dict[str, Any]]:
|
||||||
|
"""Return the full list of file-type categories that can be used in profiles."""
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"key": key,
|
||||||
|
"label": info["label"],
|
||||||
|
"description": info["description"],
|
||||||
|
}
|
||||||
|
for key, info in FILE_TYPE_CATEGORIES.items()
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/", summary="List ingestion profiles visible to the current user")
|
||||||
|
def list_profiles(request: Request, db: DbSession, owner_id: CurrentOwner) -> list[dict[str, Any]]:
|
||||||
|
"""Return all profiles: system-global (owner_id=NULL) and the user's own profiles."""
|
||||||
|
profiles = (
|
||||||
|
db.query(ImapIngestionProfile)
|
||||||
|
.filter(
|
||||||
|
(ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711
|
||||||
|
)
|
||||||
|
.order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return [_to_response(p) for p in profiles]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new ingestion profile")
|
||||||
|
def create_profile(request: Request, body: ImapProfileCreate, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
|
||||||
|
"""Create a new ingestion profile owned by the current user."""
|
||||||
|
categories = _validate_categories(body.allowed_categories)
|
||||||
|
|
||||||
|
profile = ImapIngestionProfile(
|
||||||
|
name=body.name,
|
||||||
|
description=body.description,
|
||||||
|
owner_id=owner_id,
|
||||||
|
allowed_categories=json.dumps(categories),
|
||||||
|
is_builtin=False,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
db.add(profile)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(profile)
|
||||||
|
except Exception:
|
||||||
|
db.rollback()
|
||||||
|
raise
|
||||||
|
|
||||||
|
logger.info("User %s created IMAP ingestion profile %d ('%s')", owner_id, profile.id, body.name)
|
||||||
|
return _to_response(profile)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{profile_id}", summary="Get a single ingestion profile")
|
||||||
|
def get_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
|
||||||
|
"""Return a single profile by ID. Only the owner or system profiles are accessible."""
|
||||||
|
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||||
|
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
|
||||||
|
return _to_response(profile)
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/{profile_id}", summary="Update an ingestion profile")
|
||||||
|
def update_profile(
|
||||||
|
profile_id: int,
|
||||||
|
request: Request,
|
||||||
|
body: ImapProfileUpdate,
|
||||||
|
db: DbSession,
|
||||||
|
owner_id: CurrentOwner,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Update an existing ingestion profile. Built-in profiles cannot be modified."""
|
||||||
|
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||||
|
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
|
||||||
|
if profile.is_builtin:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_403_FORBIDDEN,
|
||||||
|
detail="Built-in profiles cannot be modified.",
|
||||||
|
)
|
||||||
|
|
||||||
|
if body.name is not None:
|
||||||
|
profile.name = body.name
|
||||||
|
if "description" in body.model_fields_set:
|
||||||
|
profile.description = body.description
|
||||||
|
if body.allowed_categories is not None:
|
||||||
|
categories = _validate_categories(body.allowed_categories)
|
||||||
|
profile.allowed_categories = json.dumps(categories)
|
||||||
|
|
||||||
|
profile.updated_at = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
try:
|
||||||
|
db.commit()
|
||||||
|
db.refresh(profile)
|
||||||
|
except Exception:
|
||||||
|
db.rollback()
|
||||||
|
raise
|
||||||
|
|
||||||
|
logger.info("User %s updated IMAP ingestion profile %d", owner_id, profile_id)
|
||||||
|
return _to_response(profile)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{profile_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an ingestion profile")
|
||||||
|
def delete_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> None:
|
||||||
|
"""Delete an ingestion profile. Built-in profiles cannot be deleted."""
|
||||||
|
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||||
|
if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
|
||||||
|
if profile.is_builtin:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_403_FORBIDDEN,
|
||||||
|
detail="Built-in profiles cannot be deleted.",
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
db.delete(profile)
|
||||||
|
db.commit()
|
||||||
|
except Exception:
|
||||||
|
db.rollback()
|
||||||
|
raise
|
||||||
|
|
||||||
|
logger.info("User %s deleted IMAP ingestion profile %d", owner_id, profile_id)
|
||||||
+45
-4
@@ -401,6 +401,48 @@ class PipelineStep(Base):
|
|||||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|
||||||
|
|
||||||
|
class ImapIngestionProfile(Base):
|
||||||
|
"""Named ingestion profile controlling which attachment types are accepted from IMAP emails.
|
||||||
|
|
||||||
|
Profiles group file-type categories (e.g. "pdf", "office", "images") so users
|
||||||
|
can precisely control what gets ingested from each mailbox.
|
||||||
|
|
||||||
|
System-provided built-in profiles (``is_builtin=True``) are seeded by the
|
||||||
|
migration and cannot be deleted or renamed. Users may create their own profiles
|
||||||
|
(``owner_id`` set to their identifier) or rely on the global system profiles
|
||||||
|
(``owner_id=None``).
|
||||||
|
|
||||||
|
``allowed_categories`` stores a JSON list of category strings, e.g.::
|
||||||
|
|
||||||
|
'["pdf", "office", "opendocument", "text", "web"]'
|
||||||
|
|
||||||
|
Valid category names are defined in ``app.utils.allowed_types.FILE_TYPE_CATEGORIES``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "imap_ingestion_profiles"
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True, index=True)
|
||||||
|
|
||||||
|
# Human-readable profile name (e.g. "Documents Only", "Documents + Images")
|
||||||
|
name = Column(String(255), nullable=False)
|
||||||
|
|
||||||
|
# Optional description shown in the UI
|
||||||
|
description = Column(Text, nullable=True)
|
||||||
|
|
||||||
|
# Owner of this profile. NULL = global/system profile available to all users.
|
||||||
|
owner_id = Column(String, nullable=True, index=True)
|
||||||
|
|
||||||
|
# JSON-encoded list of enabled category keys. Example: '["pdf","office","text"]'
|
||||||
|
# See FILE_TYPE_CATEGORIES in app/utils/allowed_types.py for valid values.
|
||||||
|
allowed_categories = Column(Text, nullable=False, default='["pdf","office","opendocument","text","web"]')
|
||||||
|
|
||||||
|
# Built-in system profiles that cannot be deleted or modified via the API.
|
||||||
|
is_builtin = Column(Boolean, nullable=False, default=False)
|
||||||
|
|
||||||
|
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|
||||||
|
|
||||||
class UserImapAccount(Base):
|
class UserImapAccount(Base):
|
||||||
"""Per-user IMAP ingestion account.
|
"""Per-user IMAP ingestion account.
|
||||||
|
|
||||||
@@ -439,10 +481,9 @@ class UserImapAccount(Base):
|
|||||||
# When True, emails are deleted from the mailbox after their attachments are processed
|
# When True, emails are deleted from the mailbox after their attachments are processed
|
||||||
delete_after_process = Column(Boolean, nullable=False, default=False)
|
delete_after_process = Column(Boolean, nullable=False, default=False)
|
||||||
|
|
||||||
# Override for which attachment types to ingest.
|
# Optional reference to an ImapIngestionProfile.
|
||||||
# NULL means "inherit the global imap_attachment_filter setting".
|
# NULL means "use the global imap_attachment_filter setting" (system default).
|
||||||
# Allowed values: 'documents_only', 'all'
|
profile_id = Column(Integer, ForeignKey("imap_ingestion_profiles.id"), nullable=True)
|
||||||
attachment_filter = Column(String(50), nullable=True, default=None)
|
|
||||||
|
|
||||||
# When False the account is not polled by the periodic task (but not deleted)
|
# When False the account is not polled by the periodic task (but not deleted)
|
||||||
is_active = Column(Boolean, nullable=False, default=True)
|
is_active = Column(Boolean, nullable=False, default=True)
|
||||||
|
|||||||
+79
-71
@@ -13,7 +13,11 @@ from celery import shared_task
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
|
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
|
||||||
from app.tasks.process_document import process_document # Updated import
|
from app.tasks.process_document import process_document # Updated import
|
||||||
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, DOCUMENT_MIME_TYPES, IMAGE_MIME_TYPES
|
from app.utils.allowed_types import (
|
||||||
|
ALL_CATEGORIES,
|
||||||
|
DEFAULT_CATEGORIES,
|
||||||
|
get_allowed_types_for_categories,
|
||||||
|
)
|
||||||
|
|
||||||
# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
|
# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
|
||||||
_db_session_factory = None
|
_db_session_factory = None
|
||||||
@@ -50,6 +54,33 @@ def _decrypt_imap_password(password: str | None) -> str | None:
|
|||||||
return decrypt_value(password)
|
return decrypt_value(password)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_categories_for_profile(profile_id: int | None) -> list[str]:
|
||||||
|
"""Return the list of allowed categories for a profile ID.
|
||||||
|
|
||||||
|
Loads the profile from the database. If ``profile_id`` is ``None`` or the
|
||||||
|
profile is not found, falls back to the global ``settings.imap_attachment_filter``
|
||||||
|
string (``'documents_only'`` → default categories; ``'all'`` → all categories).
|
||||||
|
"""
|
||||||
|
if profile_id is not None:
|
||||||
|
try:
|
||||||
|
from app.models import ImapIngestionProfile
|
||||||
|
|
||||||
|
db = _get_db_session()
|
||||||
|
try:
|
||||||
|
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||||
|
if profile:
|
||||||
|
return json.loads(profile.allowed_categories)
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
logger.warning("Could not load IMAP ingestion profile %d: %s — using global default", profile_id, exc)
|
||||||
|
|
||||||
|
# Fall back to global setting
|
||||||
|
if settings.imap_attachment_filter == "all":
|
||||||
|
return ALL_CATEGORIES
|
||||||
|
return DEFAULT_CATEGORIES
|
||||||
|
|
||||||
|
|
||||||
LOCK_KEY = "imap_lock" # Unique key for locking
|
LOCK_KEY = "imap_lock" # Unique key for locking
|
||||||
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
|
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
|
||||||
|
|
||||||
@@ -180,7 +211,7 @@ def _pull_user_imap_accounts() -> None:
|
|||||||
use_ssl=acct.use_ssl,
|
use_ssl=acct.use_ssl,
|
||||||
delete_after_process=acct.delete_after_process,
|
delete_after_process=acct.delete_after_process,
|
||||||
owner_id=acct.owner_id,
|
owner_id=acct.owner_id,
|
||||||
attachment_filter=acct.attachment_filter or settings.imap_attachment_filter,
|
allowed_categories=_resolve_categories_for_profile(acct.profile_id),
|
||||||
)
|
)
|
||||||
# Record successful poll
|
# Record successful poll
|
||||||
acct.last_checked_at = datetime.now(timezone.utc)
|
acct.last_checked_at = datetime.now(timezone.utc)
|
||||||
@@ -251,7 +282,9 @@ def _pull_user_integration_imap() -> None:
|
|||||||
use_ssl = cfg.get("use_ssl", True)
|
use_ssl = cfg.get("use_ssl", True)
|
||||||
delete_after = cfg.get("delete_after_process", False)
|
delete_after = cfg.get("delete_after_process", False)
|
||||||
gmail_labels = cfg.get("gmail_apply_labels", True)
|
gmail_labels = cfg.get("gmail_apply_labels", True)
|
||||||
attachment_filter = cfg.get("attachment_filter") or settings.imap_attachment_filter
|
# Integrations can store a profile_id in config; fall back to global default
|
||||||
|
profile_id = cfg.get("profile_id")
|
||||||
|
allowed_categories = _resolve_categories_for_profile(profile_id)
|
||||||
|
|
||||||
if not (host and username and password):
|
if not (host and username and password):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
@@ -271,7 +304,7 @@ def _pull_user_integration_imap() -> None:
|
|||||||
delete_after_process=delete_after,
|
delete_after_process=delete_after,
|
||||||
owner_id=integ.owner_id,
|
owner_id=integ.owner_id,
|
||||||
gmail_apply_labels=gmail_labels,
|
gmail_apply_labels=gmail_labels,
|
||||||
attachment_filter=attachment_filter,
|
allowed_categories=allowed_categories,
|
||||||
)
|
)
|
||||||
integ.last_used_at = datetime.now(timezone.utc)
|
integ.last_used_at = datetime.now(timezone.utc)
|
||||||
integ.last_error = None
|
integ.last_error = None
|
||||||
@@ -332,7 +365,7 @@ def pull_inbox(
|
|||||||
delete_after_process,
|
delete_after_process,
|
||||||
owner_id=None,
|
owner_id=None,
|
||||||
gmail_apply_labels=True,
|
gmail_apply_labels=True,
|
||||||
attachment_filter=None,
|
allowed_categories=None,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
|
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
|
||||||
@@ -349,13 +382,22 @@ def pull_inbox(
|
|||||||
attributed to this user via ``process_document`` / ``convert_to_pdf``.
|
attributed to this user via ``process_document`` / ``convert_to_pdf``.
|
||||||
gmail_apply_labels: Whether to apply Gmail-specific labels and stars to
|
gmail_apply_labels: Whether to apply Gmail-specific labels and stars to
|
||||||
processed emails. Only relevant for Gmail hosts. Defaults to True.
|
processed emails. Only relevant for Gmail hosts. Defaults to True.
|
||||||
attachment_filter: Controls which attachment types to ingest.
|
allowed_categories: List of file-type category keys to ingest (e.g.
|
||||||
``'documents_only'`` (default) – PDFs and office files only.
|
``["pdf", "office", "images"]``). ``None`` falls back to the
|
||||||
``'all'`` – all supported types including images.
|
global ``settings.imap_attachment_filter`` mapping.
|
||||||
``None`` falls back to the global ``settings.imap_attachment_filter``.
|
|
||||||
"""
|
"""
|
||||||
resolved_filter = attachment_filter or settings.imap_attachment_filter
|
if allowed_categories is None:
|
||||||
logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
|
allowed_categories = _resolve_categories_for_profile(None)
|
||||||
|
|
||||||
|
effective_mime_types, effective_extensions = get_allowed_types_for_categories(allowed_categories)
|
||||||
|
logger.info(
|
||||||
|
"Connecting to %s at %s:%s (SSL=%s) — categories: %s",
|
||||||
|
mailbox_key,
|
||||||
|
host,
|
||||||
|
port,
|
||||||
|
use_ssl,
|
||||||
|
allowed_categories,
|
||||||
|
)
|
||||||
processed_emails = load_processed_emails()
|
processed_emails = load_processed_emails()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -414,9 +456,13 @@ def pull_inbox(
|
|||||||
logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
|
logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Process attachments (and convert non-PDF files).
|
# Process attachments using the resolved mime types / extensions.
|
||||||
# We call the function without assigning its return value since it is not used.
|
fetch_attachments_and_enqueue(
|
||||||
fetch_attachments_and_enqueue(email_message, owner_id=owner_id, attachment_filter=resolved_filter)
|
email_message,
|
||||||
|
owner_id=owner_id,
|
||||||
|
effective_mime_types=effective_mime_types,
|
||||||
|
effective_extensions=effective_extensions,
|
||||||
|
)
|
||||||
|
|
||||||
if settings.imap_readonly_mode:
|
if settings.imap_readonly_mode:
|
||||||
logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
|
logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
|
||||||
@@ -448,19 +494,20 @@ def pull_inbox(
|
|||||||
def fetch_attachments_and_enqueue(
|
def fetch_attachments_and_enqueue(
|
||||||
email_message,
|
email_message,
|
||||||
owner_id: str | None = None,
|
owner_id: str | None = None,
|
||||||
attachment_filter: str | None = None,
|
effective_mime_types: frozenset[str] | None = None,
|
||||||
|
effective_extensions: frozenset[str] | None = None,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Extracts attachments from the email and processes only allowed file types.
|
Extracts attachments from the email and processes only allowed file types.
|
||||||
|
|
||||||
Files are first checked against the ``attachment_filter`` to determine which
|
The caller is responsible for computing ``effective_mime_types`` and
|
||||||
broad categories are permitted, then validated against known MIME types /
|
``effective_extensions`` from the relevant :class:`ImapIngestionProfile` (or
|
||||||
extensions.
|
the global default) via :func:`app.utils.allowed_types.get_allowed_types_for_categories`
|
||||||
|
before calling this function. ``pull_inbox`` does this automatically.
|
||||||
|
|
||||||
Attachment filter values:
|
If either set is ``None`` the function falls back to the default category list
|
||||||
- ``'documents_only'`` (default): PDFs, office files (Word, Excel, PowerPoint,
|
so the function still works correctly when called directly in tests or from
|
||||||
OpenDocument, RTF), plain text, CSV, HTML, and Markdown. Images are skipped.
|
other contexts.
|
||||||
- ``'all'``: All supported file types, including images (JPEG, PNG, GIF, etc.).
|
|
||||||
|
|
||||||
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
|
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
|
||||||
any other allowed file is enqueued for conversion to PDF.
|
any other allowed file is enqueued for conversion to PDF.
|
||||||
@@ -469,36 +516,13 @@ def fetch_attachments_and_enqueue(
|
|||||||
email_message: The parsed email message to extract attachments from.
|
email_message: The parsed email message to extract attachments from.
|
||||||
owner_id: Optional user identifier forwarded to ``process_document`` /
|
owner_id: Optional user identifier forwarded to ``process_document`` /
|
||||||
``convert_to_pdf`` for multi-tenant attribution.
|
``convert_to_pdf`` for multi-tenant attribution.
|
||||||
attachment_filter: Override for the filter level. Defaults to
|
effective_mime_types: Pre-computed frozenset of allowed MIME type strings.
|
||||||
``settings.imap_attachment_filter`` when not provided.
|
effective_extensions: Pre-computed frozenset of allowed file extension strings.
|
||||||
|
|
||||||
Returns True if at least one allowed attachment was processed.
|
Returns True if at least one allowed attachment was processed.
|
||||||
"""
|
"""
|
||||||
resolved_filter = attachment_filter or settings.imap_attachment_filter
|
if effective_mime_types is None or effective_extensions is None:
|
||||||
|
effective_mime_types, effective_extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
|
||||||
# Build the effective allowed MIME type set based on the filter
|
|
||||||
if resolved_filter == "all":
|
|
||||||
effective_mime_types = ALLOWED_MIME_TYPES
|
|
||||||
else:
|
|
||||||
# 'documents_only' (and any unrecognised value): exclude images
|
|
||||||
effective_mime_types = DOCUMENT_MIME_TYPES
|
|
||||||
|
|
||||||
# Build the effective allowed extensions set (images excluded for documents_only)
|
|
||||||
if resolved_filter == "all":
|
|
||||||
effective_extensions = ALLOWED_EXTENSIONS
|
|
||||||
else:
|
|
||||||
image_extensions = {
|
|
||||||
".jpg",
|
|
||||||
".jpeg",
|
|
||||||
".png",
|
|
||||||
".gif",
|
|
||||||
".bmp",
|
|
||||||
".tiff",
|
|
||||||
".tif",
|
|
||||||
".webp",
|
|
||||||
".svg",
|
|
||||||
}
|
|
||||||
effective_extensions = ALLOWED_EXTENSIONS - image_extensions
|
|
||||||
|
|
||||||
has_attachment = False
|
has_attachment = False
|
||||||
for part in email_message.walk():
|
for part in email_message.walk():
|
||||||
@@ -515,30 +539,14 @@ def fetch_attachments_and_enqueue(
|
|||||||
mime_type = part.get_content_type()
|
mime_type = part.get_content_type()
|
||||||
file_ext = os.path.splitext(filename)[1].lower()
|
file_ext = os.path.splitext(filename)[1].lower()
|
||||||
|
|
||||||
# Skip images when filter is documents_only
|
|
||||||
is_image = mime_type in IMAGE_MIME_TYPES or file_ext in {
|
|
||||||
".jpg",
|
|
||||||
".jpeg",
|
|
||||||
".png",
|
|
||||||
".gif",
|
|
||||||
".bmp",
|
|
||||||
".tiff",
|
|
||||||
".tif",
|
|
||||||
".webp",
|
|
||||||
".svg",
|
|
||||||
}
|
|
||||||
if is_image and resolved_filter != "all":
|
|
||||||
logger.info(
|
|
||||||
"Skipping image attachment %s (MIME: %s) — attachment_filter=%s",
|
|
||||||
filename,
|
|
||||||
mime_type,
|
|
||||||
resolved_filter,
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
|
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
|
||||||
if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension:
|
if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension:
|
||||||
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
|
logger.info(
|
||||||
|
"Skipping attachment %s (MIME: %s, ext: %s) — not in effective allowed set",
|
||||||
|
filename,
|
||||||
|
mime_type,
|
||||||
|
file_ext,
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
file_path = os.path.join(settings.workdir, filename)
|
file_path = os.path.join(settings.workdir, filename)
|
||||||
|
|||||||
@@ -131,3 +131,157 @@ ALLOWED_EXTENSIONS: set[str] = {
|
|||||||
".md",
|
".md",
|
||||||
".markdown",
|
".markdown",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Fine-grained file-type categories used by IMAP ingestion profiles.
|
||||||
|
# Each category groups related MIME types and extensions so that users can
|
||||||
|
# enable/disable a logical collection of formats (e.g. "images") rather than
|
||||||
|
# having to manage individual MIME strings.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
FILE_TYPE_CATEGORIES: dict[str, dict] = {
|
||||||
|
"pdf": {
|
||||||
|
"label": "PDF",
|
||||||
|
"description": "PDF documents (.pdf)",
|
||||||
|
"mime_types": frozenset({"application/pdf"}),
|
||||||
|
"extensions": frozenset({".pdf"}),
|
||||||
|
},
|
||||||
|
"office": {
|
||||||
|
"label": "Microsoft Office",
|
||||||
|
"description": "Word, Excel and PowerPoint files (.doc, .docx, .xls, .xlsx, .ppt, .pptx, …)",
|
||||||
|
"mime_types": frozenset(
|
||||||
|
{
|
||||||
|
"application/msword",
|
||||||
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||||
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
|
||||||
|
"application/vnd.ms-word.document.macroEnabled.12",
|
||||||
|
"application/vnd.ms-word.template.macroEnabled.12",
|
||||||
|
"application/vnd.ms-excel",
|
||||||
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||||
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.template",
|
||||||
|
"application/vnd.ms-excel.sheet.macroEnabled.12",
|
||||||
|
"application/vnd.ms-excel.sheet.binary.macroEnabled.12",
|
||||||
|
"application/vnd.ms-powerpoint",
|
||||||
|
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||||
|
"application/vnd.openxmlformats-officedocument.presentationml.template",
|
||||||
|
"application/vnd.openxmlformats-officedocument.presentationml.slideshow",
|
||||||
|
"application/vnd.ms-powerpoint.presentation.macroEnabled.12",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
"extensions": frozenset(
|
||||||
|
{
|
||||||
|
".doc",
|
||||||
|
".docx",
|
||||||
|
".docm",
|
||||||
|
".dot",
|
||||||
|
".dotx",
|
||||||
|
".dotm",
|
||||||
|
".xls",
|
||||||
|
".xlsx",
|
||||||
|
".xlsm",
|
||||||
|
".xlsb",
|
||||||
|
".xlt",
|
||||||
|
".xltx",
|
||||||
|
".xlw",
|
||||||
|
".ppt",
|
||||||
|
".pptx",
|
||||||
|
".pptm",
|
||||||
|
".pps",
|
||||||
|
".ppsx",
|
||||||
|
".pot",
|
||||||
|
".potx",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"opendocument": {
|
||||||
|
"label": "OpenDocument (LibreOffice)",
|
||||||
|
"description": "LibreOffice / OpenOffice files (.odt, .ods, .odp, …)",
|
||||||
|
"mime_types": frozenset(
|
||||||
|
{
|
||||||
|
"application/vnd.oasis.opendocument.text",
|
||||||
|
"application/vnd.oasis.opendocument.spreadsheet",
|
||||||
|
"application/vnd.oasis.opendocument.presentation",
|
||||||
|
"application/vnd.oasis.opendocument.graphics",
|
||||||
|
"application/vnd.oasis.opendocument.formula",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
"extensions": frozenset({".odt", ".ods", ".odp", ".odg", ".odf"}),
|
||||||
|
},
|
||||||
|
"text": {
|
||||||
|
"label": "Text & Data",
|
||||||
|
"description": "Plain text, CSV and RTF files (.txt, .csv, .rtf)",
|
||||||
|
"mime_types": frozenset(
|
||||||
|
{
|
||||||
|
"text/plain",
|
||||||
|
"text/csv",
|
||||||
|
"application/rtf",
|
||||||
|
"text/rtf",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
"extensions": frozenset({".txt", ".csv", ".rtf"}),
|
||||||
|
},
|
||||||
|
"web": {
|
||||||
|
"label": "Web & Markup",
|
||||||
|
"description": "HTML and Markdown files (.html, .htm, .md, .markdown)",
|
||||||
|
"mime_types": frozenset(
|
||||||
|
{
|
||||||
|
"text/html",
|
||||||
|
"text/markdown",
|
||||||
|
"text/x-markdown",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
"extensions": frozenset({".html", ".htm", ".md", ".markdown"}),
|
||||||
|
},
|
||||||
|
"images": {
|
||||||
|
"label": "Images",
|
||||||
|
"description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)",
|
||||||
|
"mime_types": frozenset(
|
||||||
|
{
|
||||||
|
"image/jpeg",
|
||||||
|
"image/jpg",
|
||||||
|
"image/png",
|
||||||
|
"image/gif",
|
||||||
|
"image/bmp",
|
||||||
|
"image/tiff",
|
||||||
|
"image/webp",
|
||||||
|
"image/svg+xml",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
"extensions": frozenset(
|
||||||
|
{
|
||||||
|
".jpg",
|
||||||
|
".jpeg",
|
||||||
|
".png",
|
||||||
|
".gif",
|
||||||
|
".bmp",
|
||||||
|
".tiff",
|
||||||
|
".tif",
|
||||||
|
".webp",
|
||||||
|
".svg",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
# Default categories for the "documents only" built-in profile (no images)
|
||||||
|
DEFAULT_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web"]
|
||||||
|
# All categories including images
|
||||||
|
ALL_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web", "images"]
|
||||||
|
|
||||||
|
|
||||||
|
def get_allowed_types_for_categories(
|
||||||
|
categories: list[str],
|
||||||
|
) -> tuple[frozenset[str], frozenset[str]]:
|
||||||
|
"""Return ``(mime_types, extensions)`` for the given category list.
|
||||||
|
|
||||||
|
Unknown category names are silently ignored so that future categories
|
||||||
|
don't break existing profiles.
|
||||||
|
"""
|
||||||
|
mime_types: set[str] = set()
|
||||||
|
extensions: set[str] = set()
|
||||||
|
for cat in categories:
|
||||||
|
info = FILE_TYPE_CATEGORIES.get(cat)
|
||||||
|
if info:
|
||||||
|
mime_types |= info["mime_types"]
|
||||||
|
extensions |= info["extensions"]
|
||||||
|
return frozenset(mime_types), frozenset(extensions)
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
"""User-facing view for the per-user IMAP ingestion dashboard."""
|
"""User-facing view for the per-user IMAP ingestion dashboard."""
|
||||||
|
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.models import UserImapAccount
|
from app.models import ImapIngestionProfile, UserImapAccount
|
||||||
|
from app.utils.allowed_types import FILE_TYPE_CATEGORIES
|
||||||
from app.utils.subscription import get_tier, get_user_tier_id
|
from app.utils.subscription import get_tier, get_user_tier_id
|
||||||
from app.utils.user_scope import get_current_owner_id
|
from app.utils.user_scope import get_current_owner_id
|
||||||
from app.views.base import APIRouter, Depends, get_db, require_login, templates
|
from app.views.base import APIRouter, Depends, get_db, require_login, templates
|
||||||
@@ -25,6 +27,22 @@ def _get_max_mailboxes(tier: dict) -> int | None:
|
|||||||
return max_mb
|
return max_mb
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize_profile(profile: ImapIngestionProfile) -> dict:
|
||||||
|
"""Serialize a profile for JSON embedding in the template."""
|
||||||
|
try:
|
||||||
|
categories = json.loads(profile.allowed_categories)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
categories = []
|
||||||
|
return {
|
||||||
|
"id": profile.id,
|
||||||
|
"name": profile.name,
|
||||||
|
"description": profile.description,
|
||||||
|
"owner_id": profile.owner_id,
|
||||||
|
"allowed_categories": categories,
|
||||||
|
"is_builtin": profile.is_builtin,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/imap-accounts")
|
@router.get("/imap-accounts")
|
||||||
@require_login
|
@require_login
|
||||||
async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
|
async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
|
||||||
@@ -49,11 +67,39 @@ async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
|
|||||||
max_mailboxes = _get_max_mailboxes(tier)
|
max_mailboxes = _get_max_mailboxes(tier)
|
||||||
can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes)
|
can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes)
|
||||||
|
|
||||||
|
# Load ingestion profiles: system-global + user's own
|
||||||
|
profiles: list[ImapIngestionProfile] = (
|
||||||
|
db.query(ImapIngestionProfile)
|
||||||
|
.filter(
|
||||||
|
(ImapIngestionProfile.owner_id == None) # noqa: E711
|
||||||
|
| (ImapIngestionProfile.owner_id == owner_id)
|
||||||
|
)
|
||||||
|
.order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
|
||||||
|
.all()
|
||||||
|
if owner_id
|
||||||
|
else db.query(ImapIngestionProfile)
|
||||||
|
.filter(ImapIngestionProfile.owner_id == None) # noqa: E711
|
||||||
|
.order_by(ImapIngestionProfile.id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
# Category definitions for the UI checkbox builder
|
||||||
|
categories = [
|
||||||
|
{
|
||||||
|
"key": key,
|
||||||
|
"label": info["label"],
|
||||||
|
"description": info["description"],
|
||||||
|
}
|
||||||
|
for key, info in FILE_TYPE_CATEGORIES.items()
|
||||||
|
]
|
||||||
|
|
||||||
return templates.TemplateResponse(
|
return templates.TemplateResponse(
|
||||||
"imap_accounts.html",
|
"imap_accounts.html",
|
||||||
{
|
{
|
||||||
"request": request,
|
"request": request,
|
||||||
"accounts": accounts,
|
"accounts": accounts,
|
||||||
|
"profiles": [_serialize_profile(p) for p in profiles],
|
||||||
|
"categories": categories,
|
||||||
"current_count": current_count,
|
"current_count": current_count,
|
||||||
"max_mailboxes": max_mailboxes,
|
"max_mailboxes": max_mailboxes,
|
||||||
"can_add": can_add,
|
"can_add": can_add,
|
||||||
|
|||||||
@@ -172,14 +172,9 @@
|
|||||||
<i class="fas fa-trash-alt mr-1" aria-hidden="true"></i>Delete after process
|
<i class="fas fa-trash-alt mr-1" aria-hidden="true"></i>Delete after process
|
||||||
</span>
|
</span>
|
||||||
</template>
|
</template>
|
||||||
<template x-if="acct.attachment_filter === 'all'">
|
<template x-if="acct.profile_id">
|
||||||
<span class="inline-flex items-center px-2 py-0.5 rounded bg-blue-50 text-blue-700">
|
<span class="inline-flex items-center px-2 py-0.5 rounded bg-blue-50 text-blue-700">
|
||||||
<i class="fas fa-paperclip mr-1" aria-hidden="true"></i>All attachments
|
<i class="fas fa-filter mr-1" aria-hidden="true"></i><span x-text="profileName(acct.profile_id)"></span>
|
||||||
</span>
|
|
||||||
</template>
|
|
||||||
<template x-if="acct.attachment_filter === 'documents_only'">
|
|
||||||
<span class="inline-flex items-center px-2 py-0.5 rounded bg-gray-50 text-gray-600">
|
|
||||||
<i class="fas fa-file-alt mr-1" aria-hidden="true"></i>Documents only
|
|
||||||
</span>
|
</span>
|
||||||
</template>
|
</template>
|
||||||
<template x-if="acct.last_checked_at">
|
<template x-if="acct.last_checked_at">
|
||||||
@@ -398,22 +393,45 @@
|
|||||||
|
|
||||||
<!-- Attachment filter -->
|
<!-- Attachment filter -->
|
||||||
<div>
|
<div>
|
||||||
<label for="acct-attachment-filter" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
|
<label for="acct-profile" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
|
||||||
Attachment Types to Ingest
|
Ingestion Profile
|
||||||
</label>
|
</label>
|
||||||
|
<div class="flex gap-2 items-center">
|
||||||
<select
|
<select
|
||||||
id="acct-attachment-filter"
|
id="acct-profile"
|
||||||
x-model="form.attachment_filter"
|
x-model.number="form.profile_id"
|
||||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
|
class="flex-1 px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
|
||||||
aria-describedby="acct-attachment-filter-hint"
|
aria-describedby="acct-profile-hint"
|
||||||
>
|
>
|
||||||
<option value="">Use global default</option>
|
<option :value="null">Use global default</option>
|
||||||
<option value="documents_only">Documents only (PDF, Office, ODT, RTF, TXT, CSV) — no images</option>
|
<template x-for="profile in profiles" :key="profile.id">
|
||||||
<option value="all">All supported types (including images)</option>
|
<option :value="profile.id" x-text="profile.name + (profile.is_builtin ? '' : ' (custom)')"></option>
|
||||||
|
</template>
|
||||||
</select>
|
</select>
|
||||||
<p id="acct-attachment-filter-hint" class="mt-1 text-xs text-gray-400 dark:text-gray-500">
|
<button
|
||||||
Override the system-wide attachment filter for this mailbox. Leave blank to use the global default (documents only).
|
type="button"
|
||||||
|
@click="openProfileModal(null)"
|
||||||
|
class="inline-flex items-center px-3 py-2 text-sm font-medium rounded border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-blue-500"
|
||||||
|
style="min-height:40px;"
|
||||||
|
title="Create new profile"
|
||||||
|
aria-label="Create new ingestion profile"
|
||||||
|
>
|
||||||
|
<i class="fas fa-plus" aria-hidden="true"></i>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<p id="acct-profile-hint" class="mt-1 text-xs text-gray-400 dark:text-gray-500">
|
||||||
|
Controls which attachment types are ingested. Leave blank to use the global default (documents only).
|
||||||
|
<button type="button" @click="showProfilesSection = !showProfilesSection" class="underline hover:text-blue-600 focus:outline-none">
|
||||||
|
Manage profiles
|
||||||
|
</button>
|
||||||
</p>
|
</p>
|
||||||
|
<!-- Inline selected profile summary -->
|
||||||
|
<template x-if="form.profile_id">
|
||||||
|
<div class="mt-2 text-xs bg-blue-50 dark:bg-blue-900/20 rounded px-2 py-1.5 text-blue-800 dark:text-blue-200">
|
||||||
|
<strong x-text="profileName(form.profile_id)"></strong>:
|
||||||
|
<span x-text="profileCategorySummary(form.profile_id)"></span>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Test connection result -->
|
<!-- Test connection result -->
|
||||||
@@ -524,10 +542,276 @@
|
|||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- ── Ingestion Profiles panel ─────────────────────────────────────────────── -->
|
||||||
|
<div
|
||||||
|
x-show="showProfilesSection"
|
||||||
|
x-transition
|
||||||
|
class="fixed inset-0 z-50 flex items-center justify-center bg-black/50 px-4"
|
||||||
|
role="dialog"
|
||||||
|
aria-modal="true"
|
||||||
|
aria-labelledby="profiles-panel-title"
|
||||||
|
@keydown.escape.window="showProfilesSection = false"
|
||||||
|
style="display:none;"
|
||||||
|
>
|
||||||
|
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-2xl overflow-y-auto max-h-[90vh]" @click.stop>
|
||||||
|
<div class="px-6 pt-6 pb-2 flex items-center justify-between">
|
||||||
|
<h2 id="profiles-panel-title" class="text-lg font-semibold text-gray-900 dark:text-white flex items-center gap-2">
|
||||||
|
<i class="fas fa-filter text-blue-500" aria-hidden="true"></i>
|
||||||
|
Ingestion Profiles
|
||||||
|
</h2>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
@click="showProfilesSection = false"
|
||||||
|
class="text-gray-400 hover:text-gray-600 dark:hover:text-gray-200 focus:outline-none focus:ring-2 focus:ring-blue-500 rounded"
|
||||||
|
aria-label="Close profiles panel"
|
||||||
|
>
|
||||||
|
<i class="fas fa-times text-xl" aria-hidden="true"></i>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div class="px-6 pb-6 pt-2">
|
||||||
|
<p class="text-sm text-gray-500 dark:text-gray-400 mb-4">
|
||||||
|
Profiles define which attachment types are ingested from emails. Assign a profile to each IMAP account for fine-grained control.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<!-- Profile list -->
|
||||||
|
<div class="space-y-3 mb-4">
|
||||||
|
<template x-for="profile in profiles" :key="profile.id">
|
||||||
|
<div class="border border-gray-100 dark:border-gray-700 rounded-lg p-4 flex items-start gap-3">
|
||||||
|
<div class="flex-1 min-w-0">
|
||||||
|
<div class="flex items-center gap-2 flex-wrap">
|
||||||
|
<span class="font-medium text-sm text-gray-900 dark:text-white" x-text="profile.name"></span>
|
||||||
|
<template x-if="profile.is_builtin">
|
||||||
|
<span class="text-xs px-1.5 py-0.5 rounded bg-gray-100 dark:bg-gray-700 text-gray-500">built-in</span>
|
||||||
|
</template>
|
||||||
|
</div>
|
||||||
|
<p class="text-xs text-gray-500 dark:text-gray-400 mt-0.5" x-text="profile.description || ''"></p>
|
||||||
|
<!-- Category badges -->
|
||||||
|
<div class="flex flex-wrap gap-1 mt-2">
|
||||||
|
<template x-for="cat in profile.allowed_categories" :key="cat">
|
||||||
|
<span class="inline-flex items-center px-2 py-0.5 rounded text-xs font-medium bg-blue-50 dark:bg-blue-900/30 text-blue-700 dark:text-blue-300">
|
||||||
|
<span x-text="categoryLabel(cat)"></span>
|
||||||
|
</span>
|
||||||
|
</template>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="flex gap-1 shrink-0">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
@click="openProfileModal(profile)"
|
||||||
|
:disabled="profile.is_builtin"
|
||||||
|
:class="profile.is_builtin ? 'opacity-40 cursor-not-allowed' : ''"
|
||||||
|
class="p-1.5 text-gray-400 hover:text-blue-600 focus:outline-none focus:ring-2 focus:ring-blue-500 rounded"
|
||||||
|
:aria-label="`Edit profile ${profile.name}`"
|
||||||
|
:title="profile.is_builtin ? 'Built-in profiles cannot be edited' : 'Edit profile'"
|
||||||
|
>
|
||||||
|
<i class="fas fa-edit text-sm" aria-hidden="true"></i>
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
@click="confirmDeleteProfile(profile)"
|
||||||
|
:disabled="profile.is_builtin"
|
||||||
|
:class="profile.is_builtin ? 'opacity-40 cursor-not-allowed' : ''"
|
||||||
|
class="p-1.5 text-gray-400 hover:text-red-600 focus:outline-none focus:ring-2 focus:ring-red-500 rounded"
|
||||||
|
:aria-label="`Delete profile ${profile.name}`"
|
||||||
|
:title="profile.is_builtin ? 'Built-in profiles cannot be deleted' : 'Delete profile'"
|
||||||
|
>
|
||||||
|
<i class="fas fa-trash-alt text-sm" aria-hidden="true"></i>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
@click="openProfileModal(null)"
|
||||||
|
class="inline-flex items-center px-4 py-2 text-sm font-medium rounded bg-blue-600 hover:bg-blue-700 text-white focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||||
|
style="min-height:40px;"
|
||||||
|
>
|
||||||
|
<i class="fas fa-plus mr-2" aria-hidden="true"></i> New Profile
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- ── Profile create / edit modal ────────────────────────────────────────────── -->
|
||||||
|
<div
|
||||||
|
x-show="profileModalOpen"
|
||||||
|
x-transition:enter="ease-out duration-200"
|
||||||
|
x-transition:enter-start="opacity-0"
|
||||||
|
x-transition:enter-end="opacity-100"
|
||||||
|
x-transition:leave="ease-in duration-150"
|
||||||
|
x-transition:leave-start="opacity-100"
|
||||||
|
x-transition:leave-end="opacity-0"
|
||||||
|
class="fixed inset-0 z-[60] flex items-center justify-center bg-black/50 px-4"
|
||||||
|
role="dialog"
|
||||||
|
aria-modal="true"
|
||||||
|
:aria-labelledby="editingProfile ? 'profile-modal-title-edit' : 'profile-modal-title-create'"
|
||||||
|
@keydown.escape.window="closeProfileModal()"
|
||||||
|
style="display:none;"
|
||||||
|
>
|
||||||
|
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-lg overflow-y-auto max-h-[90vh]" @click.stop>
|
||||||
|
<div class="px-6 pt-6 pb-2 flex items-center justify-between">
|
||||||
|
<h2
|
||||||
|
:id="editingProfile ? 'profile-modal-title-edit' : 'profile-modal-title-create'"
|
||||||
|
class="text-lg font-semibold text-gray-900 dark:text-white"
|
||||||
|
x-text="editingProfile ? 'Edit Profile' : 'New Ingestion Profile'"
|
||||||
|
></h2>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
@click="closeProfileModal()"
|
||||||
|
class="text-gray-400 hover:text-gray-600 dark:hover:text-gray-200 focus:outline-none focus:ring-2 focus:ring-blue-500 rounded"
|
||||||
|
aria-label="Close modal"
|
||||||
|
>
|
||||||
|
<i class="fas fa-times text-xl" aria-hidden="true"></i>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<form @submit.prevent="saveProfile()" class="px-6 pb-6 pt-4 space-y-4">
|
||||||
|
|
||||||
|
<!-- Profile name -->
|
||||||
|
<div>
|
||||||
|
<label for="profile-name" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
|
||||||
|
Profile Name <span class="text-red-500" aria-hidden="true">*</span>
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
id="profile-name"
|
||||||
|
type="text"
|
||||||
|
x-model="profileForm.name"
|
||||||
|
placeholder="e.g. Scanner Inbox, Finance Documents"
|
||||||
|
required
|
||||||
|
maxlength="255"
|
||||||
|
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
|
||||||
|
aria-required="true"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Description -->
|
||||||
|
<div>
|
||||||
|
<label for="profile-description" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
|
||||||
|
Description
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
id="profile-description"
|
||||||
|
type="text"
|
||||||
|
x-model="profileForm.description"
|
||||||
|
placeholder="Optional description"
|
||||||
|
maxlength="500"
|
||||||
|
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Category checkboxes -->
|
||||||
|
<div>
|
||||||
|
<fieldset>
|
||||||
|
<legend class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-2">
|
||||||
|
File Type Categories <span class="text-red-500" aria-hidden="true">*</span>
|
||||||
|
</legend>
|
||||||
|
<div class="space-y-2">
|
||||||
|
<template x-for="cat in categories" :key="cat.key">
|
||||||
|
<label class="flex items-start gap-3 cursor-pointer group">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
:value="cat.key"
|
||||||
|
:checked="profileForm.allowed_categories.includes(cat.key)"
|
||||||
|
@change="toggleCategory(cat.key)"
|
||||||
|
class="mt-0.5 h-4 w-4 rounded border-gray-300 text-blue-600 focus:ring-blue-500"
|
||||||
|
/>
|
||||||
|
<span class="flex-1">
|
||||||
|
<span class="text-sm font-medium text-gray-800 dark:text-gray-100" x-text="cat.label"></span>
|
||||||
|
<span class="block text-xs text-gray-500 dark:text-gray-400" x-text="cat.description"></span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
</template>
|
||||||
|
</div>
|
||||||
|
<p x-show="profileForm.allowed_categories.length === 0" class="mt-2 text-xs text-red-600" role="alert">
|
||||||
|
Select at least one category.
|
||||||
|
</p>
|
||||||
|
</fieldset>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Form error -->
|
||||||
|
<template x-if="profileFormError">
|
||||||
|
<p class="text-sm text-red-600" role="alert" x-text="profileFormError"></p>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<!-- Actions -->
|
||||||
|
<div class="flex justify-end gap-2 pt-2 border-t border-gray-100 dark:border-gray-700">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
@click="closeProfileModal()"
|
||||||
|
class="px-4 py-2 text-sm font-medium rounded border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-blue-500"
|
||||||
|
style="min-height:40px;"
|
||||||
|
>
|
||||||
|
Cancel
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="submit"
|
||||||
|
:disabled="savingProfile || profileForm.allowed_categories.length === 0"
|
||||||
|
class="inline-flex items-center px-4 py-2 text-sm font-medium rounded bg-blue-600 hover:bg-blue-700 text-white focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50"
|
||||||
|
style="min-height:40px;"
|
||||||
|
>
|
||||||
|
<template x-if="savingProfile">
|
||||||
|
<i class="fas fa-spinner fa-spin mr-2" aria-hidden="true"></i>
|
||||||
|
</template>
|
||||||
|
<span x-text="editingProfile ? 'Save Changes' : 'Create Profile'"></span>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- ── Delete profile confirmation modal ────────────────────────────────────── -->
|
||||||
|
<div
|
||||||
|
x-show="deleteProfileModalOpen"
|
||||||
|
x-transition
|
||||||
|
class="fixed inset-0 z-[60] flex items-center justify-center bg-black/50 px-4"
|
||||||
|
role="dialog"
|
||||||
|
aria-modal="true"
|
||||||
|
aria-labelledby="delete-profile-modal-title"
|
||||||
|
@keydown.escape.window="deleteProfileModalOpen = false"
|
||||||
|
style="display:none;"
|
||||||
|
>
|
||||||
|
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-sm p-6" @click.stop>
|
||||||
|
<h2 id="delete-profile-modal-title" class="text-lg font-semibold text-gray-900 dark:text-white mb-2">
|
||||||
|
Delete Profile?
|
||||||
|
</h2>
|
||||||
|
<p class="text-sm text-gray-600 dark:text-gray-300 mb-4">
|
||||||
|
Are you sure you want to delete the profile
|
||||||
|
<strong x-text="profileToDelete ? profileToDelete.name : ''"></strong>?
|
||||||
|
IMAP accounts using this profile will fall back to the global default.
|
||||||
|
</p>
|
||||||
|
<div class="flex justify-end gap-2">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
@click="deleteProfileModalOpen = false; profileToDelete = null"
|
||||||
|
class="px-4 py-2 text-sm font-medium rounded border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-gray-500"
|
||||||
|
style="min-height:40px;"
|
||||||
|
>
|
||||||
|
Cancel
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
@click="deleteProfile()"
|
||||||
|
:disabled="deletingProfile"
|
||||||
|
class="inline-flex items-center px-4 py-2 text-sm font-medium rounded bg-red-600 hover:bg-red-700 text-white focus:outline-none focus:ring-2 focus:ring-red-500 disabled:opacity-50"
|
||||||
|
style="min-height:40px;"
|
||||||
|
>
|
||||||
|
<template x-if="deletingProfile">
|
||||||
|
<i class="fas fa-spinner fa-spin mr-2" aria-hidden="true"></i>
|
||||||
|
</template>
|
||||||
|
Delete
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
function imapAccountsApp() {
|
function imapAccountsApp() {
|
||||||
return {
|
return {
|
||||||
accounts: {{ accounts | tojson }},
|
accounts: {{ accounts | tojson }},
|
||||||
|
profiles: {{ profiles | tojson }},
|
||||||
|
categories: {{ categories | tojson }},
|
||||||
quota: {
|
quota: {
|
||||||
current_count: {{ current_count }},
|
current_count: {{ current_count }},
|
||||||
max_mailboxes: {{ max_mailboxes | tojson }},
|
max_mailboxes: {{ max_mailboxes | tojson }},
|
||||||
@@ -538,17 +822,17 @@ function imapAccountsApp() {
|
|||||||
loading: false,
|
loading: false,
|
||||||
alert: { show: false, type: '', title: '', message: '' },
|
alert: { show: false, type: '', title: '', message: '' },
|
||||||
|
|
||||||
// Modal state
|
// Account modal state
|
||||||
modalOpen: false,
|
modalOpen: false,
|
||||||
editingAccount: null,
|
editingAccount: null,
|
||||||
form: { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true },
|
form: { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true, profile_id: null },
|
||||||
showPassword: false,
|
showPassword: false,
|
||||||
saving: false,
|
saving: false,
|
||||||
testing: false,
|
testing: false,
|
||||||
testResult: null,
|
testResult: null,
|
||||||
formError: null,
|
formError: null,
|
||||||
|
|
||||||
// Delete state
|
// Account delete state
|
||||||
deleteModalOpen: false,
|
deleteModalOpen: false,
|
||||||
accountToDelete: null,
|
accountToDelete: null,
|
||||||
deleting: false,
|
deleting: false,
|
||||||
@@ -556,6 +840,17 @@ function imapAccountsApp() {
|
|||||||
// Test (saved account) state
|
// Test (saved account) state
|
||||||
testingId: null,
|
testingId: null,
|
||||||
|
|
||||||
|
// Profiles panel & modal state
|
||||||
|
showProfilesSection: false,
|
||||||
|
profileModalOpen: false,
|
||||||
|
editingProfile: null,
|
||||||
|
profileForm: { name: '', description: '', allowed_categories: [] },
|
||||||
|
savingProfile: false,
|
||||||
|
profileFormError: null,
|
||||||
|
deleteProfileModalOpen: false,
|
||||||
|
profileToDelete: null,
|
||||||
|
deletingProfile: false,
|
||||||
|
|
||||||
get canAdd() {
|
get canAdd() {
|
||||||
return this.quota.can_add;
|
return this.quota.can_add;
|
||||||
},
|
},
|
||||||
@@ -579,9 +874,33 @@ function imapAccountsApp() {
|
|||||||
setTimeout(() => { this.alert.show = false; }, 5000);
|
setTimeout(() => { this.alert.show = false; }, 5000);
|
||||||
},
|
},
|
||||||
|
|
||||||
|
// ── Profile helpers ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
profileById(id) {
|
||||||
|
return this.profiles.find(p => p.id === id) || null;
|
||||||
|
},
|
||||||
|
|
||||||
|
profileName(id) {
|
||||||
|
const p = this.profileById(id);
|
||||||
|
return p ? p.name : 'Unknown profile';
|
||||||
|
},
|
||||||
|
|
||||||
|
profileCategorySummary(id) {
|
||||||
|
const p = this.profileById(id);
|
||||||
|
if (!p) return '';
|
||||||
|
return p.allowed_categories.map(k => this.categoryLabel(k)).join(', ');
|
||||||
|
},
|
||||||
|
|
||||||
|
categoryLabel(key) {
|
||||||
|
const cat = this.categories.find(c => c.key === key);
|
||||||
|
return cat ? cat.label : key;
|
||||||
|
},
|
||||||
|
|
||||||
|
// ── Account modal ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
openCreateModal() {
|
openCreateModal() {
|
||||||
this.editingAccount = null;
|
this.editingAccount = null;
|
||||||
this.form = { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true, attachment_filter: '' };
|
this.form = { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true, profile_id: null };
|
||||||
this.showPassword = false;
|
this.showPassword = false;
|
||||||
this.testResult = null;
|
this.testResult = null;
|
||||||
this.formError = null;
|
this.formError = null;
|
||||||
@@ -599,7 +918,7 @@ function imapAccountsApp() {
|
|||||||
use_ssl: acct.use_ssl,
|
use_ssl: acct.use_ssl,
|
||||||
delete_after_process: acct.delete_after_process,
|
delete_after_process: acct.delete_after_process,
|
||||||
is_active: acct.is_active,
|
is_active: acct.is_active,
|
||||||
attachment_filter: acct.attachment_filter || '',
|
profile_id: acct.profile_id || null,
|
||||||
};
|
};
|
||||||
this.showPassword = false;
|
this.showPassword = false;
|
||||||
this.testResult = null;
|
this.testResult = null;
|
||||||
@@ -623,6 +942,8 @@ function imapAccountsApp() {
|
|||||||
if (this.editingAccount && payload.password === '') {
|
if (this.editingAccount && payload.password === '') {
|
||||||
delete payload.password;
|
delete payload.password;
|
||||||
}
|
}
|
||||||
|
// Normalise profile_id: empty string → null
|
||||||
|
if (payload.profile_id === '' || payload.profile_id === 0) payload.profile_id = null;
|
||||||
let resp;
|
let resp;
|
||||||
if (this.editingAccount) {
|
if (this.editingAccount) {
|
||||||
resp = await fetch(`/api/imap-accounts/${this.editingAccount.id}`, {
|
resp = await fetch(`/api/imap-accounts/${this.editingAccount.id}`, {
|
||||||
@@ -750,6 +1071,126 @@ function imapAccountsApp() {
|
|||||||
this.deleting = false;
|
this.deleting = false;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
|
// ── Profile CRUD ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
openProfileModal(profile) {
|
||||||
|
this.editingProfile = profile;
|
||||||
|
if (profile) {
|
||||||
|
this.profileForm = {
|
||||||
|
name: profile.name,
|
||||||
|
description: profile.description || '',
|
||||||
|
allowed_categories: [...profile.allowed_categories],
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
this.profileForm = { name: '', description: '', allowed_categories: ['pdf', 'office', 'opendocument', 'text', 'web'] };
|
||||||
|
}
|
||||||
|
this.profileFormError = null;
|
||||||
|
this.profileModalOpen = true;
|
||||||
|
},
|
||||||
|
|
||||||
|
closeProfileModal() {
|
||||||
|
this.profileModalOpen = false;
|
||||||
|
this.editingProfile = null;
|
||||||
|
this.profileFormError = null;
|
||||||
|
},
|
||||||
|
|
||||||
|
toggleCategory(key) {
|
||||||
|
const idx = this.profileForm.allowed_categories.indexOf(key);
|
||||||
|
if (idx === -1) {
|
||||||
|
this.profileForm.allowed_categories.push(key);
|
||||||
|
} else {
|
||||||
|
this.profileForm.allowed_categories.splice(idx, 1);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
async saveProfile() {
|
||||||
|
if (this.profileForm.allowed_categories.length === 0) {
|
||||||
|
this.profileFormError = 'Select at least one category.';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.profileFormError = null;
|
||||||
|
this.savingProfile = true;
|
||||||
|
try {
|
||||||
|
const payload = {
|
||||||
|
name: this.profileForm.name,
|
||||||
|
description: this.profileForm.description || null,
|
||||||
|
allowed_categories: this.profileForm.allowed_categories,
|
||||||
|
};
|
||||||
|
let resp;
|
||||||
|
if (this.editingProfile) {
|
||||||
|
resp = await fetch(`/api/imap-profiles/${this.editingProfile.id}`, {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': getCsrfToken() },
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
resp = await fetch('/api/imap-profiles/', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': getCsrfToken() },
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const data = await resp.json();
|
||||||
|
if (!resp.ok) {
|
||||||
|
this.profileFormError = data.detail || 'Failed to save profile.';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Flatten response into the same shape used locally
|
||||||
|
const normalized = {
|
||||||
|
id: data.id,
|
||||||
|
name: data.name,
|
||||||
|
description: data.description,
|
||||||
|
owner_id: data.owner_id,
|
||||||
|
allowed_categories: data.allowed_categories,
|
||||||
|
is_builtin: data.is_builtin,
|
||||||
|
};
|
||||||
|
if (this.editingProfile) {
|
||||||
|
const idx = this.profiles.findIndex(p => p.id === normalized.id);
|
||||||
|
if (idx !== -1) this.profiles.splice(idx, 1, normalized);
|
||||||
|
} else {
|
||||||
|
this.profiles.push(normalized);
|
||||||
|
}
|
||||||
|
this.closeProfileModal();
|
||||||
|
this.showAlert('success', 'Saved', this.editingProfile ? 'Profile updated.' : 'Profile created.');
|
||||||
|
} catch (err) {
|
||||||
|
this.profileFormError = `Network error: ${err.message || 'Unknown error'}. Please try again.`;
|
||||||
|
} finally {
|
||||||
|
this.savingProfile = false;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
confirmDeleteProfile(profile) {
|
||||||
|
this.profileToDelete = profile;
|
||||||
|
this.deleteProfileModalOpen = true;
|
||||||
|
},
|
||||||
|
|
||||||
|
async deleteProfile() {
|
||||||
|
if (!this.profileToDelete) return;
|
||||||
|
this.deletingProfile = true;
|
||||||
|
try {
|
||||||
|
const resp = await fetch(`/api/imap-profiles/${this.profileToDelete.id}`, {
|
||||||
|
method: 'DELETE',
|
||||||
|
headers: { 'X-CSRF-Token': getCsrfToken() },
|
||||||
|
});
|
||||||
|
if (resp.ok || resp.status === 204) {
|
||||||
|
const deletedId = this.profileToDelete.id;
|
||||||
|
this.profiles = this.profiles.filter(p => p.id !== deletedId);
|
||||||
|
// Clear profile_id on accounts that referenced this profile
|
||||||
|
this.accounts.forEach(a => { if (a.profile_id === deletedId) a.profile_id = null; });
|
||||||
|
this.showAlert('success', 'Deleted', `Profile "${this.profileToDelete.name}" removed.`);
|
||||||
|
this.deleteProfileModalOpen = false;
|
||||||
|
this.profileToDelete = null;
|
||||||
|
} else {
|
||||||
|
const data = await resp.json();
|
||||||
|
this.showAlert('error', 'Delete Failed', data.detail || 'Could not delete profile.');
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
this.showAlert('error', 'Delete Failed', `Network error: ${err.message || 'Unknown error'}`);
|
||||||
|
} finally {
|
||||||
|
this.deletingProfile = false;
|
||||||
|
}
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ from app.models import ( # noqa: F401
|
|||||||
DocumentMetadata,
|
DocumentMetadata,
|
||||||
FileProcessingStep,
|
FileProcessingStep,
|
||||||
FileRecord,
|
FileRecord,
|
||||||
|
ImapIngestionProfile,
|
||||||
InAppNotification,
|
InAppNotification,
|
||||||
LocalUser,
|
LocalUser,
|
||||||
MobileDevice,
|
MobileDevice,
|
||||||
|
|||||||
@@ -0,0 +1,146 @@
|
|||||||
|
"""Add imap_ingestion_profiles table and migrate user_imap_accounts.
|
||||||
|
|
||||||
|
Creates the ``imap_ingestion_profiles`` table, seeds the two built-in profiles
|
||||||
|
("Documents Only" and "All Files"), and replaces the ``attachment_filter``
|
||||||
|
string column on ``user_imap_accounts`` with a ``profile_id`` FK that references
|
||||||
|
the new table.
|
||||||
|
|
||||||
|
Revision ID: 033_add_imap_ingestion_profiles
|
||||||
|
Revises: 032_add_imap_attachment_filter
|
||||||
|
Create Date: 2026-03-12
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "033_add_imap_ingestion_profiles"
|
||||||
|
down_revision: Union[str, None] = "032_add_imap_attachment_filter"
|
||||||
|
depends_on: Union[str, None] = None
|
||||||
|
|
||||||
|
# Fixed IDs for the built-in profiles so that the FK migration is reproducible.
|
||||||
|
_BUILTIN_DOCUMENTS_ONLY_ID = 1
|
||||||
|
_BUILTIN_ALL_FILES_ID = 2
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Create profiles table, seed built-ins, migrate accounts column."""
|
||||||
|
# 1 — Create the imap_ingestion_profiles table
|
||||||
|
op.create_table(
|
||||||
|
"imap_ingestion_profiles",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("name", sa.String(255), nullable=False),
|
||||||
|
sa.Column("description", sa.Text(), nullable=True),
|
||||||
|
sa.Column("owner_id", sa.String(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"allowed_categories",
|
||||||
|
sa.Text(),
|
||||||
|
nullable=False,
|
||||||
|
server_default='["pdf","office","opendocument","text","web"]',
|
||||||
|
),
|
||||||
|
sa.Column("is_builtin", sa.Boolean(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_imap_ingestion_profiles_id", "imap_ingestion_profiles", ["id"])
|
||||||
|
op.create_index("ix_imap_ingestion_profiles_owner_id", "imap_ingestion_profiles", ["owner_id"])
|
||||||
|
|
||||||
|
# 2 — Seed the two built-in profiles
|
||||||
|
bind = op.get_bind()
|
||||||
|
bind.execute(
|
||||||
|
sa.text(
|
||||||
|
"INSERT INTO imap_ingestion_profiles "
|
||||||
|
"(id, name, description, owner_id, allowed_categories, is_builtin) "
|
||||||
|
"VALUES (:id, :name, :desc, NULL, :cats, 1)"
|
||||||
|
),
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": _BUILTIN_DOCUMENTS_ONLY_ID,
|
||||||
|
"name": "Documents Only",
|
||||||
|
"desc": (
|
||||||
|
"Ingest PDFs, Microsoft Office files, OpenDocument files, "
|
||||||
|
"plain text, CSV, RTF, HTML and Markdown. Images are excluded."
|
||||||
|
),
|
||||||
|
"cats": '["pdf","office","opendocument","text","web"]',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": _BUILTIN_ALL_FILES_ID,
|
||||||
|
"name": "All Files",
|
||||||
|
"desc": "Ingest all supported file types, including images.",
|
||||||
|
"cats": '["pdf","office","opendocument","text","web","images"]',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
# 3 — Add profile_id column to user_imap_accounts
|
||||||
|
op.add_column(
|
||||||
|
"user_imap_accounts",
|
||||||
|
sa.Column("profile_id", sa.Integer(), nullable=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
# 4 — Migrate existing attachment_filter values to profile_id
|
||||||
|
bind.execute(
|
||||||
|
sa.text(
|
||||||
|
"UPDATE user_imap_accounts SET profile_id = :pid "
|
||||||
|
"WHERE attachment_filter = 'all'"
|
||||||
|
),
|
||||||
|
{"pid": _BUILTIN_ALL_FILES_ID},
|
||||||
|
)
|
||||||
|
bind.execute(
|
||||||
|
sa.text(
|
||||||
|
"UPDATE user_imap_accounts SET profile_id = :pid "
|
||||||
|
"WHERE attachment_filter = 'documents_only'"
|
||||||
|
),
|
||||||
|
{"pid": _BUILTIN_DOCUMENTS_ONLY_ID},
|
||||||
|
)
|
||||||
|
# Rows with NULL attachment_filter keep profile_id = NULL (use global default)
|
||||||
|
|
||||||
|
# 5 — Add FK constraint (skip for SQLite which does not enforce FKs at DDL time)
|
||||||
|
# We use batch_alter_table so this works across SQLite and PostgreSQL
|
||||||
|
with op.batch_alter_table("user_imap_accounts") as batch_op:
|
||||||
|
batch_op.create_foreign_key(
|
||||||
|
"fk_user_imap_accounts_profile_id",
|
||||||
|
"imap_ingestion_profiles",
|
||||||
|
["profile_id"],
|
||||||
|
["id"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# 6 — Drop the now-redundant attachment_filter column
|
||||||
|
with op.batch_alter_table("user_imap_accounts") as batch_op:
|
||||||
|
batch_op.drop_column("attachment_filter")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Reverse the migration: restore attachment_filter, drop profiles table."""
|
||||||
|
# 1 — Re-add attachment_filter column
|
||||||
|
with op.batch_alter_table("user_imap_accounts") as batch_op:
|
||||||
|
batch_op.add_column(sa.Column("attachment_filter", sa.String(50), nullable=True))
|
||||||
|
|
||||||
|
# 2 — Restore string values from profile_id
|
||||||
|
bind = op.get_bind()
|
||||||
|
bind.execute(
|
||||||
|
sa.text(
|
||||||
|
"UPDATE user_imap_accounts SET attachment_filter = 'all' "
|
||||||
|
"WHERE profile_id = :pid"
|
||||||
|
),
|
||||||
|
{"pid": _BUILTIN_ALL_FILES_ID},
|
||||||
|
)
|
||||||
|
bind.execute(
|
||||||
|
sa.text(
|
||||||
|
"UPDATE user_imap_accounts SET attachment_filter = 'documents_only' "
|
||||||
|
"WHERE profile_id = :pid"
|
||||||
|
),
|
||||||
|
{"pid": _BUILTIN_DOCUMENTS_ONLY_ID},
|
||||||
|
)
|
||||||
|
|
||||||
|
# 3 — Drop the FK and profile_id column
|
||||||
|
with op.batch_alter_table("user_imap_accounts") as batch_op:
|
||||||
|
batch_op.drop_constraint("fk_user_imap_accounts_profile_id", type_="foreignkey")
|
||||||
|
batch_op.drop_column("profile_id")
|
||||||
|
|
||||||
|
# 4 — Drop the profiles table
|
||||||
|
op.drop_index("ix_imap_ingestion_profiles_owner_id", "imap_ingestion_profiles")
|
||||||
|
op.drop_index("ix_imap_ingestion_profiles_id", "imap_ingestion_profiles")
|
||||||
|
op.drop_table("imap_ingestion_profiles")
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
"""Tests for app/api/imap_profiles.py and app/utils/allowed_types category helpers."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.utils.allowed_types import (
|
||||||
|
ALL_CATEGORIES,
|
||||||
|
DEFAULT_CATEGORIES,
|
||||||
|
FILE_TYPE_CATEGORIES,
|
||||||
|
get_allowed_types_for_categories,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
class TestFileTypeCategories:
|
||||||
|
"""Tests for FILE_TYPE_CATEGORIES and get_allowed_types_for_categories."""
|
||||||
|
|
||||||
|
def test_all_category_keys_present(self):
|
||||||
|
"""Test that the six expected categories exist."""
|
||||||
|
assert set(FILE_TYPE_CATEGORIES.keys()) == {"pdf", "office", "opendocument", "text", "web", "images"}
|
||||||
|
|
||||||
|
def test_each_category_has_required_fields(self):
|
||||||
|
"""Test that every category entry has label, description, mime_types, extensions."""
|
||||||
|
for key, info in FILE_TYPE_CATEGORIES.items():
|
||||||
|
assert "label" in info, f"Category '{key}' missing 'label'"
|
||||||
|
assert "description" in info, f"Category '{key}' missing 'description'"
|
||||||
|
assert "mime_types" in info, f"Category '{key}' missing 'mime_types'"
|
||||||
|
assert "extensions" in info, f"Category '{key}' missing 'extensions'"
|
||||||
|
|
||||||
|
def test_pdf_category_contains_pdf_mime(self):
|
||||||
|
"""Test that the pdf category includes application/pdf."""
|
||||||
|
assert "application/pdf" in FILE_TYPE_CATEGORIES["pdf"]["mime_types"]
|
||||||
|
assert ".pdf" in FILE_TYPE_CATEGORIES["pdf"]["extensions"]
|
||||||
|
|
||||||
|
def test_images_category_contains_jpeg(self):
|
||||||
|
"""Test that the images category includes image/jpeg."""
|
||||||
|
assert "image/jpeg" in FILE_TYPE_CATEGORIES["images"]["mime_types"]
|
||||||
|
assert ".jpg" in FILE_TYPE_CATEGORIES["images"]["extensions"]
|
||||||
|
assert ".png" in FILE_TYPE_CATEGORIES["images"]["extensions"]
|
||||||
|
|
||||||
|
def test_get_allowed_types_for_default_categories(self):
|
||||||
|
"""Test that DEFAULT_CATEGORIES excludes image MIME types."""
|
||||||
|
mime_types, extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
|
||||||
|
assert "application/pdf" in mime_types
|
||||||
|
assert "application/msword" in mime_types
|
||||||
|
# images should NOT be in the default
|
||||||
|
assert "image/jpeg" not in mime_types
|
||||||
|
assert ".jpg" not in extensions
|
||||||
|
|
||||||
|
def test_get_allowed_types_for_all_categories(self):
|
||||||
|
"""Test that ALL_CATEGORIES includes image MIME types."""
|
||||||
|
mime_types, extensions = get_allowed_types_for_categories(ALL_CATEGORIES)
|
||||||
|
assert "image/jpeg" in mime_types
|
||||||
|
assert ".jpg" in extensions
|
||||||
|
assert "application/pdf" in mime_types
|
||||||
|
|
||||||
|
def test_get_allowed_types_returns_frozensets(self):
|
||||||
|
"""Test that returned sets are frozensets."""
|
||||||
|
mime_types, extensions = get_allowed_types_for_categories(["pdf"])
|
||||||
|
assert isinstance(mime_types, frozenset)
|
||||||
|
assert isinstance(extensions, frozenset)
|
||||||
|
|
||||||
|
def test_get_allowed_types_unknown_category_ignored(self):
|
||||||
|
"""Test that unknown category keys are silently ignored."""
|
||||||
|
mime_types, extensions = get_allowed_types_for_categories(["pdf", "nonexistent_category"])
|
||||||
|
assert "application/pdf" in mime_types # 'pdf' still works
|
||||||
|
# No crash for unknown key
|
||||||
|
|
||||||
|
def test_get_allowed_types_empty_list(self):
|
||||||
|
"""Test empty category list returns empty sets."""
|
||||||
|
mime_types, extensions = get_allowed_types_for_categories([])
|
||||||
|
assert mime_types == frozenset()
|
||||||
|
assert extensions == frozenset()
|
||||||
|
|
||||||
|
def test_default_categories_excludes_images(self):
|
||||||
|
"""Test that DEFAULT_CATEGORIES does not include 'images'."""
|
||||||
|
assert "images" not in DEFAULT_CATEGORIES
|
||||||
|
|
||||||
|
def test_all_categories_includes_images(self):
|
||||||
|
"""Test that ALL_CATEGORIES includes 'images'."""
|
||||||
|
assert "images" in ALL_CATEGORIES
|
||||||
|
|
||||||
|
def test_all_categories_is_superset_of_default(self):
|
||||||
|
"""Test that ALL_CATEGORIES contains all DEFAULT_CATEGORIES."""
|
||||||
|
for cat in DEFAULT_CATEGORIES:
|
||||||
|
assert cat in ALL_CATEGORIES
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
class TestImapProfilesApiLogic:
|
||||||
|
"""Tests for ingestion profile validation helpers."""
|
||||||
|
|
||||||
|
def test_validate_categories_accepts_valid_keys(self):
|
||||||
|
"""Test that valid category keys pass validation."""
|
||||||
|
from app.api.imap_profiles import _validate_categories
|
||||||
|
|
||||||
|
result = _validate_categories(["pdf", "office", "images"])
|
||||||
|
assert set(result) == {"pdf", "office", "images"}
|
||||||
|
|
||||||
|
def test_validate_categories_rejects_unknown_key(self):
|
||||||
|
"""Test that unknown category keys raise 422."""
|
||||||
|
from fastapi import HTTPException
|
||||||
|
|
||||||
|
from app.api.imap_profiles import _validate_categories
|
||||||
|
|
||||||
|
with pytest.raises(HTTPException) as exc_info:
|
||||||
|
_validate_categories(["pdf", "nonexistent"])
|
||||||
|
assert exc_info.value.status_code == 422
|
||||||
|
assert "nonexistent" in str(exc_info.value.detail)
|
||||||
|
|
||||||
|
def test_validate_categories_deduplicates(self):
|
||||||
|
"""Test that duplicate category keys are de-duplicated while preserving order."""
|
||||||
|
from app.api.imap_profiles import _validate_categories
|
||||||
|
|
||||||
|
result = _validate_categories(["pdf", "pdf", "office", "pdf"])
|
||||||
|
assert result == ["pdf", "office"]
|
||||||
|
|
||||||
|
def test_to_response_serializes_profile(self, tmp_path):
|
||||||
|
"""Test _to_response produces expected dict shape."""
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from app.api.imap_profiles import _to_response
|
||||||
|
|
||||||
|
profile = MagicMock()
|
||||||
|
profile.id = 42
|
||||||
|
profile.name = "My Profile"
|
||||||
|
profile.description = "Test description"
|
||||||
|
profile.owner_id = "user@example.com"
|
||||||
|
profile.allowed_categories = '["pdf","office"]'
|
||||||
|
profile.is_builtin = False
|
||||||
|
profile.created_at = None
|
||||||
|
profile.updated_at = None
|
||||||
|
|
||||||
|
result = _to_response(profile)
|
||||||
|
assert result["id"] == 42
|
||||||
|
assert result["name"] == "My Profile"
|
||||||
|
assert result["allowed_categories"] == ["pdf", "office"]
|
||||||
|
assert len(result["categories_detail"]) == 2
|
||||||
|
assert result["categories_detail"][0]["key"] == "pdf"
|
||||||
|
assert result["is_builtin"] is False
|
||||||
|
|
||||||
|
def test_to_response_handles_invalid_categories_json(self):
|
||||||
|
"""Test _to_response gracefully handles invalid JSON in allowed_categories."""
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from app.api.imap_profiles import _to_response
|
||||||
|
|
||||||
|
profile = MagicMock()
|
||||||
|
profile.id = 1
|
||||||
|
profile.name = "Broken"
|
||||||
|
profile.description = None
|
||||||
|
profile.owner_id = None
|
||||||
|
profile.allowed_categories = "this is not valid json {"
|
||||||
|
profile.is_builtin = True
|
||||||
|
profile.created_at = None
|
||||||
|
profile.updated_at = None
|
||||||
|
|
||||||
|
result = _to_response(profile)
|
||||||
|
assert result["allowed_categories"] == []
|
||||||
+18
-15
@@ -24,6 +24,7 @@ from app.tasks.imap_tasks import (
|
|||||||
release_lock,
|
release_lock,
|
||||||
save_processed_emails,
|
save_processed_emails,
|
||||||
)
|
)
|
||||||
|
from app.utils.allowed_types import ALL_CATEGORIES, DEFAULT_CATEGORIES, get_allowed_types_for_categories
|
||||||
|
|
||||||
_TEST_CREDENTIAL = "pass" # noqa: S105
|
_TEST_CREDENTIAL = "pass" # noqa: S105
|
||||||
|
|
||||||
@@ -182,27 +183,29 @@ class TestFetchAttachmentsAndEnqueue:
|
|||||||
@patch("app.tasks.imap_tasks.process_document")
|
@patch("app.tasks.imap_tasks.process_document")
|
||||||
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
||||||
def test_skips_image_when_documents_only(self, mock_convert, mock_process):
|
def test_skips_image_when_documents_only(self, mock_convert, mock_process):
|
||||||
"""Test that image attachments are skipped with documents_only filter."""
|
"""Test that image attachments are skipped with the default (documents-only) categories."""
|
||||||
msg = EmailMessage()
|
msg = EmailMessage()
|
||||||
msg["Subject"] = "Photo"
|
msg["Subject"] = "Photo"
|
||||||
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
|
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
|
||||||
|
|
||||||
result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
|
doc_mime, doc_ext = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
|
||||||
|
result = fetch_attachments_and_enqueue(msg, effective_mime_types=doc_mime, effective_extensions=doc_ext)
|
||||||
assert result is False
|
assert result is False
|
||||||
mock_process.delay.assert_not_called()
|
mock_process.delay.assert_not_called()
|
||||||
mock_convert.delay.assert_not_called()
|
mock_convert.delay.assert_not_called()
|
||||||
|
|
||||||
@patch("app.tasks.imap_tasks.process_document")
|
@patch("app.tasks.imap_tasks.process_document")
|
||||||
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
||||||
def test_processes_image_when_all_filter(self, mock_convert, mock_process, tmp_path):
|
def test_processes_image_when_all_categories(self, mock_convert, mock_process, tmp_path):
|
||||||
"""Test that image attachments are processed with 'all' filter."""
|
"""Test that image attachments are processed when images category is included."""
|
||||||
msg = EmailMessage()
|
msg = EmailMessage()
|
||||||
msg["Subject"] = "Photo"
|
msg["Subject"] = "Photo"
|
||||||
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
|
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
|
||||||
|
|
||||||
|
all_mime, all_ext = get_allowed_types_for_categories(ALL_CATEGORIES)
|
||||||
with patch("app.tasks.imap_tasks.settings") as mock_settings:
|
with patch("app.tasks.imap_tasks.settings") as mock_settings:
|
||||||
mock_settings.workdir = str(tmp_path)
|
mock_settings.workdir = str(tmp_path)
|
||||||
result = fetch_attachments_and_enqueue(msg, attachment_filter="all")
|
result = fetch_attachments_and_enqueue(msg, effective_mime_types=all_mime, effective_extensions=all_ext)
|
||||||
|
|
||||||
assert result is True
|
assert result is True
|
||||||
mock_convert.delay.assert_called_once()
|
mock_convert.delay.assert_called_once()
|
||||||
@@ -210,42 +213,42 @@ class TestFetchAttachmentsAndEnqueue:
|
|||||||
@patch("app.tasks.imap_tasks.process_document")
|
@patch("app.tasks.imap_tasks.process_document")
|
||||||
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
||||||
def test_skips_image_with_image_extension_documents_only(self, mock_convert, mock_process):
|
def test_skips_image_with_image_extension_documents_only(self, mock_convert, mock_process):
|
||||||
"""Test image files identified by extension are skipped with documents_only."""
|
"""Test image files identified by extension are skipped with documents-only categories."""
|
||||||
msg = EmailMessage()
|
msg = EmailMessage()
|
||||||
msg["Subject"] = "Screenshot"
|
msg["Subject"] = "Screenshot"
|
||||||
msg.add_attachment(b"\x89PNG", maintype="application", subtype="octet-stream", filename="screenshot.png")
|
msg.add_attachment(b"\x89PNG", maintype="application", subtype="octet-stream", filename="screenshot.png")
|
||||||
|
|
||||||
result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
|
doc_mime, doc_ext = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
|
||||||
|
result = fetch_attachments_and_enqueue(msg, effective_mime_types=doc_mime, effective_extensions=doc_ext)
|
||||||
assert result is False
|
assert result is False
|
||||||
mock_process.delay.assert_not_called()
|
mock_process.delay.assert_not_called()
|
||||||
mock_convert.delay.assert_not_called()
|
mock_convert.delay.assert_not_called()
|
||||||
|
|
||||||
@patch("app.tasks.imap_tasks.process_document")
|
@patch("app.tasks.imap_tasks.process_document")
|
||||||
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
||||||
def test_processes_pdf_regardless_of_filter(self, mock_convert, mock_process, tmp_path):
|
def test_processes_pdf_regardless_of_categories(self, mock_convert, mock_process, tmp_path):
|
||||||
"""Test that PDFs are always processed, even with documents_only filter."""
|
"""Test that PDFs are always processed (pdf category always included in DEFAULT_CATEGORIES)."""
|
||||||
msg = EmailMessage()
|
msg = EmailMessage()
|
||||||
msg["Subject"] = "Invoice"
|
msg["Subject"] = "Invoice"
|
||||||
msg.add_attachment(b"%PDF-1.4", maintype="application", subtype="pdf", filename="invoice.pdf")
|
msg.add_attachment(b"%PDF-1.4", maintype="application", subtype="pdf", filename="invoice.pdf")
|
||||||
|
|
||||||
|
doc_mime, doc_ext = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
|
||||||
with patch("app.tasks.imap_tasks.settings") as mock_settings:
|
with patch("app.tasks.imap_tasks.settings") as mock_settings:
|
||||||
mock_settings.workdir = str(tmp_path)
|
mock_settings.workdir = str(tmp_path)
|
||||||
result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
|
result = fetch_attachments_and_enqueue(msg, effective_mime_types=doc_mime, effective_extensions=doc_ext)
|
||||||
|
|
||||||
assert result is True
|
assert result is True
|
||||||
mock_process.delay.assert_called_once()
|
mock_process.delay.assert_called_once()
|
||||||
|
|
||||||
@patch("app.tasks.imap_tasks.process_document")
|
@patch("app.tasks.imap_tasks.process_document")
|
||||||
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
@patch("app.tasks.imap_tasks.convert_to_pdf")
|
||||||
def test_uses_global_setting_when_no_filter(self, mock_convert, mock_process):
|
def test_uses_default_categories_when_no_types_provided(self, mock_convert, mock_process):
|
||||||
"""Test that the global settings.imap_attachment_filter is used when no filter is passed."""
|
"""Test that images are skipped when no effective_mime_types / extensions are passed (defaults to DEFAULT_CATEGORIES)."""
|
||||||
msg = EmailMessage()
|
msg = EmailMessage()
|
||||||
msg["Subject"] = "Photo"
|
msg["Subject"] = "Photo"
|
||||||
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
|
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
|
||||||
|
|
||||||
with patch("app.tasks.imap_tasks.settings") as mock_settings:
|
# No effective_mime_types passed → function defaults to DEFAULT_CATEGORIES (no images)
|
||||||
mock_settings.imap_attachment_filter = "documents_only"
|
|
||||||
mock_settings.workdir = "/tmp"
|
|
||||||
result = fetch_attachments_and_enqueue(msg)
|
result = fetch_attachments_and_enqueue(msg)
|
||||||
|
|
||||||
assert result is False
|
assert result is False
|
||||||
|
|||||||
Reference in New Issue
Block a user