From 5e45b68cc12e68b63672f67797321fd5cd4463b6 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Thu, 12 Mar 2026 00:31:45 +0000
Subject: [PATCH 1/4] Initial plan
From 554bb21d329ca2cca868a61adf337e29e4284ca4 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Thu, 12 Mar 2026 00:59:54 +0000
Subject: [PATCH 2/4] feat(imap): add attachment type filter for IMAP ingestion
Add a configurable switch to control which attachment types are ingested
via IMAP. Images are excluded by default; office files and PDFs are ingested.
- Add global `IMAP_ATTACHMENT_FILTER` config setting (default: `documents_only`)
- Add `attachment_filter` column to `UserImapAccount` model for per-user override
- Migration 032 adds the column to `user_imap_accounts` table
- Update `fetch_attachments_and_enqueue()` to respect filter (documents_only/all)
- Update `pull_inbox()`, `_pull_user_imap_accounts()`, and
`_pull_user_integration_imap()` to pass the resolved filter
- Update IMAP accounts API (schemas, create/update handlers, response serializer)
- Update IMAP accounts UI to show attachment filter dropdown in modal and
display filter badges on account cards
- Add 6 new tests covering attachment filter behaviour
- Update ConfigurationGuide.md, EmailIngestion.md, and .env.demo
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
---
.env.demo | 6 ++
app/api/imap_accounts.py | 26 +++++
app/config.py | 11 +++
app/models.py | 5 +
app/tasks/imap_tasks.py | 96 +++++++++++++++----
app/utils/settings_service.py | 12 +++
docs/ConfigurationGuide.md | 1 +
docs/howto/EmailIngestion.md | 31 ++++++
frontend/templates/imap_accounts.html | 33 ++++++-
.../032_add_imap_attachment_filter.py | 28 ++++++
tests/test_imap_tasks.py | 73 ++++++++++++++
11 files changed, 300 insertions(+), 22 deletions(-)
create mode 100644 migrations/versions/032_add_imap_attachment_filter.py
diff --git a/.env.demo b/.env.demo
index 59f17f94..b58c649f 100644
--- a/.env.demo
+++ b/.env.demo
@@ -311,6 +311,12 @@ IMAP2_DELETE_AFTER_PROCESS=false
# Use for pre-production instances that share a mailbox with production.
IMAP_READONLY_MODE=false
+# Controls which attachment types are ingested from IMAP emails.
+# 'documents_only' (default) – PDFs and office files only; images are skipped.
+# 'all' – all supported file types including images.
+# Per-user IMAP accounts can override this global default.
+IMAP_ATTACHMENT_FILTER=documents_only
+
# **Storage/Document Services**
# Amazon S3
AWS_REGION=us-east-1
diff --git a/app/api/imap_accounts.py b/app/api/imap_accounts.py
index 69b26c72..169fdef7 100644
--- a/app/api/imap_accounts.py
+++ b/app/api/imap_accounts.py
@@ -110,6 +110,15 @@ class ImapAccountCreate(BaseModel):
use_ssl: bool = Field(default=True, description="Use SSL/TLS connection")
delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing")
is_active: bool = Field(default=True, description="Whether to poll this mailbox")
+ attachment_filter: str | None = Field(
+ default=None,
+ description=(
+ "Controls which attachment types to ingest. "
+ "'documents_only' – PDFs and office files only (default when None). "
+ "'all' – all supported types including images. "
+ "Null inherits the global imap_attachment_filter setting."
+ ),
+ )
class ImapAccountUpdate(BaseModel):
@@ -123,6 +132,15 @@ class ImapAccountUpdate(BaseModel):
use_ssl: bool | None = None
delete_after_process: bool | None = None
is_active: bool | None = None
+ attachment_filter: str | None = Field(
+ default=None,
+ description=(
+ "Controls which attachment types to ingest. "
+ "'documents_only' – PDFs and office files only. "
+ "'all' – all supported types including images. "
+ "Null or empty string clears the override (inherits global setting)."
+ ),
+ )
class ImapTestRequest(BaseModel):
@@ -155,6 +173,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]:
"use_ssl": acct.use_ssl,
"delete_after_process": acct.delete_after_process,
"is_active": acct.is_active,
+ "attachment_filter": acct.attachment_filter,
"last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None,
"last_error": acct.last_error,
"created_at": acct.created_at.isoformat() if acct.created_at else None,
@@ -222,6 +241,7 @@ def create_imap_account(
use_ssl=body.use_ssl,
delete_after_process=body.delete_after_process,
is_active=body.is_active,
+ attachment_filter=body.attachment_filter or None,
)
try:
db.add(acct)
@@ -277,6 +297,12 @@ def update_imap_account(
acct.delete_after_process = body.delete_after_process
if body.is_active is not None:
acct.is_active = body.is_active
+ # attachment_filter uses a sentinel check: the field is always present in the
+ # model (defaulting to None in Pydantic) so we update it unconditionally when
+ # the caller sends any value (including explicit null to clear the override).
+ # An empty string is normalised to None to avoid storing a non-meaningful value.
+ if "attachment_filter" in body.model_fields_set:
+ acct.attachment_filter = body.attachment_filter or None
# Reset last_error so the next poll gives a fresh result
acct.last_error = None
diff --git a/app/config.py b/app/config.py
index 7ffd15e1..884e1a4d 100644
--- a/app/config.py
+++ b/app/config.py
@@ -571,6 +571,17 @@ class Settings(BaseSettings):
),
)
+ imap_attachment_filter: str = Field(
+ default="documents_only",
+ description=(
+ "Controls which attachment types are ingested from IMAP emails. "
+ "Accepted values: "
+ "'documents_only' – ingest only PDFs and office files (Word, Excel, PowerPoint, ODT, etc.); "
+ "'all' – ingest all supported file types including images. "
+ "This is the global default; individual user IMAP accounts can override it."
+ ),
+ )
+
# Batch processing settings
processall_throttle_threshold: int = Field(
default=20,
diff --git a/app/models.py b/app/models.py
index 9b3ea4d1..94cb4e62 100644
--- a/app/models.py
+++ b/app/models.py
@@ -439,6 +439,11 @@ class UserImapAccount(Base):
# When True, emails are deleted from the mailbox after their attachments are processed
delete_after_process = Column(Boolean, nullable=False, default=False)
+ # Override for which attachment types to ingest.
+ # NULL means "inherit the global imap_attachment_filter setting".
+ # Allowed values: 'documents_only', 'all'
+ attachment_filter = Column(String(50), nullable=True, default=None)
+
# When False the account is not polled by the periodic task (but not deleted)
is_active = Column(Boolean, nullable=False, default=True)
diff --git a/app/tasks/imap_tasks.py b/app/tasks/imap_tasks.py
index 064ccf06..be9f3bd4 100644
--- a/app/tasks/imap_tasks.py
+++ b/app/tasks/imap_tasks.py
@@ -13,7 +13,7 @@ from celery import shared_task
from app.config import settings
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
from app.tasks.process_document import process_document # Updated import
-from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES
+from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, DOCUMENT_MIME_TYPES, IMAGE_MIME_TYPES
# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
_db_session_factory = None
@@ -180,6 +180,7 @@ def _pull_user_imap_accounts() -> None:
use_ssl=acct.use_ssl,
delete_after_process=acct.delete_after_process,
owner_id=acct.owner_id,
+ attachment_filter=acct.attachment_filter or settings.imap_attachment_filter,
)
# Record successful poll
acct.last_checked_at = datetime.now(timezone.utc)
@@ -250,6 +251,7 @@ def _pull_user_integration_imap() -> None:
use_ssl = cfg.get("use_ssl", True)
delete_after = cfg.get("delete_after_process", False)
gmail_labels = cfg.get("gmail_apply_labels", True)
+ attachment_filter = cfg.get("attachment_filter") or settings.imap_attachment_filter
if not (host and username and password):
logger.warning(
@@ -269,6 +271,7 @@ def _pull_user_integration_imap() -> None:
delete_after_process=delete_after,
owner_id=integ.owner_id,
gmail_apply_labels=gmail_labels,
+ attachment_filter=attachment_filter,
)
integ.last_used_at = datetime.now(timezone.utc)
integ.last_error = None
@@ -329,6 +332,7 @@ def pull_inbox(
delete_after_process,
owner_id=None,
gmail_apply_labels=True,
+ attachment_filter=None,
):
"""
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
@@ -345,7 +349,12 @@ def pull_inbox(
attributed to this user via ``process_document`` / ``convert_to_pdf``.
gmail_apply_labels: Whether to apply Gmail-specific labels and stars to
processed emails. Only relevant for Gmail hosts. Defaults to True.
+ attachment_filter: Controls which attachment types to ingest.
+ ``'documents_only'`` (default) – PDFs and office files only.
+ ``'all'`` – all supported types including images.
+ ``None`` falls back to the global ``settings.imap_attachment_filter``.
"""
+ resolved_filter = attachment_filter or settings.imap_attachment_filter
logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
processed_emails = load_processed_emails()
@@ -407,7 +416,7 @@ def pull_inbox(
# Process attachments (and convert non-PDF files).
# We call the function without assigning its return value since it is not used.
- fetch_attachments_and_enqueue(email_message, owner_id=owner_id)
+ fetch_attachments_and_enqueue(email_message, owner_id=owner_id, attachment_filter=resolved_filter)
if settings.imap_readonly_mode:
logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
@@ -436,27 +445,22 @@ def pull_inbox(
logger.exception("Error pulling mailbox %s: %s", mailbox_key, e)
-def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
+def fetch_attachments_and_enqueue(
+ email_message,
+ owner_id: str | None = None,
+ attachment_filter: str | None = None,
+):
"""
Extracts attachments from the email and processes only allowed file types.
- Files are accepted if either:
- 1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR
- 2. They have a '.pdf' file extension (regardless of MIME type)
+ Files are first checked against the ``attachment_filter`` to determine which
+ broad categories are permitted, then validated against known MIME types /
+ extensions.
- Allowed file types include:
- - PDF: application/pdf or *.pdf extension
- - Microsoft Office files:
- - Word: application/msword,
- application/vnd.openxmlformats-officedocument.wordprocessingml.document
- - Excel: application/vnd.ms-excel,
- application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
- - PowerPoint: application/vnd.ms-powerpoint,
- application/vnd.openxmlformats-officedocument.presentationml.presentation
- - Other meaningful attachments:
- - Plain text: text/plain
- - CSV: text/csv
- - Rich Text Format: application/rtf, text/rtf
+ Attachment filter values:
+ - ``'documents_only'`` (default): PDFs, office files (Word, Excel, PowerPoint,
+ OpenDocument, RTF), plain text, CSV, HTML, and Markdown. Images are skipped.
+ - ``'all'``: All supported file types, including images (JPEG, PNG, GIF, etc.).
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
any other allowed file is enqueued for conversion to PDF.
@@ -465,9 +469,37 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
email_message: The parsed email message to extract attachments from.
owner_id: Optional user identifier forwarded to ``process_document`` /
``convert_to_pdf`` for multi-tenant attribution.
+ attachment_filter: Override for the filter level. Defaults to
+ ``settings.imap_attachment_filter`` when not provided.
Returns True if at least one allowed attachment was processed.
"""
+ resolved_filter = attachment_filter or settings.imap_attachment_filter
+
+ # Build the effective allowed MIME type set based on the filter
+ if resolved_filter == "all":
+ effective_mime_types = ALLOWED_MIME_TYPES
+ else:
+ # 'documents_only' (and any unrecognised value): exclude images
+ effective_mime_types = DOCUMENT_MIME_TYPES
+
+ # Build the effective allowed extensions set (images excluded for documents_only)
+ if resolved_filter == "all":
+ effective_extensions = ALLOWED_EXTENSIONS
+ else:
+ image_extensions = {
+ ".jpg",
+ ".jpeg",
+ ".png",
+ ".gif",
+ ".bmp",
+ ".tiff",
+ ".tif",
+ ".webp",
+ ".svg",
+ }
+ effective_extensions = ALLOWED_EXTENSIONS - image_extensions
+
has_attachment = False
for part in email_message.walk():
if part.get_content_maintype() == "multipart":
@@ -482,8 +514,30 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
mime_type = part.get_content_type()
file_ext = os.path.splitext(filename)[1].lower()
+
+ # Skip images when filter is documents_only
+ is_image = mime_type in IMAGE_MIME_TYPES or file_ext in {
+ ".jpg",
+ ".jpeg",
+ ".png",
+ ".gif",
+ ".bmp",
+ ".tiff",
+ ".tif",
+ ".webp",
+ ".svg",
+ }
+ if is_image and resolved_filter != "all":
+ logger.info(
+ "Skipping image attachment %s (MIME: %s) — attachment_filter=%s",
+ filename,
+ mime_type,
+ resolved_filter,
+ )
+ continue
+
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
- if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension:
+ if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension:
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
continue
@@ -495,7 +549,7 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
if mime_type == "application/pdf" or is_pdf_by_extension:
process_document.delay(file_path, owner_id=owner_id)
logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type)
- elif mime_type in ALLOWED_MIME_TYPES:
+ elif mime_type in effective_mime_types:
# Other allowed files are sent for conversion
convert_to_pdf.delay(file_path, owner_id=owner_id)
logger.info("Enqueued file for conversion to PDF: %s", filename)
diff --git a/app/utils/settings_service.py b/app/utils/settings_service.py
index 78188b9e..e9b040d0 100644
--- a/app/utils/settings_service.py
+++ b/app/utils/settings_service.py
@@ -1425,6 +1425,18 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
+ "imap_attachment_filter": {
+ "category": "IMAP",
+ "description": (
+ "Controls which attachment types are ingested from IMAP emails. "
+ "Accepted values: 'documents_only' (PDFs and office files only, default) or 'all' (including images). "
+ "Per-user IMAP accounts can override this global default."
+ ),
+ "type": "string",
+ "sensitive": False,
+ "required": False,
+ "restart_required": False,
+ },
# Monitoring - Uptime Kuma
"uptime_kuma_url": {
"category": "Monitoring",
diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md
index 398aed82..747db3ee 100644
--- a/docs/ConfigurationGuide.md
+++ b/docs/ConfigurationGuide.md
@@ -304,6 +304,7 @@ DocuElevate can automatically pull document attachments from IMAP mailboxes —
| `IMAP1_SSL` | Use SSL (`true`/`false`). | `true` |
| `IMAP1_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll for new mail. | `5` |
| `IMAP_READONLY_MODE` | When `true`, fetches and processes attachments but does **not** modify the mailbox (no starring, labeling, deleting, or flag changes). Use for pre-production instances sharing a mailbox with production. Default: `false`. | `false` |
+| `IMAP_ATTACHMENT_FILTER` | Controls which attachment types are ingested from emails. `documents_only` (default) ingests PDFs and office files only — images are skipped. `all` ingests every supported file type including images. Individual per-user IMAP accounts can override this global default. | `documents_only` |
#### Per-User IMAP Integrations
diff --git a/docs/howto/EmailIngestion.md b/docs/howto/EmailIngestion.md
index 1a911f6f..12e76eae 100644
--- a/docs/howto/EmailIngestion.md
+++ b/docs/howto/EmailIngestion.md
@@ -63,6 +63,37 @@ DocuElevate will process the following attachment types from emails:
| TIFF | `.tif`, `.tiff` | Common format from older scanners/fax |
| Multi-page TIFF | `.tif` | Full multi-page support |
+### Controlling Which Attachment Types Are Ingested
+
+By default, DocuElevate only ingests **document** attachments (PDFs, Word, Excel, PowerPoint, OpenDocument, RTF, TXT, CSV, HTML, Markdown). Images are **not** ingested by default — this prevents cluttering your document archive with inline images or unrelated photo attachments.
+
+#### Global Default (Admin Setting)
+
+Set the `IMAP_ATTACHMENT_FILTER` environment variable to control the system-wide default:
+
+| Value | Behaviour |
+|-------|-----------|
+| `documents_only` | **(Default)** Only PDFs and office/document files. Images (JPEG, PNG, GIF, BMP, TIFF, WebP, SVG) are skipped. |
+| `all` | All supported file types, including images. |
+
+```env
+# Only ingest document-type attachments (default behaviour)
+IMAP_ATTACHMENT_FILTER=documents_only
+
+# Ingest all supported file types, including images
+IMAP_ATTACHMENT_FILTER=all
+```
+
+#### Per-User Override
+
+Each user can override the global default for their personal IMAP accounts via the **Email Ingestion** dashboard (`/imap-accounts`). When creating or editing an account, select the desired setting from the **Attachment Types to Ingest** dropdown:
+
+- **Use global default** — inherits the `IMAP_ATTACHMENT_FILTER` setting above.
+- **Documents only** — PDFs and office files, no images.
+- **All supported types (including images)** — overrides the global setting to allow images for this specific account.
+
+This allows administrators to restrict image ingestion system-wide while individual users can opt-in to image ingestion on a per-mailbox basis.
+
---
## Setting Up Your Scanner/Device
diff --git a/frontend/templates/imap_accounts.html b/frontend/templates/imap_accounts.html
index e22614e6..16a6cdaa 100644
--- a/frontend/templates/imap_accounts.html
+++ b/frontend/templates/imap_accounts.html
@@ -172,6 +172,16 @@
Delete after process
+
+
+ All attachments
+
+
+
+
+ Documents only
+
+
Last polled:
@@ -386,6 +396,26 @@
+
+
+
+
+
+ Override the system-wide attachment filter for this mailbox. Leave blank to use the global default (documents only).
+
+
+
None:
+ """Add attachment_filter column to user_imap_accounts."""
+ op.add_column(
+ "user_imap_accounts",
+ sa.Column("attachment_filter", sa.String(50), nullable=True),
+ )
+
+
+def downgrade() -> None:
+ """Remove attachment_filter column from user_imap_accounts."""
+ op.drop_column("user_imap_accounts", "attachment_filter")
diff --git a/tests/test_imap_tasks.py b/tests/test_imap_tasks.py
index b35363bc..6b78edb5 100644
--- a/tests/test_imap_tasks.py
+++ b/tests/test_imap_tasks.py
@@ -179,6 +179,79 @@ class TestFetchAttachmentsAndEnqueue:
assert result is True
mock_convert.delay.assert_called_once()
+ @patch("app.tasks.imap_tasks.process_document")
+ @patch("app.tasks.imap_tasks.convert_to_pdf")
+ def test_skips_image_when_documents_only(self, mock_convert, mock_process):
+ """Test that image attachments are skipped with documents_only filter."""
+ msg = EmailMessage()
+ msg["Subject"] = "Photo"
+ msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
+
+ result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
+ assert result is False
+ mock_process.delay.assert_not_called()
+ mock_convert.delay.assert_not_called()
+
+ @patch("app.tasks.imap_tasks.process_document")
+ @patch("app.tasks.imap_tasks.convert_to_pdf")
+ def test_processes_image_when_all_filter(self, mock_convert, mock_process, tmp_path):
+ """Test that image attachments are processed with 'all' filter."""
+ msg = EmailMessage()
+ msg["Subject"] = "Photo"
+ msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
+
+ with patch("app.tasks.imap_tasks.settings") as mock_settings:
+ mock_settings.workdir = str(tmp_path)
+ result = fetch_attachments_and_enqueue(msg, attachment_filter="all")
+
+ assert result is True
+ mock_convert.delay.assert_called_once()
+
+ @patch("app.tasks.imap_tasks.process_document")
+ @patch("app.tasks.imap_tasks.convert_to_pdf")
+ def test_skips_image_with_image_extension_documents_only(self, mock_convert, mock_process):
+ """Test image files identified by extension are skipped with documents_only."""
+ msg = EmailMessage()
+ msg["Subject"] = "Screenshot"
+ msg.add_attachment(b"\x89PNG", maintype="application", subtype="octet-stream", filename="screenshot.png")
+
+ result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
+ assert result is False
+ mock_process.delay.assert_not_called()
+ mock_convert.delay.assert_not_called()
+
+ @patch("app.tasks.imap_tasks.process_document")
+ @patch("app.tasks.imap_tasks.convert_to_pdf")
+ def test_processes_pdf_regardless_of_filter(self, mock_convert, mock_process, tmp_path):
+ """Test that PDFs are always processed, even with documents_only filter."""
+ msg = EmailMessage()
+ msg["Subject"] = "Invoice"
+ msg.add_attachment(b"%PDF-1.4", maintype="application", subtype="pdf", filename="invoice.pdf")
+
+ with patch("app.tasks.imap_tasks.settings") as mock_settings:
+ mock_settings.workdir = str(tmp_path)
+ result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
+
+ assert result is True
+ mock_process.delay.assert_called_once()
+
+ @patch("app.tasks.imap_tasks.process_document")
+ @patch("app.tasks.imap_tasks.convert_to_pdf")
+ def test_uses_global_setting_when_no_filter(self, mock_convert, mock_process):
+ """Test that the global settings.imap_attachment_filter is used when no filter is passed."""
+ msg = EmailMessage()
+ msg["Subject"] = "Photo"
+ msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
+
+ with patch("app.tasks.imap_tasks.settings") as mock_settings:
+ mock_settings.imap_attachment_filter = "documents_only"
+ mock_settings.workdir = "/tmp"
+ result = fetch_attachments_and_enqueue(msg)
+
+ assert result is False
+ mock_process.delay.assert_not_called()
+ mock_convert.delay.assert_not_called()
+
@pytest.mark.unit
class TestEmailAlreadyHasLabel:
From c9f554465d5bf027a92bb4aa56db244aad4a6ff6 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Thu, 12 Mar 2026 01:14:19 +0000
Subject: [PATCH 3/4] feat(imap): add ImapIngestionProfile model, API,
migration and UI
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Replaces the simple binary attachment_filter string with a full
ingestion profiles system:
- Add FILE_TYPE_CATEGORIES dict to allowed_types.py (6 categories:
pdf, office, opendocument, text, web, images) + DEFAULT_CATEGORIES
/ ALL_CATEGORIES + get_allowed_types_for_categories() helper
- Add ImapIngestionProfile model (id, name, description, owner_id,
allowed_categories JSON, is_builtin)
- Update UserImapAccount: replace attachment_filter string with
profile_id FK to imap_ingestion_profiles
- Migration 033: creates profiles table, seeds 2 built-in profiles
(Documents Only, All Files), migrates attachment_filter → profile_id
- New /api/imap-profiles/ CRUD endpoints (list, create, get, update,
delete) with category validation
- Register imap_profiles router in app/api/__init__.py
- Update imap_tasks.py: replace attachment_filter string param with
profile-based allowed_categories; add _resolve_categories_for_profile()
- Update imap_accounts.py API to use profile_id instead of attachment_filter
- Update imap_accounts view to pass profiles + categories to template
- Full UI overhaul: profiles panel + profile create/edit modal with
category checkboxes; profile selector in account modal
- 17 new tests (141 total), all passing
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
---
app/api/__init__.py | 2 +
app/api/imap_accounts.py | 28 +-
app/api/imap_profiles.py | 256 +++++++++
app/models.py | 49 +-
app/tasks/imap_tasks.py | 150 +++---
app/utils/allowed_types.py | 154 ++++++
app/views/imap_accounts.py | 48 +-
frontend/templates/imap_accounts.html | 493 +++++++++++++++++-
migrations/env.py | 1 +
.../033_add_imap_ingestion_profiles.py | 146 ++++++
tests/test_imap_profiles.py | 160 ++++++
tests/test_imap_tasks.py | 35 +-
12 files changed, 1387 insertions(+), 135 deletions(-)
create mode 100644 app/api/imap_profiles.py
create mode 100644 migrations/versions/033_add_imap_ingestion_profiles.py
create mode 100644 tests/test_imap_profiles.py
diff --git a/app/api/__init__.py b/app/api/__init__.py
index ca331229..28b9fbba 100644
--- a/app/api/__init__.py
+++ b/app/api/__init__.py
@@ -21,6 +21,7 @@ from app.api.files import router as files_router
from app.api.google_drive import router as google_drive_router
from app.api.i18n import router as i18n_router
from app.api.imap_accounts import router as imap_accounts_router
+from app.api.imap_profiles import router as imap_profiles_router
from app.api.integrations import router as integrations_router
from app.api.logs import router as logs_router
from app.api.mobile import router as mobile_router
@@ -83,6 +84,7 @@ router.include_router(onboarding_router)
router.include_router(billing_router)
router.include_router(pipelines_router)
router.include_router(imap_accounts_router)
+router.include_router(imap_profiles_router)
router.include_router(integrations_router)
router.include_router(notifications_router)
router.include_router(scheduled_jobs_router)
diff --git a/app/api/imap_accounts.py b/app/api/imap_accounts.py
index 169fdef7..2aa9b039 100644
--- a/app/api/imap_accounts.py
+++ b/app/api/imap_accounts.py
@@ -110,12 +110,10 @@ class ImapAccountCreate(BaseModel):
use_ssl: bool = Field(default=True, description="Use SSL/TLS connection")
delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing")
is_active: bool = Field(default=True, description="Whether to poll this mailbox")
- attachment_filter: str | None = Field(
+ profile_id: int | None = Field(
default=None,
description=(
- "Controls which attachment types to ingest. "
- "'documents_only' – PDFs and office files only (default when None). "
- "'all' – all supported types including images. "
+ "ID of the ImapIngestionProfile that controls which attachment types to ingest. "
"Null inherits the global imap_attachment_filter setting."
),
)
@@ -132,13 +130,11 @@ class ImapAccountUpdate(BaseModel):
use_ssl: bool | None = None
delete_after_process: bool | None = None
is_active: bool | None = None
- attachment_filter: str | None = Field(
+ profile_id: int | None = Field(
default=None,
description=(
- "Controls which attachment types to ingest. "
- "'documents_only' – PDFs and office files only. "
- "'all' – all supported types including images. "
- "Null or empty string clears the override (inherits global setting)."
+ "ID of the ImapIngestionProfile to use. "
+ "Explicitly sending null clears the override (falls back to global setting)."
),
)
@@ -173,7 +169,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]:
"use_ssl": acct.use_ssl,
"delete_after_process": acct.delete_after_process,
"is_active": acct.is_active,
- "attachment_filter": acct.attachment_filter,
+ "profile_id": acct.profile_id,
"last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None,
"last_error": acct.last_error,
"created_at": acct.created_at.isoformat() if acct.created_at else None,
@@ -241,7 +237,7 @@ def create_imap_account(
use_ssl=body.use_ssl,
delete_after_process=body.delete_after_process,
is_active=body.is_active,
- attachment_filter=body.attachment_filter or None,
+ profile_id=body.profile_id,
)
try:
db.add(acct)
@@ -297,12 +293,10 @@ def update_imap_account(
acct.delete_after_process = body.delete_after_process
if body.is_active is not None:
acct.is_active = body.is_active
- # attachment_filter uses a sentinel check: the field is always present in the
- # model (defaulting to None in Pydantic) so we update it unconditionally when
- # the caller sends any value (including explicit null to clear the override).
- # An empty string is normalised to None to avoid storing a non-meaningful value.
- if "attachment_filter" in body.model_fields_set:
- acct.attachment_filter = body.attachment_filter or None
+ # profile_id: update whenever the field is explicitly present in the request payload
+ # (including sending null to clear the override).
+ if "profile_id" in body.model_fields_set:
+ acct.profile_id = body.profile_id
# Reset last_error so the next poll gives a fresh result
acct.last_error = None
diff --git a/app/api/imap_profiles.py b/app/api/imap_profiles.py
new file mode 100644
index 00000000..8f267f63
--- /dev/null
+++ b/app/api/imap_profiles.py
@@ -0,0 +1,256 @@
+"""API endpoints for managing IMAP ingestion profiles.
+
+Ingestion profiles allow fine-grained control over which attachment types are
+accepted when ingesting emails via IMAP. Each profile carries a list of enabled
+file-type categories (e.g. ``["pdf", "office", "images"]``) drawn from the
+canonical set defined in :mod:`app.utils.allowed_types`.
+
+Built-in system profiles (``is_builtin=True``) are read-only and cannot be
+deleted or modified. Users may create their own profiles which are private to
+their ``owner_id``. System-level global profiles (``owner_id=None``) are visible
+to all users but can only be created by administrators.
+"""
+
+import json
+import logging
+from datetime import datetime, timezone
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, HTTPException, Request, status
+from pydantic import BaseModel, Field
+from sqlalchemy.orm import Session
+
+from app.database import get_db
+from app.models import ImapIngestionProfile
+from app.utils.allowed_types import FILE_TYPE_CATEGORIES
+from app.utils.user_scope import get_current_owner_id
+
+logger = logging.getLogger(__name__)
+router = APIRouter(prefix="/imap-profiles", tags=["imap-profiles"])
+
+DbSession = Annotated[Session, Depends(get_db)]
+
+# ---------------------------------------------------------------------------
+# Auth helpers
+# ---------------------------------------------------------------------------
+
+
+def _get_owner_id(request: Request) -> str:
+ """Return the current user's owner ID, raising 401 if unauthenticated."""
+ owner_id = get_current_owner_id(request)
+ if owner_id is None:
+ raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
+ return owner_id
+
+
+CurrentOwner = Annotated[str, Depends(_get_owner_id)]
+
+# ---------------------------------------------------------------------------
+# Pydantic schemas
+# ---------------------------------------------------------------------------
+
+_VALID_CATEGORIES = set(FILE_TYPE_CATEGORIES.keys())
+
+
+class ImapProfileCreate(BaseModel):
+ """Schema for creating a new ingestion profile."""
+
+ name: str = Field(..., min_length=1, max_length=255, description="Human-readable profile name")
+ description: str | None = Field(default=None, description="Optional description")
+ allowed_categories: list[str] = Field(
+ ...,
+ min_length=1,
+ description=(f"List of enabled file-type category keys. Valid values: {sorted(_VALID_CATEGORIES)}"),
+ )
+
+
+class ImapProfileUpdate(BaseModel):
+ """Schema for updating an existing profile (all fields optional)."""
+
+ name: str | None = Field(default=None, min_length=1, max_length=255)
+ description: str | None = None
+ allowed_categories: list[str] | None = Field(default=None, min_length=1)
+
+
+# ---------------------------------------------------------------------------
+# Validation helpers
+# ---------------------------------------------------------------------------
+
+
+def _validate_categories(categories: list[str]) -> list[str]:
+ """Raise 422 if any category key is unknown; return the cleaned list."""
+ unknown = [c for c in categories if c not in _VALID_CATEGORIES]
+ if unknown:
+ raise HTTPException(
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+ detail=f"Unknown category key(s): {unknown}. Valid keys: {sorted(_VALID_CATEGORIES)}",
+ )
+ # Deduplicate while preserving order
+ seen: set[str] = set()
+ result: list[str] = []
+ for cat in categories:
+ if cat not in seen:
+ seen.add(cat)
+ result.append(cat)
+ return result
+
+
+# ---------------------------------------------------------------------------
+# Serialisation
+# ---------------------------------------------------------------------------
+
+
+def _to_response(profile: ImapIngestionProfile) -> dict[str, Any]:
+ """Serialize a profile row to a response dict."""
+ try:
+ categories = json.loads(profile.allowed_categories)
+ except (ValueError, TypeError):
+ categories = []
+
+ # Enrich categories with display metadata
+ categories_detail = [
+ {
+ "key": cat,
+ "label": FILE_TYPE_CATEGORIES[cat]["label"] if cat in FILE_TYPE_CATEGORIES else cat,
+ "description": FILE_TYPE_CATEGORIES[cat]["description"] if cat in FILE_TYPE_CATEGORIES else "",
+ }
+ for cat in categories
+ ]
+
+ return {
+ "id": profile.id,
+ "name": profile.name,
+ "description": profile.description,
+ "owner_id": profile.owner_id,
+ "allowed_categories": categories,
+ "categories_detail": categories_detail,
+ "is_builtin": profile.is_builtin,
+ "created_at": profile.created_at.isoformat() if profile.created_at else None,
+ "updated_at": profile.updated_at.isoformat() if profile.updated_at else None,
+ }
+
+
+# ---------------------------------------------------------------------------
+# Endpoints
+# ---------------------------------------------------------------------------
+
+
+@router.get("/categories", summary="List available file-type categories")
+def list_categories(request: Request, owner_id: CurrentOwner) -> list[dict[str, Any]]:
+ """Return the full list of file-type categories that can be used in profiles."""
+ return [
+ {
+ "key": key,
+ "label": info["label"],
+ "description": info["description"],
+ }
+ for key, info in FILE_TYPE_CATEGORIES.items()
+ ]
+
+
+@router.get("/", summary="List ingestion profiles visible to the current user")
+def list_profiles(request: Request, db: DbSession, owner_id: CurrentOwner) -> list[dict[str, Any]]:
+ """Return all profiles: system-global (owner_id=NULL) and the user's own profiles."""
+ profiles = (
+ db.query(ImapIngestionProfile)
+ .filter(
+ (ImapIngestionProfile.owner_id == None) | (ImapIngestionProfile.owner_id == owner_id) # noqa: E711
+ )
+ .order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
+ .all()
+ )
+ return [_to_response(p) for p in profiles]
+
+
+@router.post("/", status_code=status.HTTP_201_CREATED, summary="Create a new ingestion profile")
+def create_profile(request: Request, body: ImapProfileCreate, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
+ """Create a new ingestion profile owned by the current user."""
+ categories = _validate_categories(body.allowed_categories)
+
+ profile = ImapIngestionProfile(
+ name=body.name,
+ description=body.description,
+ owner_id=owner_id,
+ allowed_categories=json.dumps(categories),
+ is_builtin=False,
+ )
+ try:
+ db.add(profile)
+ db.commit()
+ db.refresh(profile)
+ except Exception:
+ db.rollback()
+ raise
+
+ logger.info("User %s created IMAP ingestion profile %d ('%s')", owner_id, profile.id, body.name)
+ return _to_response(profile)
+
+
+@router.get("/{profile_id}", summary="Get a single ingestion profile")
+def get_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> dict[str, Any]:
+ """Return a single profile by ID. Only the owner or system profiles are accessible."""
+ profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
+ if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
+ return _to_response(profile)
+
+
+@router.put("/{profile_id}", summary="Update an ingestion profile")
+def update_profile(
+ profile_id: int,
+ request: Request,
+ body: ImapProfileUpdate,
+ db: DbSession,
+ owner_id: CurrentOwner,
+) -> dict[str, Any]:
+ """Update an existing ingestion profile. Built-in profiles cannot be modified."""
+ profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
+ if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
+ if profile.is_builtin:
+ raise HTTPException(
+ status_code=status.HTTP_403_FORBIDDEN,
+ detail="Built-in profiles cannot be modified.",
+ )
+
+ if body.name is not None:
+ profile.name = body.name
+ if "description" in body.model_fields_set:
+ profile.description = body.description
+ if body.allowed_categories is not None:
+ categories = _validate_categories(body.allowed_categories)
+ profile.allowed_categories = json.dumps(categories)
+
+ profile.updated_at = datetime.now(timezone.utc)
+
+ try:
+ db.commit()
+ db.refresh(profile)
+ except Exception:
+ db.rollback()
+ raise
+
+ logger.info("User %s updated IMAP ingestion profile %d", owner_id, profile_id)
+ return _to_response(profile)
+
+
+@router.delete("/{profile_id}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete an ingestion profile")
+def delete_profile(profile_id: int, request: Request, db: DbSession, owner_id: CurrentOwner) -> None:
+ """Delete an ingestion profile. Built-in profiles cannot be deleted."""
+ profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
+ if not profile or (profile.owner_id is not None and profile.owner_id != owner_id):
+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Ingestion profile not found")
+ if profile.is_builtin:
+ raise HTTPException(
+ status_code=status.HTTP_403_FORBIDDEN,
+ detail="Built-in profiles cannot be deleted.",
+ )
+
+ try:
+ db.delete(profile)
+ db.commit()
+ except Exception:
+ db.rollback()
+ raise
+
+ logger.info("User %s deleted IMAP ingestion profile %d", owner_id, profile_id)
diff --git a/app/models.py b/app/models.py
index 94cb4e62..5ba06e81 100644
--- a/app/models.py
+++ b/app/models.py
@@ -401,6 +401,48 @@ class PipelineStep(Base):
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+class ImapIngestionProfile(Base):
+ """Named ingestion profile controlling which attachment types are accepted from IMAP emails.
+
+ Profiles group file-type categories (e.g. "pdf", "office", "images") so users
+ can precisely control what gets ingested from each mailbox.
+
+ System-provided built-in profiles (``is_builtin=True``) are seeded by the
+ migration and cannot be deleted or renamed. Users may create their own profiles
+ (``owner_id`` set to their identifier) or rely on the global system profiles
+ (``owner_id=None``).
+
+ ``allowed_categories`` stores a JSON list of category strings, e.g.::
+
+ '["pdf", "office", "opendocument", "text", "web"]'
+
+ Valid category names are defined in ``app.utils.allowed_types.FILE_TYPE_CATEGORIES``.
+ """
+
+ __tablename__ = "imap_ingestion_profiles"
+
+ id = Column(Integer, primary_key=True, index=True)
+
+ # Human-readable profile name (e.g. "Documents Only", "Documents + Images")
+ name = Column(String(255), nullable=False)
+
+ # Optional description shown in the UI
+ description = Column(Text, nullable=True)
+
+ # Owner of this profile. NULL = global/system profile available to all users.
+ owner_id = Column(String, nullable=True, index=True)
+
+ # JSON-encoded list of enabled category keys. Example: '["pdf","office","text"]'
+ # See FILE_TYPE_CATEGORIES in app/utils/allowed_types.py for valid values.
+ allowed_categories = Column(Text, nullable=False, default='["pdf","office","opendocument","text","web"]')
+
+ # Built-in system profiles that cannot be deleted or modified via the API.
+ is_builtin = Column(Boolean, nullable=False, default=False)
+
+ created_at = Column(DateTime(timezone=True), server_default=func.now())
+ updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+
class UserImapAccount(Base):
"""Per-user IMAP ingestion account.
@@ -439,10 +481,9 @@ class UserImapAccount(Base):
# When True, emails are deleted from the mailbox after their attachments are processed
delete_after_process = Column(Boolean, nullable=False, default=False)
- # Override for which attachment types to ingest.
- # NULL means "inherit the global imap_attachment_filter setting".
- # Allowed values: 'documents_only', 'all'
- attachment_filter = Column(String(50), nullable=True, default=None)
+ # Optional reference to an ImapIngestionProfile.
+ # NULL means "use the global imap_attachment_filter setting" (system default).
+ profile_id = Column(Integer, ForeignKey("imap_ingestion_profiles.id"), nullable=True)
# When False the account is not polled by the periodic task (but not deleted)
is_active = Column(Boolean, nullable=False, default=True)
diff --git a/app/tasks/imap_tasks.py b/app/tasks/imap_tasks.py
index be9f3bd4..affd7a9d 100644
--- a/app/tasks/imap_tasks.py
+++ b/app/tasks/imap_tasks.py
@@ -13,7 +13,11 @@ from celery import shared_task
from app.config import settings
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
from app.tasks.process_document import process_document # Updated import
-from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, DOCUMENT_MIME_TYPES, IMAGE_MIME_TYPES
+from app.utils.allowed_types import (
+ ALL_CATEGORIES,
+ DEFAULT_CATEGORIES,
+ get_allowed_types_for_categories,
+)
# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
_db_session_factory = None
@@ -50,6 +54,33 @@ def _decrypt_imap_password(password: str | None) -> str | None:
return decrypt_value(password)
+def _resolve_categories_for_profile(profile_id: int | None) -> list[str]:
+ """Return the list of allowed categories for a profile ID.
+
+ Loads the profile from the database. If ``profile_id`` is ``None`` or the
+ profile is not found, falls back to the global ``settings.imap_attachment_filter``
+ string (``'documents_only'`` → default categories; ``'all'`` → all categories).
+ """
+ if profile_id is not None:
+ try:
+ from app.models import ImapIngestionProfile
+
+ db = _get_db_session()
+ try:
+ profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
+ if profile:
+ return json.loads(profile.allowed_categories)
+ finally:
+ db.close()
+ except Exception as exc: # noqa: BLE001
+ logger.warning("Could not load IMAP ingestion profile %d: %s — using global default", profile_id, exc)
+
+ # Fall back to global setting
+ if settings.imap_attachment_filter == "all":
+ return ALL_CATEGORIES
+ return DEFAULT_CATEGORIES
+
+
LOCK_KEY = "imap_lock" # Unique key for locking
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
@@ -180,7 +211,7 @@ def _pull_user_imap_accounts() -> None:
use_ssl=acct.use_ssl,
delete_after_process=acct.delete_after_process,
owner_id=acct.owner_id,
- attachment_filter=acct.attachment_filter or settings.imap_attachment_filter,
+ allowed_categories=_resolve_categories_for_profile(acct.profile_id),
)
# Record successful poll
acct.last_checked_at = datetime.now(timezone.utc)
@@ -251,7 +282,9 @@ def _pull_user_integration_imap() -> None:
use_ssl = cfg.get("use_ssl", True)
delete_after = cfg.get("delete_after_process", False)
gmail_labels = cfg.get("gmail_apply_labels", True)
- attachment_filter = cfg.get("attachment_filter") or settings.imap_attachment_filter
+ # Integrations can store a profile_id in config; fall back to global default
+ profile_id = cfg.get("profile_id")
+ allowed_categories = _resolve_categories_for_profile(profile_id)
if not (host and username and password):
logger.warning(
@@ -271,7 +304,7 @@ def _pull_user_integration_imap() -> None:
delete_after_process=delete_after,
owner_id=integ.owner_id,
gmail_apply_labels=gmail_labels,
- attachment_filter=attachment_filter,
+ allowed_categories=allowed_categories,
)
integ.last_used_at = datetime.now(timezone.utc)
integ.last_error = None
@@ -332,7 +365,7 @@ def pull_inbox(
delete_after_process,
owner_id=None,
gmail_apply_labels=True,
- attachment_filter=None,
+ allowed_categories=None,
):
"""
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
@@ -349,13 +382,22 @@ def pull_inbox(
attributed to this user via ``process_document`` / ``convert_to_pdf``.
gmail_apply_labels: Whether to apply Gmail-specific labels and stars to
processed emails. Only relevant for Gmail hosts. Defaults to True.
- attachment_filter: Controls which attachment types to ingest.
- ``'documents_only'`` (default) – PDFs and office files only.
- ``'all'`` – all supported types including images.
- ``None`` falls back to the global ``settings.imap_attachment_filter``.
+ allowed_categories: List of file-type category keys to ingest (e.g.
+ ``["pdf", "office", "images"]``). ``None`` falls back to the
+ global ``settings.imap_attachment_filter`` mapping.
"""
- resolved_filter = attachment_filter or settings.imap_attachment_filter
- logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
+ if allowed_categories is None:
+ allowed_categories = _resolve_categories_for_profile(None)
+
+ effective_mime_types, effective_extensions = get_allowed_types_for_categories(allowed_categories)
+ logger.info(
+ "Connecting to %s at %s:%s (SSL=%s) — categories: %s",
+ mailbox_key,
+ host,
+ port,
+ use_ssl,
+ allowed_categories,
+ )
processed_emails = load_processed_emails()
try:
@@ -414,9 +456,13 @@ def pull_inbox(
logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
continue
- # Process attachments (and convert non-PDF files).
- # We call the function without assigning its return value since it is not used.
- fetch_attachments_and_enqueue(email_message, owner_id=owner_id, attachment_filter=resolved_filter)
+ # Process attachments using the resolved mime types / extensions.
+ fetch_attachments_and_enqueue(
+ email_message,
+ owner_id=owner_id,
+ effective_mime_types=effective_mime_types,
+ effective_extensions=effective_extensions,
+ )
if settings.imap_readonly_mode:
logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
@@ -448,19 +494,20 @@ def pull_inbox(
def fetch_attachments_and_enqueue(
email_message,
owner_id: str | None = None,
- attachment_filter: str | None = None,
+ effective_mime_types: frozenset[str] | None = None,
+ effective_extensions: frozenset[str] | None = None,
):
"""
Extracts attachments from the email and processes only allowed file types.
- Files are first checked against the ``attachment_filter`` to determine which
- broad categories are permitted, then validated against known MIME types /
- extensions.
+ The caller is responsible for computing ``effective_mime_types`` and
+ ``effective_extensions`` from the relevant :class:`ImapIngestionProfile` (or
+ the global default) via :func:`app.utils.allowed_types.get_allowed_types_for_categories`
+ before calling this function. ``pull_inbox`` does this automatically.
- Attachment filter values:
- - ``'documents_only'`` (default): PDFs, office files (Word, Excel, PowerPoint,
- OpenDocument, RTF), plain text, CSV, HTML, and Markdown. Images are skipped.
- - ``'all'``: All supported file types, including images (JPEG, PNG, GIF, etc.).
+ If either set is ``None`` the function falls back to the default category list
+ so the function still works correctly when called directly in tests or from
+ other contexts.
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
any other allowed file is enqueued for conversion to PDF.
@@ -469,36 +516,13 @@ def fetch_attachments_and_enqueue(
email_message: The parsed email message to extract attachments from.
owner_id: Optional user identifier forwarded to ``process_document`` /
``convert_to_pdf`` for multi-tenant attribution.
- attachment_filter: Override for the filter level. Defaults to
- ``settings.imap_attachment_filter`` when not provided.
+ effective_mime_types: Pre-computed frozenset of allowed MIME type strings.
+ effective_extensions: Pre-computed frozenset of allowed file extension strings.
Returns True if at least one allowed attachment was processed.
"""
- resolved_filter = attachment_filter or settings.imap_attachment_filter
-
- # Build the effective allowed MIME type set based on the filter
- if resolved_filter == "all":
- effective_mime_types = ALLOWED_MIME_TYPES
- else:
- # 'documents_only' (and any unrecognised value): exclude images
- effective_mime_types = DOCUMENT_MIME_TYPES
-
- # Build the effective allowed extensions set (images excluded for documents_only)
- if resolved_filter == "all":
- effective_extensions = ALLOWED_EXTENSIONS
- else:
- image_extensions = {
- ".jpg",
- ".jpeg",
- ".png",
- ".gif",
- ".bmp",
- ".tiff",
- ".tif",
- ".webp",
- ".svg",
- }
- effective_extensions = ALLOWED_EXTENSIONS - image_extensions
+ if effective_mime_types is None or effective_extensions is None:
+ effective_mime_types, effective_extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
has_attachment = False
for part in email_message.walk():
@@ -515,30 +539,14 @@ def fetch_attachments_and_enqueue(
mime_type = part.get_content_type()
file_ext = os.path.splitext(filename)[1].lower()
- # Skip images when filter is documents_only
- is_image = mime_type in IMAGE_MIME_TYPES or file_ext in {
- ".jpg",
- ".jpeg",
- ".png",
- ".gif",
- ".bmp",
- ".tiff",
- ".tif",
- ".webp",
- ".svg",
- }
- if is_image and resolved_filter != "all":
- logger.info(
- "Skipping image attachment %s (MIME: %s) — attachment_filter=%s",
- filename,
- mime_type,
- resolved_filter,
- )
- continue
-
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension:
- logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
+ logger.info(
+ "Skipping attachment %s (MIME: %s, ext: %s) — not in effective allowed set",
+ filename,
+ mime_type,
+ file_ext,
+ )
continue
file_path = os.path.join(settings.workdir, filename)
diff --git a/app/utils/allowed_types.py b/app/utils/allowed_types.py
index 733ff10e..eaa37e6e 100644
--- a/app/utils/allowed_types.py
+++ b/app/utils/allowed_types.py
@@ -131,3 +131,157 @@ ALLOWED_EXTENSIONS: set[str] = {
".md",
".markdown",
}
+
+# ---------------------------------------------------------------------------
+# Fine-grained file-type categories used by IMAP ingestion profiles.
+# Each category groups related MIME types and extensions so that users can
+# enable/disable a logical collection of formats (e.g. "images") rather than
+# having to manage individual MIME strings.
+# ---------------------------------------------------------------------------
+
+FILE_TYPE_CATEGORIES: dict[str, dict] = {
+ "pdf": {
+ "label": "PDF",
+ "description": "PDF documents (.pdf)",
+ "mime_types": frozenset({"application/pdf"}),
+ "extensions": frozenset({".pdf"}),
+ },
+ "office": {
+ "label": "Microsoft Office",
+ "description": "Word, Excel and PowerPoint files (.doc, .docx, .xls, .xlsx, .ppt, .pptx, …)",
+ "mime_types": frozenset(
+ {
+ "application/msword",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
+ "application/vnd.ms-word.document.macroEnabled.12",
+ "application/vnd.ms-word.template.macroEnabled.12",
+ "application/vnd.ms-excel",
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
+ "application/vnd.ms-excel.sheet.macroEnabled.12",
+ "application/vnd.ms-excel.sheet.binary.macroEnabled.12",
+ "application/vnd.ms-powerpoint",
+ "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+ "application/vnd.openxmlformats-officedocument.presentationml.template",
+ "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
+ "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
+ }
+ ),
+ "extensions": frozenset(
+ {
+ ".doc",
+ ".docx",
+ ".docm",
+ ".dot",
+ ".dotx",
+ ".dotm",
+ ".xls",
+ ".xlsx",
+ ".xlsm",
+ ".xlsb",
+ ".xlt",
+ ".xltx",
+ ".xlw",
+ ".ppt",
+ ".pptx",
+ ".pptm",
+ ".pps",
+ ".ppsx",
+ ".pot",
+ ".potx",
+ }
+ ),
+ },
+ "opendocument": {
+ "label": "OpenDocument (LibreOffice)",
+ "description": "LibreOffice / OpenOffice files (.odt, .ods, .odp, …)",
+ "mime_types": frozenset(
+ {
+ "application/vnd.oasis.opendocument.text",
+ "application/vnd.oasis.opendocument.spreadsheet",
+ "application/vnd.oasis.opendocument.presentation",
+ "application/vnd.oasis.opendocument.graphics",
+ "application/vnd.oasis.opendocument.formula",
+ }
+ ),
+ "extensions": frozenset({".odt", ".ods", ".odp", ".odg", ".odf"}),
+ },
+ "text": {
+ "label": "Text & Data",
+ "description": "Plain text, CSV and RTF files (.txt, .csv, .rtf)",
+ "mime_types": frozenset(
+ {
+ "text/plain",
+ "text/csv",
+ "application/rtf",
+ "text/rtf",
+ }
+ ),
+ "extensions": frozenset({".txt", ".csv", ".rtf"}),
+ },
+ "web": {
+ "label": "Web & Markup",
+ "description": "HTML and Markdown files (.html, .htm, .md, .markdown)",
+ "mime_types": frozenset(
+ {
+ "text/html",
+ "text/markdown",
+ "text/x-markdown",
+ }
+ ),
+ "extensions": frozenset({".html", ".htm", ".md", ".markdown"}),
+ },
+ "images": {
+ "label": "Images",
+ "description": "Image files (.jpg, .png, .gif, .bmp, .tiff, .webp, .svg)",
+ "mime_types": frozenset(
+ {
+ "image/jpeg",
+ "image/jpg",
+ "image/png",
+ "image/gif",
+ "image/bmp",
+ "image/tiff",
+ "image/webp",
+ "image/svg+xml",
+ }
+ ),
+ "extensions": frozenset(
+ {
+ ".jpg",
+ ".jpeg",
+ ".png",
+ ".gif",
+ ".bmp",
+ ".tiff",
+ ".tif",
+ ".webp",
+ ".svg",
+ }
+ ),
+ },
+}
+
+# Default categories for the "documents only" built-in profile (no images)
+DEFAULT_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web"]
+# All categories including images
+ALL_CATEGORIES: list[str] = ["pdf", "office", "opendocument", "text", "web", "images"]
+
+
+def get_allowed_types_for_categories(
+ categories: list[str],
+) -> tuple[frozenset[str], frozenset[str]]:
+ """Return ``(mime_types, extensions)`` for the given category list.
+
+ Unknown category names are silently ignored so that future categories
+ don't break existing profiles.
+ """
+ mime_types: set[str] = set()
+ extensions: set[str] = set()
+ for cat in categories:
+ info = FILE_TYPE_CATEGORIES.get(cat)
+ if info:
+ mime_types |= info["mime_types"]
+ extensions |= info["extensions"]
+ return frozenset(mime_types), frozenset(extensions)
diff --git a/app/views/imap_accounts.py b/app/views/imap_accounts.py
index 460c83ba..a8a2dcbd 100644
--- a/app/views/imap_accounts.py
+++ b/app/views/imap_accounts.py
@@ -1,11 +1,13 @@
"""User-facing view for the per-user IMAP ingestion dashboard."""
+import json
import logging
from fastapi import Request
from sqlalchemy.orm import Session
-from app.models import UserImapAccount
+from app.models import ImapIngestionProfile, UserImapAccount
+from app.utils.allowed_types import FILE_TYPE_CATEGORIES
from app.utils.subscription import get_tier, get_user_tier_id
from app.utils.user_scope import get_current_owner_id
from app.views.base import APIRouter, Depends, get_db, require_login, templates
@@ -25,6 +27,22 @@ def _get_max_mailboxes(tier: dict) -> int | None:
return max_mb
+def _serialize_profile(profile: ImapIngestionProfile) -> dict:
+ """Serialize a profile for JSON embedding in the template."""
+ try:
+ categories = json.loads(profile.allowed_categories)
+ except (ValueError, TypeError):
+ categories = []
+ return {
+ "id": profile.id,
+ "name": profile.name,
+ "description": profile.description,
+ "owner_id": profile.owner_id,
+ "allowed_categories": categories,
+ "is_builtin": profile.is_builtin,
+ }
+
+
@router.get("/imap-accounts")
@require_login
async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
@@ -49,11 +67,39 @@ async def imap_accounts_page(request: Request, db: Session = Depends(get_db)):
max_mailboxes = _get_max_mailboxes(tier)
can_add = max_mailboxes is None or (max_mailboxes > 0 and current_count < max_mailboxes)
+ # Load ingestion profiles: system-global + user's own
+ profiles: list[ImapIngestionProfile] = (
+ db.query(ImapIngestionProfile)
+ .filter(
+ (ImapIngestionProfile.owner_id == None) # noqa: E711
+ | (ImapIngestionProfile.owner_id == owner_id)
+ )
+ .order_by(ImapIngestionProfile.is_builtin.desc(), ImapIngestionProfile.id)
+ .all()
+ if owner_id
+ else db.query(ImapIngestionProfile)
+ .filter(ImapIngestionProfile.owner_id == None) # noqa: E711
+ .order_by(ImapIngestionProfile.id)
+ .all()
+ )
+
+ # Category definitions for the UI checkbox builder
+ categories = [
+ {
+ "key": key,
+ "label": info["label"],
+ "description": info["description"],
+ }
+ for key, info in FILE_TYPE_CATEGORIES.items()
+ ]
+
return templates.TemplateResponse(
"imap_accounts.html",
{
"request": request,
"accounts": accounts,
+ "profiles": [_serialize_profile(p) for p in profiles],
+ "categories": categories,
"current_count": current_count,
"max_mailboxes": max_mailboxes,
"can_add": can_add,
diff --git a/frontend/templates/imap_accounts.html b/frontend/templates/imap_accounts.html
index 16a6cdaa..ecbe7eaa 100644
--- a/frontend/templates/imap_accounts.html
+++ b/frontend/templates/imap_accounts.html
@@ -172,14 +172,9 @@
Delete after process
-
+
- All attachments
-
-
-
-
- Documents only
+
@@ -398,22 +393,45 @@
-
@@ -524,10 +542,276 @@
+
+
+
+
+
+
+ Ingestion Profiles
+
+
+
+
+
+ Profiles define which attachment types are ingested from emails. Assign a profile to each IMAP account for fine-grained control.
+
+
+
+
+
+
+
+
+
+
+ built-in
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Delete Profile?
+
+
+ Are you sure you want to delete the profile
+ ?
+ IMAP accounts using this profile will fall back to the global default.
+