feat(imap): add attachment type filter for IMAP ingestion

Add a configurable switch to control which attachment types are ingested
via IMAP. Images are excluded by default; office files and PDFs are ingested.

- Add global `IMAP_ATTACHMENT_FILTER` config setting (default: `documents_only`)
- Add `attachment_filter` column to `UserImapAccount` model for per-user override
- Migration 032 adds the column to `user_imap_accounts` table
- Update `fetch_attachments_and_enqueue()` to respect filter (documents_only/all)
- Update `pull_inbox()`, `_pull_user_imap_accounts()`, and
  `_pull_user_integration_imap()` to pass the resolved filter
- Update IMAP accounts API (schemas, create/update handlers, response serializer)
- Update IMAP accounts UI to show attachment filter dropdown in modal and
  display filter badges on account cards
- Add 6 new tests covering attachment filter behaviour
- Update ConfigurationGuide.md, EmailIngestion.md, and .env.demo

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-12 00:59:54 +00:00
parent 5e45b68cc1
commit 554bb21d32
11 changed files with 300 additions and 22 deletions
+6
View File
@@ -311,6 +311,12 @@ IMAP2_DELETE_AFTER_PROCESS=false
# Use for pre-production instances that share a mailbox with production.
IMAP_READONLY_MODE=false
# Controls which attachment types are ingested from IMAP emails.
# 'documents_only' (default) PDFs and office files only; images are skipped.
# 'all' all supported file types including images.
# Per-user IMAP accounts can override this global default.
IMAP_ATTACHMENT_FILTER=documents_only
# **Storage/Document Services**
# Amazon S3
AWS_REGION=us-east-1
+26
View File
@@ -110,6 +110,15 @@ class ImapAccountCreate(BaseModel):
use_ssl: bool = Field(default=True, description="Use SSL/TLS connection")
delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing")
is_active: bool = Field(default=True, description="Whether to poll this mailbox")
attachment_filter: str | None = Field(
default=None,
description=(
"Controls which attachment types to ingest. "
"'documents_only' PDFs and office files only (default when None). "
"'all' all supported types including images. "
"Null inherits the global imap_attachment_filter setting."
),
)
class ImapAccountUpdate(BaseModel):
@@ -123,6 +132,15 @@ class ImapAccountUpdate(BaseModel):
use_ssl: bool | None = None
delete_after_process: bool | None = None
is_active: bool | None = None
attachment_filter: str | None = Field(
default=None,
description=(
"Controls which attachment types to ingest. "
"'documents_only' PDFs and office files only. "
"'all' all supported types including images. "
"Null or empty string clears the override (inherits global setting)."
),
)
class ImapTestRequest(BaseModel):
@@ -155,6 +173,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]:
"use_ssl": acct.use_ssl,
"delete_after_process": acct.delete_after_process,
"is_active": acct.is_active,
"attachment_filter": acct.attachment_filter,
"last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None,
"last_error": acct.last_error,
"created_at": acct.created_at.isoformat() if acct.created_at else None,
@@ -222,6 +241,7 @@ def create_imap_account(
use_ssl=body.use_ssl,
delete_after_process=body.delete_after_process,
is_active=body.is_active,
attachment_filter=body.attachment_filter or None,
)
try:
db.add(acct)
@@ -277,6 +297,12 @@ def update_imap_account(
acct.delete_after_process = body.delete_after_process
if body.is_active is not None:
acct.is_active = body.is_active
# attachment_filter uses a sentinel check: the field is always present in the
# model (defaulting to None in Pydantic) so we update it unconditionally when
# the caller sends any value (including explicit null to clear the override).
# An empty string is normalised to None to avoid storing a non-meaningful value.
if "attachment_filter" in body.model_fields_set:
acct.attachment_filter = body.attachment_filter or None
# Reset last_error so the next poll gives a fresh result
acct.last_error = None
+11
View File
@@ -571,6 +571,17 @@ class Settings(BaseSettings):
),
)
imap_attachment_filter: str = Field(
default="documents_only",
description=(
"Controls which attachment types are ingested from IMAP emails. "
"Accepted values: "
"'documents_only' ingest only PDFs and office files (Word, Excel, PowerPoint, ODT, etc.); "
"'all' ingest all supported file types including images. "
"This is the global default; individual user IMAP accounts can override it."
),
)
# Batch processing settings
processall_throttle_threshold: int = Field(
default=20,
+5
View File
@@ -439,6 +439,11 @@ class UserImapAccount(Base):
# When True, emails are deleted from the mailbox after their attachments are processed
delete_after_process = Column(Boolean, nullable=False, default=False)
# Override for which attachment types to ingest.
# NULL means "inherit the global imap_attachment_filter setting".
# Allowed values: 'documents_only', 'all'
attachment_filter = Column(String(50), nullable=True, default=None)
# When False the account is not polled by the periodic task (but not deleted)
is_active = Column(Boolean, nullable=False, default=True)
+75 -21
View File
@@ -13,7 +13,7 @@ from celery import shared_task
from app.config import settings
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
from app.tasks.process_document import process_document # Updated import
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, DOCUMENT_MIME_TYPES, IMAGE_MIME_TYPES
# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
_db_session_factory = None
@@ -180,6 +180,7 @@ def _pull_user_imap_accounts() -> None:
use_ssl=acct.use_ssl,
delete_after_process=acct.delete_after_process,
owner_id=acct.owner_id,
attachment_filter=acct.attachment_filter or settings.imap_attachment_filter,
)
# Record successful poll
acct.last_checked_at = datetime.now(timezone.utc)
@@ -250,6 +251,7 @@ def _pull_user_integration_imap() -> None:
use_ssl = cfg.get("use_ssl", True)
delete_after = cfg.get("delete_after_process", False)
gmail_labels = cfg.get("gmail_apply_labels", True)
attachment_filter = cfg.get("attachment_filter") or settings.imap_attachment_filter
if not (host and username and password):
logger.warning(
@@ -269,6 +271,7 @@ def _pull_user_integration_imap() -> None:
delete_after_process=delete_after,
owner_id=integ.owner_id,
gmail_apply_labels=gmail_labels,
attachment_filter=attachment_filter,
)
integ.last_used_at = datetime.now(timezone.utc)
integ.last_error = None
@@ -329,6 +332,7 @@ def pull_inbox(
delete_after_process,
owner_id=None,
gmail_apply_labels=True,
attachment_filter=None,
):
"""
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
@@ -345,7 +349,12 @@ def pull_inbox(
attributed to this user via ``process_document`` / ``convert_to_pdf``.
gmail_apply_labels: Whether to apply Gmail-specific labels and stars to
processed emails. Only relevant for Gmail hosts. Defaults to True.
attachment_filter: Controls which attachment types to ingest.
``'documents_only'`` (default) PDFs and office files only.
``'all'`` all supported types including images.
``None`` falls back to the global ``settings.imap_attachment_filter``.
"""
resolved_filter = attachment_filter or settings.imap_attachment_filter
logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
processed_emails = load_processed_emails()
@@ -407,7 +416,7 @@ def pull_inbox(
# Process attachments (and convert non-PDF files).
# We call the function without assigning its return value since it is not used.
fetch_attachments_and_enqueue(email_message, owner_id=owner_id)
fetch_attachments_and_enqueue(email_message, owner_id=owner_id, attachment_filter=resolved_filter)
if settings.imap_readonly_mode:
logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
@@ -436,27 +445,22 @@ def pull_inbox(
logger.exception("Error pulling mailbox %s: %s", mailbox_key, e)
def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
def fetch_attachments_and_enqueue(
email_message,
owner_id: str | None = None,
attachment_filter: str | None = None,
):
"""
Extracts attachments from the email and processes only allowed file types.
Files are accepted if either:
1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR
2. They have a '.pdf' file extension (regardless of MIME type)
Files are first checked against the ``attachment_filter`` to determine which
broad categories are permitted, then validated against known MIME types /
extensions.
Allowed file types include:
- PDF: application/pdf or *.pdf extension
- Microsoft Office files:
- Word: application/msword,
application/vnd.openxmlformats-officedocument.wordprocessingml.document
- Excel: application/vnd.ms-excel,
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
- PowerPoint: application/vnd.ms-powerpoint,
application/vnd.openxmlformats-officedocument.presentationml.presentation
- Other meaningful attachments:
- Plain text: text/plain
- CSV: text/csv
- Rich Text Format: application/rtf, text/rtf
Attachment filter values:
- ``'documents_only'`` (default): PDFs, office files (Word, Excel, PowerPoint,
OpenDocument, RTF), plain text, CSV, HTML, and Markdown. Images are skipped.
- ``'all'``: All supported file types, including images (JPEG, PNG, GIF, etc.).
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
any other allowed file is enqueued for conversion to PDF.
@@ -465,9 +469,37 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
email_message: The parsed email message to extract attachments from.
owner_id: Optional user identifier forwarded to ``process_document`` /
``convert_to_pdf`` for multi-tenant attribution.
attachment_filter: Override for the filter level. Defaults to
``settings.imap_attachment_filter`` when not provided.
Returns True if at least one allowed attachment was processed.
"""
resolved_filter = attachment_filter or settings.imap_attachment_filter
# Build the effective allowed MIME type set based on the filter
if resolved_filter == "all":
effective_mime_types = ALLOWED_MIME_TYPES
else:
# 'documents_only' (and any unrecognised value): exclude images
effective_mime_types = DOCUMENT_MIME_TYPES
# Build the effective allowed extensions set (images excluded for documents_only)
if resolved_filter == "all":
effective_extensions = ALLOWED_EXTENSIONS
else:
image_extensions = {
".jpg",
".jpeg",
".png",
".gif",
".bmp",
".tiff",
".tif",
".webp",
".svg",
}
effective_extensions = ALLOWED_EXTENSIONS - image_extensions
has_attachment = False
for part in email_message.walk():
if part.get_content_maintype() == "multipart":
@@ -482,8 +514,30 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
mime_type = part.get_content_type()
file_ext = os.path.splitext(filename)[1].lower()
# Skip images when filter is documents_only
is_image = mime_type in IMAGE_MIME_TYPES or file_ext in {
".jpg",
".jpeg",
".png",
".gif",
".bmp",
".tiff",
".tif",
".webp",
".svg",
}
if is_image and resolved_filter != "all":
logger.info(
"Skipping image attachment %s (MIME: %s) — attachment_filter=%s",
filename,
mime_type,
resolved_filter,
)
continue
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension:
if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension:
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
continue
@@ -495,7 +549,7 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
if mime_type == "application/pdf" or is_pdf_by_extension:
process_document.delay(file_path, owner_id=owner_id)
logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type)
elif mime_type in ALLOWED_MIME_TYPES:
elif mime_type in effective_mime_types:
# Other allowed files are sent for conversion
convert_to_pdf.delay(file_path, owner_id=owner_id)
logger.info("Enqueued file for conversion to PDF: %s", filename)
+12
View File
@@ -1425,6 +1425,18 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
"imap_attachment_filter": {
"category": "IMAP",
"description": (
"Controls which attachment types are ingested from IMAP emails. "
"Accepted values: 'documents_only' (PDFs and office files only, default) or 'all' (including images). "
"Per-user IMAP accounts can override this global default."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Monitoring - Uptime Kuma
"uptime_kuma_url": {
"category": "Monitoring",
+1
View File
@@ -304,6 +304,7 @@ DocuElevate can automatically pull document attachments from IMAP mailboxes —
| `IMAP1_SSL` | Use SSL (`true`/`false`). | `true` |
| `IMAP1_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll for new mail. | `5` |
| `IMAP_READONLY_MODE` | When `true`, fetches and processes attachments but does **not** modify the mailbox (no starring, labeling, deleting, or flag changes). Use for pre-production instances sharing a mailbox with production. Default: `false`. | `false` |
| `IMAP_ATTACHMENT_FILTER` | Controls which attachment types are ingested from emails. `documents_only` (default) ingests PDFs and office files only — images are skipped. `all` ingests every supported file type including images. Individual per-user IMAP accounts can override this global default. | `documents_only` |
#### Per-User IMAP Integrations
+31
View File
@@ -63,6 +63,37 @@ DocuElevate will process the following attachment types from emails:
| TIFF | `.tif`, `.tiff` | Common format from older scanners/fax |
| Multi-page TIFF | `.tif` | Full multi-page support |
### Controlling Which Attachment Types Are Ingested
By default, DocuElevate only ingests **document** attachments (PDFs, Word, Excel, PowerPoint, OpenDocument, RTF, TXT, CSV, HTML, Markdown). Images are **not** ingested by default — this prevents cluttering your document archive with inline images or unrelated photo attachments.
#### Global Default (Admin Setting)
Set the `IMAP_ATTACHMENT_FILTER` environment variable to control the system-wide default:
| Value | Behaviour |
|-------|-----------|
| `documents_only` | **(Default)** Only PDFs and office/document files. Images (JPEG, PNG, GIF, BMP, TIFF, WebP, SVG) are skipped. |
| `all` | All supported file types, including images. |
```env
# Only ingest document-type attachments (default behaviour)
IMAP_ATTACHMENT_FILTER=documents_only
# Ingest all supported file types, including images
IMAP_ATTACHMENT_FILTER=all
```
#### Per-User Override
Each user can override the global default for their personal IMAP accounts via the **Email Ingestion** dashboard (`/imap-accounts`). When creating or editing an account, select the desired setting from the **Attachment Types to Ingest** dropdown:
- **Use global default** — inherits the `IMAP_ATTACHMENT_FILTER` setting above.
- **Documents only** — PDFs and office files, no images.
- **All supported types (including images)** — overrides the global setting to allow images for this specific account.
This allows administrators to restrict image ingestion system-wide while individual users can opt-in to image ingestion on a per-mailbox basis.
---
## Setting Up Your Scanner/Device
+32 -1
View File
@@ -172,6 +172,16 @@
<i class="fas fa-trash-alt mr-1" aria-hidden="true"></i>Delete after process
</span>
</template>
<template x-if="acct.attachment_filter === 'all'">
<span class="inline-flex items-center px-2 py-0.5 rounded bg-blue-50 text-blue-700">
<i class="fas fa-paperclip mr-1" aria-hidden="true"></i>All attachments
</span>
</template>
<template x-if="acct.attachment_filter === 'documents_only'">
<span class="inline-flex items-center px-2 py-0.5 rounded bg-gray-50 text-gray-600">
<i class="fas fa-file-alt mr-1" aria-hidden="true"></i>Documents only
</span>
</template>
<template x-if="acct.last_checked_at">
<span class="text-gray-400" :title="acct.last_checked_at">
<i class="fas fa-clock mr-1" aria-hidden="true"></i>Last polled: <span x-text="formatDate(acct.last_checked_at)"></span>
@@ -386,6 +396,26 @@
</label>
</div>
<!-- Attachment filter -->
<div>
<label for="acct-attachment-filter" class="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
Attachment Types to Ingest
</label>
<select
id="acct-attachment-filter"
x-model="form.attachment_filter"
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-white text-sm"
aria-describedby="acct-attachment-filter-hint"
>
<option value="">Use global default</option>
<option value="documents_only">Documents only (PDF, Office, ODT, RTF, TXT, CSV) — no images</option>
<option value="all">All supported types (including images)</option>
</select>
<p id="acct-attachment-filter-hint" class="mt-1 text-xs text-gray-400 dark:text-gray-500">
Override the system-wide attachment filter for this mailbox. Leave blank to use the global default (documents only).
</p>
</div>
<!-- Test connection result -->
<template x-if="testResult">
<div
@@ -551,7 +581,7 @@ function imapAccountsApp() {
openCreateModal() {
this.editingAccount = null;
this.form = { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true };
this.form = { name: '', host: '', port: 993, username: '', password: '', use_ssl: true, delete_after_process: false, is_active: true, attachment_filter: '' };
this.showPassword = false;
this.testResult = null;
this.formError = null;
@@ -569,6 +599,7 @@ function imapAccountsApp() {
use_ssl: acct.use_ssl,
delete_after_process: acct.delete_after_process,
is_active: acct.is_active,
attachment_filter: acct.attachment_filter || '',
};
this.showPassword = false;
this.testResult = null;
@@ -0,0 +1,28 @@
"""Add attachment_filter column to user_imap_accounts table.
Revision ID: 032_add_imap_attachment_filter
Revises: 031_add_compliance_templates
Create Date: 2026-03-12
"""
from typing import Union
import sqlalchemy as sa
from alembic import op
revision: str = "032_add_imap_attachment_filter"
down_revision: Union[str, None] = "031_add_compliance_templates"
depends_on: Union[str, None] = None
def upgrade() -> None:
"""Add attachment_filter column to user_imap_accounts."""
op.add_column(
"user_imap_accounts",
sa.Column("attachment_filter", sa.String(50), nullable=True),
)
def downgrade() -> None:
"""Remove attachment_filter column from user_imap_accounts."""
op.drop_column("user_imap_accounts", "attachment_filter")
+73
View File
@@ -179,6 +179,79 @@ class TestFetchAttachmentsAndEnqueue:
assert result is True
mock_convert.delay.assert_called_once()
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_skips_image_when_documents_only(self, mock_convert, mock_process):
"""Test that image attachments are skipped with documents_only filter."""
msg = EmailMessage()
msg["Subject"] = "Photo"
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
assert result is False
mock_process.delay.assert_not_called()
mock_convert.delay.assert_not_called()
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_processes_image_when_all_filter(self, mock_convert, mock_process, tmp_path):
"""Test that image attachments are processed with 'all' filter."""
msg = EmailMessage()
msg["Subject"] = "Photo"
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
with patch("app.tasks.imap_tasks.settings") as mock_settings:
mock_settings.workdir = str(tmp_path)
result = fetch_attachments_and_enqueue(msg, attachment_filter="all")
assert result is True
mock_convert.delay.assert_called_once()
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_skips_image_with_image_extension_documents_only(self, mock_convert, mock_process):
"""Test image files identified by extension are skipped with documents_only."""
msg = EmailMessage()
msg["Subject"] = "Screenshot"
msg.add_attachment(b"\x89PNG", maintype="application", subtype="octet-stream", filename="screenshot.png")
result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
assert result is False
mock_process.delay.assert_not_called()
mock_convert.delay.assert_not_called()
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_processes_pdf_regardless_of_filter(self, mock_convert, mock_process, tmp_path):
"""Test that PDFs are always processed, even with documents_only filter."""
msg = EmailMessage()
msg["Subject"] = "Invoice"
msg.add_attachment(b"%PDF-1.4", maintype="application", subtype="pdf", filename="invoice.pdf")
with patch("app.tasks.imap_tasks.settings") as mock_settings:
mock_settings.workdir = str(tmp_path)
result = fetch_attachments_and_enqueue(msg, attachment_filter="documents_only")
assert result is True
mock_process.delay.assert_called_once()
@patch("app.tasks.imap_tasks.process_document")
@patch("app.tasks.imap_tasks.convert_to_pdf")
def test_uses_global_setting_when_no_filter(self, mock_convert, mock_process):
"""Test that the global settings.imap_attachment_filter is used when no filter is passed."""
msg = EmailMessage()
msg["Subject"] = "Photo"
msg.add_attachment(b"\xff\xd8\xff", maintype="image", subtype="jpeg", filename="photo.jpg")
with patch("app.tasks.imap_tasks.settings") as mock_settings:
mock_settings.imap_attachment_filter = "documents_only"
mock_settings.workdir = "/tmp"
result = fetch_attachments_and_enqueue(msg)
assert result is False
mock_process.delay.assert_not_called()
mock_convert.delay.assert_not_called()
@pytest.mark.unit
class TestEmailAlreadyHasLabel: