gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(imap): add attachment type filter for IMAP ingestion

Browse Source 
Add a configurable switch to control which attachment types are ingested
via IMAP. Images are excluded by default; office files and PDFs are ingested.

- Add global `IMAP_ATTACHMENT_FILTER` config setting (default: `documents_only`)
- Add `attachment_filter` column to `UserImapAccount` model for per-user override
- Migration 032 adds the column to `user_imap_accounts` table
- Update `fetch_attachments_and_enqueue()` to respect filter (documents_only/all)
- Update `pull_inbox()`, `_pull_user_imap_accounts()`, and
  `_pull_user_integration_imap()` to pass the resolved filter
- Update IMAP accounts API (schemas, create/update handlers, response serializer)
- Update IMAP accounts UI to show attachment filter dropdown in modal and
  display filter badges on account cards
- Add 6 new tests covering attachment filter behaviour
- Update ConfigurationGuide.md, EmailIngestion.md, and .env.demo

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-12 00:59:54 +00:00
parent 5e45b68cc1
commit 554bb21d32
11 changed files with 300 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/imap_accounts.py
+26
View File
@@ -110,6 +110,15 @@ class ImapAccountCreate(BaseModel):
use_ssl: bool = Field(default=True, description="Use SSL/TLS connection")
delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing")
is_active: bool = Field(default=True, description="Whether to poll this mailbox")
attachment_filter: str | None = Field(
default=None,
description=(
"Controls which attachment types to ingest. "
"'documents_only' – PDFs and office files only (default when None). "
"'all' – all supported types including images. "
"Null inherits the global imap_attachment_filter setting."
),
)
class ImapAccountUpdate(BaseModel):
@@ -123,6 +132,15 @@ class ImapAccountUpdate(BaseModel):
use_ssl: bool | None = None
delete_after_process: bool | None = None
is_active: bool | None = None
attachment_filter: str | None = Field(
default=None,
description=(
"Controls which attachment types to ingest. "
"'documents_only' – PDFs and office files only. "
"'all' – all supported types including images. "
"Null or empty string clears the override (inherits global setting)."
),
)
class ImapTestRequest(BaseModel):
@@ -155,6 +173,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]:
"use_ssl": acct.use_ssl,
"delete_after_process": acct.delete_after_process,
"is_active": acct.is_active,
"attachment_filter": acct.attachment_filter,
"last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None,
"last_error": acct.last_error,
"created_at": acct.created_at.isoformat() if acct.created_at else None,
@@ -222,6 +241,7 @@ def create_imap_account(
use_ssl=body.use_ssl,
delete_after_process=body.delete_after_process,
is_active=body.is_active,
attachment_filter=body.attachment_filter or None,
)
try:
db.add(acct)
@@ -277,6 +297,12 @@ def update_imap_account(
acct.delete_after_process = body.delete_after_process
if body.is_active is not None:
acct.is_active = body.is_active
# attachment_filter uses a sentinel check: the field is always present in the
# model (defaulting to None in Pydantic) so we update it unconditionally when
# the caller sends any value (including explicit null to clear the override).
# An empty string is normalised to None to avoid storing a non-meaningful value.
if "attachment_filter" in body.model_fields_set:
acct.attachment_filter = body.attachment_filter or None
# Reset last_error so the next poll gives a fresh result
acct.last_error = None
app/config.py
+11
View File
@@ -571,6 +571,17 @@ class Settings(BaseSettings):
),
)
imap_attachment_filter: str = Field(
default="documents_only",
description=(
"Controls which attachment types are ingested from IMAP emails. "
"Accepted values: "
"'documents_only' – ingest only PDFs and office files (Word, Excel, PowerPoint, ODT, etc.); "
"'all' – ingest all supported file types including images. "
"This is the global default; individual user IMAP accounts can override it."
),
)
# Batch processing settings
processall_throttle_threshold: int = Field(
default=20,
app/models.py
+5
View File
@@ -439,6 +439,11 @@ class UserImapAccount(Base):
# When True, emails are deleted from the mailbox after their attachments are processed
delete_after_process = Column(Boolean, nullable=False, default=False)
# Override for which attachment types to ingest.
# NULL means "inherit the global imap_attachment_filter setting".
# Allowed values: 'documents_only', 'all'
attachment_filter = Column(String(50), nullable=True, default=None)
# When False the account is not polled by the periodic task (but not deleted)
is_active = Column(Boolean, nullable=False, default=True)
app/tasks/imap_tasks.py
+75 -21
View File
@@ -13,7 +13,7 @@ from celery import shared_task
from app.config import settings
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
from app.tasks.process_document import process_document # Updated import
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, DOCUMENT_MIME_TYPES, IMAGE_MIME_TYPES
# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
_db_session_factory = None
@@ -180,6 +180,7 @@ def _pull_user_imap_accounts() -> None:
use_ssl=acct.use_ssl,
delete_after_process=acct.delete_after_process,
owner_id=acct.owner_id,
attachment_filter=acct.attachment_filter or settings.imap_attachment_filter,
)
# Record successful poll
acct.last_checked_at = datetime.now(timezone.utc)
@@ -250,6 +251,7 @@ def _pull_user_integration_imap() -> None:
use_ssl = cfg.get("use_ssl", True)
delete_after = cfg.get("delete_after_process", False)
gmail_labels = cfg.get("gmail_apply_labels", True)
attachment_filter = cfg.get("attachment_filter") or settings.imap_attachment_filter
if not (host and username and password):
logger.warning(
@@ -269,6 +271,7 @@ def _pull_user_integration_imap() -> None:
delete_after_process=delete_after,
owner_id=integ.owner_id,
gmail_apply_labels=gmail_labels,
attachment_filter=attachment_filter,
)
integ.last_used_at = datetime.now(timezone.utc)
integ.last_error = None
@@ -329,6 +332,7 @@ def pull_inbox(
delete_after_process,
owner_id=None,
gmail_apply_labels=True,
attachment_filter=None,
):
"""
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
@@ -345,7 +349,12 @@ def pull_inbox(
attributed to this user via ``process_document`` / ``convert_to_pdf``.
gmail_apply_labels: Whether to apply Gmail-specific labels and stars to
processed emails. Only relevant for Gmail hosts. Defaults to True.
attachment_filter: Controls which attachment types to ingest.
``'documents_only'`` (default) – PDFs and office files only.
``'all'`` – all supported types including images.
``None`` falls back to the global ``settings.imap_attachment_filter``.
"""
resolved_filter = attachment_filter or settings.imap_attachment_filter
logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
processed_emails = load_processed_emails()
@@ -407,7 +416,7 @@ def pull_inbox(
# Process attachments (and convert non-PDF files).
# We call the function without assigning its return value since it is not used.
fetch_attachments_and_enqueue(email_message, owner_id=owner_id)
fetch_attachments_and_enqueue(email_message, owner_id=owner_id, attachment_filter=resolved_filter)
if settings.imap_readonly_mode:
logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
@@ -436,27 +445,22 @@ def pull_inbox(
logger.exception("Error pulling mailbox %s: %s", mailbox_key, e)
def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
def fetch_attachments_and_enqueue(
email_message,
owner_id: str | None = None,
attachment_filter: str | None = None,
):
"""
Extracts attachments from the email and processes only allowed file types.
Files are accepted if either:
1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR
2. They have a '.pdf' file extension (regardless of MIME type)
Files are first checked against the ``attachment_filter`` to determine which
broad categories are permitted, then validated against known MIME types /
extensions.
Allowed file types include:
- PDF: application/pdf or *.pdf extension
- Microsoft Office files:
- Word: application/msword,
application/vnd.openxmlformats-officedocument.wordprocessingml.document
- Excel: application/vnd.ms-excel,
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
- PowerPoint: application/vnd.ms-powerpoint,
application/vnd.openxmlformats-officedocument.presentationml.presentation
- Other meaningful attachments:
- Plain text: text/plain
- CSV: text/csv
- Rich Text Format: application/rtf, text/rtf
Attachment filter values:
- ``'documents_only'`` (default): PDFs, office files (Word, Excel, PowerPoint,
OpenDocument, RTF), plain text, CSV, HTML, and Markdown. Images are skipped.
- ``'all'``: All supported file types, including images (JPEG, PNG, GIF, etc.).
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
any other allowed file is enqueued for conversion to PDF.
@@ -465,9 +469,37 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
email_message: The parsed email message to extract attachments from.
owner_id: Optional user identifier forwarded to ``process_document`` /
``convert_to_pdf`` for multi-tenant attribution.
attachment_filter: Override for the filter level. Defaults to
``settings.imap_attachment_filter`` when not provided.
Returns True if at least one allowed attachment was processed.
"""
resolved_filter = attachment_filter or settings.imap_attachment_filter
# Build the effective allowed MIME type set based on the filter
if resolved_filter == "all":
effective_mime_types = ALLOWED_MIME_TYPES
else:
# 'documents_only' (and any unrecognised value): exclude images
effective_mime_types = DOCUMENT_MIME_TYPES
# Build the effective allowed extensions set (images excluded for documents_only)
if resolved_filter == "all":
effective_extensions = ALLOWED_EXTENSIONS
else:
image_extensions = {
".jpg",
".jpeg",
".png",
".gif",
".bmp",
".tiff",
".tif",
".webp",
".svg",
}
effective_extensions = ALLOWED_EXTENSIONS - image_extensions
has_attachment = False
for part in email_message.walk():
if part.get_content_maintype() == "multipart":
@@ -482,8 +514,30 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
mime_type = part.get_content_type()
file_ext = os.path.splitext(filename)[1].lower()
# Skip images when filter is documents_only
is_image = mime_type in IMAGE_MIME_TYPES or file_ext in {
".jpg",
".jpeg",
".png",
".gif",
".bmp",
".tiff",
".tif",
".webp",
".svg",
}
if is_image and resolved_filter != "all":
logger.info(
"Skipping image attachment %s (MIME: %s) — attachment_filter=%s",
filename,
mime_type,
resolved_filter,
)
continue
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension:
if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension:
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
continue
@@ -495,7 +549,7 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
if mime_type == "application/pdf" or is_pdf_by_extension:
process_document.delay(file_path, owner_id=owner_id)
logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type)
elif mime_type in ALLOWED_MIME_TYPES:
elif mime_type in effective_mime_types:
# Other allowed files are sent for conversion
convert_to_pdf.delay(file_path, owner_id=owner_id)
logger.info("Enqueued file for conversion to PDF: %s", filename)
app/utils/settings_service.py
+12
View File
@@ -1425,6 +1425,18 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
"imap_attachment_filter": {
"category": "IMAP",
"description": (
"Controls which attachment types are ingested from IMAP emails. "
"Accepted values: 'documents_only' (PDFs and office files only, default) or 'all' (including images). "
"Per-user IMAP accounts can override this global default."
),
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
# Monitoring - Uptime Kuma
"uptime_kuma_url": {
"category": "Monitoring",