diff --git a/.env.demo b/.env.demo index 59f17f94..b58c649f 100644 --- a/.env.demo +++ b/.env.demo @@ -311,6 +311,12 @@ IMAP2_DELETE_AFTER_PROCESS=false # Use for pre-production instances that share a mailbox with production. IMAP_READONLY_MODE=false +# Controls which attachment types are ingested from IMAP emails. +# 'documents_only' (default) – PDFs and office files only; images are skipped. +# 'all' – all supported file types including images. +# Per-user IMAP accounts can override this global default. +IMAP_ATTACHMENT_FILTER=documents_only + # **Storage/Document Services** # Amazon S3 AWS_REGION=us-east-1 diff --git a/app/api/imap_accounts.py b/app/api/imap_accounts.py index 69b26c72..169fdef7 100644 --- a/app/api/imap_accounts.py +++ b/app/api/imap_accounts.py @@ -110,6 +110,15 @@ class ImapAccountCreate(BaseModel): use_ssl: bool = Field(default=True, description="Use SSL/TLS connection") delete_after_process: bool = Field(default=False, description="Delete emails from mailbox after processing") is_active: bool = Field(default=True, description="Whether to poll this mailbox") + attachment_filter: str | None = Field( + default=None, + description=( + "Controls which attachment types to ingest. " + "'documents_only' – PDFs and office files only (default when None). " + "'all' – all supported types including images. " + "Null inherits the global imap_attachment_filter setting." + ), + ) class ImapAccountUpdate(BaseModel): @@ -123,6 +132,15 @@ class ImapAccountUpdate(BaseModel): use_ssl: bool | None = None delete_after_process: bool | None = None is_active: bool | None = None + attachment_filter: str | None = Field( + default=None, + description=( + "Controls which attachment types to ingest. " + "'documents_only' – PDFs and office files only. " + "'all' – all supported types including images. " + "Null or empty string clears the override (inherits global setting)." + ), + ) class ImapTestRequest(BaseModel): @@ -155,6 +173,7 @@ def _to_response(acct: UserImapAccount) -> dict[str, Any]: "use_ssl": acct.use_ssl, "delete_after_process": acct.delete_after_process, "is_active": acct.is_active, + "attachment_filter": acct.attachment_filter, "last_checked_at": acct.last_checked_at.isoformat() if acct.last_checked_at else None, "last_error": acct.last_error, "created_at": acct.created_at.isoformat() if acct.created_at else None, @@ -222,6 +241,7 @@ def create_imap_account( use_ssl=body.use_ssl, delete_after_process=body.delete_after_process, is_active=body.is_active, + attachment_filter=body.attachment_filter or None, ) try: db.add(acct) @@ -277,6 +297,12 @@ def update_imap_account( acct.delete_after_process = body.delete_after_process if body.is_active is not None: acct.is_active = body.is_active + # attachment_filter uses a sentinel check: the field is always present in the + # model (defaulting to None in Pydantic) so we update it unconditionally when + # the caller sends any value (including explicit null to clear the override). + # An empty string is normalised to None to avoid storing a non-meaningful value. + if "attachment_filter" in body.model_fields_set: + acct.attachment_filter = body.attachment_filter or None # Reset last_error so the next poll gives a fresh result acct.last_error = None diff --git a/app/config.py b/app/config.py index 7ffd15e1..884e1a4d 100644 --- a/app/config.py +++ b/app/config.py @@ -571,6 +571,17 @@ class Settings(BaseSettings): ), ) + imap_attachment_filter: str = Field( + default="documents_only", + description=( + "Controls which attachment types are ingested from IMAP emails. " + "Accepted values: " + "'documents_only' – ingest only PDFs and office files (Word, Excel, PowerPoint, ODT, etc.); " + "'all' – ingest all supported file types including images. " + "This is the global default; individual user IMAP accounts can override it." + ), + ) + # Batch processing settings processall_throttle_threshold: int = Field( default=20, diff --git a/app/models.py b/app/models.py index 9b3ea4d1..94cb4e62 100644 --- a/app/models.py +++ b/app/models.py @@ -439,6 +439,11 @@ class UserImapAccount(Base): # When True, emails are deleted from the mailbox after their attachments are processed delete_after_process = Column(Boolean, nullable=False, default=False) + # Override for which attachment types to ingest. + # NULL means "inherit the global imap_attachment_filter setting". + # Allowed values: 'documents_only', 'all' + attachment_filter = Column(String(50), nullable=True, default=None) + # When False the account is not polled by the periodic task (but not deleted) is_active = Column(Boolean, nullable=False, default=True) diff --git a/app/tasks/imap_tasks.py b/app/tasks/imap_tasks.py index 064ccf06..be9f3bd4 100644 --- a/app/tasks/imap_tasks.py +++ b/app/tasks/imap_tasks.py @@ -13,7 +13,7 @@ from celery import shared_task from app.config import settings from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task from app.tasks.process_document import process_document # Updated import -from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES +from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, DOCUMENT_MIME_TYPES, IMAGE_MIME_TYPES # Database session for per-user IMAP accounts (imported lazily to avoid circular imports) _db_session_factory = None @@ -180,6 +180,7 @@ def _pull_user_imap_accounts() -> None: use_ssl=acct.use_ssl, delete_after_process=acct.delete_after_process, owner_id=acct.owner_id, + attachment_filter=acct.attachment_filter or settings.imap_attachment_filter, ) # Record successful poll acct.last_checked_at = datetime.now(timezone.utc) @@ -250,6 +251,7 @@ def _pull_user_integration_imap() -> None: use_ssl = cfg.get("use_ssl", True) delete_after = cfg.get("delete_after_process", False) gmail_labels = cfg.get("gmail_apply_labels", True) + attachment_filter = cfg.get("attachment_filter") or settings.imap_attachment_filter if not (host and username and password): logger.warning( @@ -269,6 +271,7 @@ def _pull_user_integration_imap() -> None: delete_after_process=delete_after, owner_id=integ.owner_id, gmail_apply_labels=gmail_labels, + attachment_filter=attachment_filter, ) integ.last_used_at = datetime.now(timezone.utc) integ.last_error = None @@ -329,6 +332,7 @@ def pull_inbox( delete_after_process, owner_id=None, gmail_apply_labels=True, + attachment_filter=None, ): """ Connects to the IMAP inbox, fetches new unread emails from the last 3 days, @@ -345,7 +349,12 @@ def pull_inbox( attributed to this user via ``process_document`` / ``convert_to_pdf``. gmail_apply_labels: Whether to apply Gmail-specific labels and stars to processed emails. Only relevant for Gmail hosts. Defaults to True. + attachment_filter: Controls which attachment types to ingest. + ``'documents_only'`` (default) – PDFs and office files only. + ``'all'`` – all supported types including images. + ``None`` falls back to the global ``settings.imap_attachment_filter``. """ + resolved_filter = attachment_filter or settings.imap_attachment_filter logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl) processed_emails = load_processed_emails() @@ -407,7 +416,7 @@ def pull_inbox( # Process attachments (and convert non-PDF files). # We call the function without assigning its return value since it is not used. - fetch_attachments_and_enqueue(email_message, owner_id=owner_id) + fetch_attachments_and_enqueue(email_message, owner_id=owner_id, attachment_filter=resolved_filter) if settings.imap_readonly_mode: logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key) @@ -436,27 +445,22 @@ def pull_inbox( logger.exception("Error pulling mailbox %s: %s", mailbox_key, e) -def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None): +def fetch_attachments_and_enqueue( + email_message, + owner_id: str | None = None, + attachment_filter: str | None = None, +): """ Extracts attachments from the email and processes only allowed file types. - Files are accepted if either: - 1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR - 2. They have a '.pdf' file extension (regardless of MIME type) + Files are first checked against the ``attachment_filter`` to determine which + broad categories are permitted, then validated against known MIME types / + extensions. - Allowed file types include: - - PDF: application/pdf or *.pdf extension - - Microsoft Office files: - - Word: application/msword, - application/vnd.openxmlformats-officedocument.wordprocessingml.document - - Excel: application/vnd.ms-excel, - application/vnd.openxmlformats-officedocument.spreadsheetml.sheet - - PowerPoint: application/vnd.ms-powerpoint, - application/vnd.openxmlformats-officedocument.presentationml.presentation - - Other meaningful attachments: - - Plain text: text/plain - - CSV: text/csv - - Rich Text Format: application/rtf, text/rtf + Attachment filter values: + - ``'documents_only'`` (default): PDFs, office files (Word, Excel, PowerPoint, + OpenDocument, RTF), plain text, CSV, HTML, and Markdown. Images are skipped. + - ``'all'``: All supported file types, including images (JPEG, PNG, GIF, etc.). If the attachment is a PDF (by extension or MIME type), it is enqueued for upload; any other allowed file is enqueued for conversion to PDF. @@ -465,9 +469,37 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None): email_message: The parsed email message to extract attachments from. owner_id: Optional user identifier forwarded to ``process_document`` / ``convert_to_pdf`` for multi-tenant attribution. + attachment_filter: Override for the filter level. Defaults to + ``settings.imap_attachment_filter`` when not provided. Returns True if at least one allowed attachment was processed. """ + resolved_filter = attachment_filter or settings.imap_attachment_filter + + # Build the effective allowed MIME type set based on the filter + if resolved_filter == "all": + effective_mime_types = ALLOWED_MIME_TYPES + else: + # 'documents_only' (and any unrecognised value): exclude images + effective_mime_types = DOCUMENT_MIME_TYPES + + # Build the effective allowed extensions set (images excluded for documents_only) + if resolved_filter == "all": + effective_extensions = ALLOWED_EXTENSIONS + else: + image_extensions = { + ".jpg", + ".jpeg", + ".png", + ".gif", + ".bmp", + ".tiff", + ".tif", + ".webp", + ".svg", + } + effective_extensions = ALLOWED_EXTENSIONS - image_extensions + has_attachment = False for part in email_message.walk(): if part.get_content_maintype() == "multipart": @@ -482,8 +514,30 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None): mime_type = part.get_content_type() file_ext = os.path.splitext(filename)[1].lower() + + # Skip images when filter is documents_only + is_image = mime_type in IMAGE_MIME_TYPES or file_ext in { + ".jpg", + ".jpeg", + ".png", + ".gif", + ".bmp", + ".tiff", + ".tif", + ".webp", + ".svg", + } + if is_image and resolved_filter != "all": + logger.info( + "Skipping image attachment %s (MIME: %s) — attachment_filter=%s", + filename, + mime_type, + resolved_filter, + ) + continue + # Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension - if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension: + if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension: logger.info("Skipping attachment %s with MIME type %s", filename, mime_type) continue @@ -495,7 +549,7 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None): if mime_type == "application/pdf" or is_pdf_by_extension: process_document.delay(file_path, owner_id=owner_id) logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type) - elif mime_type in ALLOWED_MIME_TYPES: + elif mime_type in effective_mime_types: # Other allowed files are sent for conversion convert_to_pdf.delay(file_path, owner_id=owner_id) logger.info("Enqueued file for conversion to PDF: %s", filename) diff --git a/app/utils/settings_service.py b/app/utils/settings_service.py index 78188b9e..e9b040d0 100644 --- a/app/utils/settings_service.py +++ b/app/utils/settings_service.py @@ -1425,6 +1425,18 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + "imap_attachment_filter": { + "category": "IMAP", + "description": ( + "Controls which attachment types are ingested from IMAP emails. " + "Accepted values: 'documents_only' (PDFs and office files only, default) or 'all' (including images). " + "Per-user IMAP accounts can override this global default." + ), + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, # Monitoring - Uptime Kuma "uptime_kuma_url": { "category": "Monitoring", diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index 398aed82..747db3ee 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -304,6 +304,7 @@ DocuElevate can automatically pull document attachments from IMAP mailboxes — | `IMAP1_SSL` | Use SSL (`true`/`false`). | `true` | | `IMAP1_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll for new mail. | `5` | | `IMAP_READONLY_MODE` | When `true`, fetches and processes attachments but does **not** modify the mailbox (no starring, labeling, deleting, or flag changes). Use for pre-production instances sharing a mailbox with production. Default: `false`. | `false` | +| `IMAP_ATTACHMENT_FILTER` | Controls which attachment types are ingested from emails. `documents_only` (default) ingests PDFs and office files only — images are skipped. `all` ingests every supported file type including images. Individual per-user IMAP accounts can override this global default. | `documents_only` | #### Per-User IMAP Integrations diff --git a/docs/howto/EmailIngestion.md b/docs/howto/EmailIngestion.md index 1a911f6f..12e76eae 100644 --- a/docs/howto/EmailIngestion.md +++ b/docs/howto/EmailIngestion.md @@ -63,6 +63,37 @@ DocuElevate will process the following attachment types from emails: | TIFF | `.tif`, `.tiff` | Common format from older scanners/fax | | Multi-page TIFF | `.tif` | Full multi-page support | +### Controlling Which Attachment Types Are Ingested + +By default, DocuElevate only ingests **document** attachments (PDFs, Word, Excel, PowerPoint, OpenDocument, RTF, TXT, CSV, HTML, Markdown). Images are **not** ingested by default — this prevents cluttering your document archive with inline images or unrelated photo attachments. + +#### Global Default (Admin Setting) + +Set the `IMAP_ATTACHMENT_FILTER` environment variable to control the system-wide default: + +| Value | Behaviour | +|-------|-----------| +| `documents_only` | **(Default)** Only PDFs and office/document files. Images (JPEG, PNG, GIF, BMP, TIFF, WebP, SVG) are skipped. | +| `all` | All supported file types, including images. | + +```env +# Only ingest document-type attachments (default behaviour) +IMAP_ATTACHMENT_FILTER=documents_only + +# Ingest all supported file types, including images +IMAP_ATTACHMENT_FILTER=all +``` + +#### Per-User Override + +Each user can override the global default for their personal IMAP accounts via the **Email Ingestion** dashboard (`/imap-accounts`). When creating or editing an account, select the desired setting from the **Attachment Types to Ingest** dropdown: + +- **Use global default** — inherits the `IMAP_ATTACHMENT_FILTER` setting above. +- **Documents only** — PDFs and office files, no images. +- **All supported types (including images)** — overrides the global setting to allow images for this specific account. + +This allows administrators to restrict image ingestion system-wide while individual users can opt-in to image ingestion on a per-mailbox basis. + --- ## Setting Up Your Scanner/Device diff --git a/frontend/templates/imap_accounts.html b/frontend/templates/imap_accounts.html index e22614e6..16a6cdaa 100644 --- a/frontend/templates/imap_accounts.html +++ b/frontend/templates/imap_accounts.html @@ -172,6 +172,16 @@ Delete after process + +