diff --git a/app/models.py b/app/models.py
index e782c494..4e5bd6ef 100644
--- a/app/models.py
+++ b/app/models.py
@@ -544,9 +544,55 @@ class UserIntegration(Base):
IMAP:
config = {"host": "imap.example.com", "port": 993,
"username": "user@example.com", "use_ssl": true,
- "delete_after_process": false}
+ "delete_after_process": false,
+ "gmail_apply_labels": true}
credentials = {"password": "secret"}
+ WATCH_FOLDER (local):
+ config = {"source_type": "local",
+ "folder_path": "/data/inbox",
+ "delete_after_process": false}
+
+ WATCH_FOLDER (s3):
+ config = {"source_type": "s3", "bucket": "my-bucket",
+ "region": "us-east-1", "prefix": "inbox/",
+ "endpoint_url": null, "delete_after_process": false}
+ credentials = {"access_key_id": "AKI…", "secret_access_key": "…"}
+
+ WATCH_FOLDER (dropbox):
+ config = {"source_type": "dropbox",
+ "folder_path": "/Inbox/Scanner",
+ "delete_after_process": false}
+ credentials = {"refresh_token": "…", "app_key": "…",
+ "app_secret": "…"}
+
+ WATCH_FOLDER (google_drive):
+ config = {"source_type": "google_drive",
+ "folder_id": "1abc…",
+ "delete_after_process": false}
+ credentials = {"credentials_json": "{…service-account…}"}
+
+ WATCH_FOLDER (onedrive):
+ config = {"source_type": "onedrive",
+ "folder_path": "/Documents/Inbox",
+ "delete_after_process": false}
+ credentials = {"refresh_token": "…", "client_id": "…",
+ "client_secret": "…"}
+
+ WATCH_FOLDER (nextcloud):
+ config = {"source_type": "nextcloud",
+ "url": "https://cloud.example.com",
+ "folder_path": "/Documents/Inbox",
+ "delete_after_process": false}
+ credentials = {"username": "user", "password": "secret"}
+
+ WATCH_FOLDER (webdav):
+ config = {"source_type": "webdav",
+ "url": "https://webdav.example.com/dav/",
+ "folder_path": "/remote.php/webdav/Inbox",
+ "delete_after_process": false}
+ credentials = {"username": "user", "password": "secret"}
+
S3:
config = {"bucket": "my-bucket", "region": "us-east-1",
"endpoint_url": null, "folder_prefix": ""}
diff --git a/app/tasks/imap_tasks.py b/app/tasks/imap_tasks.py
index 2ab827cf..064ccf06 100644
--- a/app/tasks/imap_tasks.py
+++ b/app/tasks/imap_tasks.py
@@ -1,589 +1,603 @@
-#!/usr/bin/env python3
-import email
-import imaplib
-import json
-import logging
-import os
-import re
-from datetime import datetime, timedelta, timezone
-
-import redis
-from celery import shared_task
-
-from app.config import settings
-from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
-from app.tasks.process_document import process_document # Updated import
-from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES
-
-# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
-_db_session_factory = None
-
-# Maximum length to store as last_error to prevent DB bloat
-_MAX_ERROR_LENGTH = 500
-
-
-def _get_db_session():
- """Return a new SQLAlchemy session (lazy import to avoid startup issues)."""
- global _db_session_factory # noqa: PLW0603
- if _db_session_factory is None:
- from app.database import SessionLocal
-
- _db_session_factory = SessionLocal
- return _db_session_factory()
-
-
-logger = logging.getLogger(__name__)
-
-# Initialize Redis connection using Celery's Redis settings
-redis_client = redis.StrictRedis.from_url(settings.redis_url, decode_responses=True)
-
-
-def _decrypt_imap_password(password: str | None) -> str | None:
- """Decrypt an IMAP account password stored in the database.
-
- Passwords are stored encrypted (Fernet, ``enc:`` prefix) for new records;
- legacy plaintext records are returned unchanged so existing accounts
- continue to work until they are next updated via the API.
- """
- from app.utils.encryption import decrypt_value
-
- return decrypt_value(password)
-
-
-LOCK_KEY = "imap_lock" # Unique key for locking
-LOCK_EXPIRE = 300 # Lock expires in 5 minutes
-
-# Local cache file for tracking processed emails
-CACHE_FILE = os.path.join(settings.workdir, "processed_mails.json")
-
-
-def acquire_lock():
- """Attempt to acquire a Redis-based lock. If acquired, set an expiration."""
- lock_acquired = redis_client.setnx(LOCK_KEY, "locked")
- if lock_acquired:
- redis_client.expire(LOCK_KEY, LOCK_EXPIRE)
- logger.info("Lock acquired for IMAP processing.")
- return True
- logger.warning("Lock already held. Skipping this cycle.")
- return False
-
-
-def release_lock():
- """Release the lock by deleting the Redis key."""
- redis_client.delete(LOCK_KEY)
- logger.info("Lock released.")
-
-
-def load_processed_emails():
- """Load the list of already processed emails from a local JSON file."""
- if os.path.exists(CACHE_FILE):
- try:
- with open(CACHE_FILE, "r") as f:
- processed_emails = json.load(f)
- processed_emails = cleanup_old_entries(processed_emails)
- return processed_emails
- except json.JSONDecodeError:
- logger.warning("Failed to decode JSON, resetting processed emails cache.")
- return {}
- return {}
-
-
-def save_processed_emails(processed_emails):
- """Save the processed email IDs to a local JSON file."""
- with open(CACHE_FILE, "w") as f:
- json.dump(processed_emails, f, indent=4)
-
-
-def cleanup_old_entries(processed_emails):
- """Remove entries older than 7 days from the cache to avoid infinite growth."""
- seven_days_ago = datetime.now(timezone.utc) - timedelta(days=7)
- valid_emails = {}
- for msg_id, date_str in processed_emails.items():
- naive_dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S")
- aware_dt = naive_dt.replace(tzinfo=timezone.utc)
- if aware_dt > seven_days_ago:
- valid_emails[msg_id] = date_str
- return valid_emails
-
-
-@shared_task
-def pull_all_inboxes():
- """
- Periodic Celery task that checks all configured IMAP mailboxes
- and fetches attachments from new emails.
- Ensures only one instance runs at a time using Redis-based locking.
-
- Processes:
- 1. System-level mailboxes configured via environment variables (IMAP1, IMAP2).
- 2. Per-user IMAP accounts stored in the ``user_imap_accounts`` database table.
- """
- if not acquire_lock():
- logger.info("Skipping execution: Another instance is running.")
- return
-
- try:
- logger.info("Starting pull_all_inboxes")
-
- # Mailbox #1 (non-Gmail)
- check_and_pull_mailbox(
- mailbox_key="imap1",
- host=settings.imap1_host,
- port=settings.imap1_port,
- username=settings.imap1_username,
- password=settings.imap1_password,
- use_ssl=settings.imap1_ssl,
- delete_after_process=settings.imap1_delete_after_process,
- )
-
- # Mailbox #2 (Gmail)
- check_and_pull_mailbox(
- mailbox_key="imap2",
- host=settings.imap2_host,
- port=settings.imap2_port,
- username=settings.imap2_username,
- password=settings.imap2_password,
- use_ssl=settings.imap2_ssl,
- delete_after_process=settings.imap2_delete_after_process,
- )
-
- # Per-user IMAP accounts from the database
- _pull_user_imap_accounts()
-
- # Per-user IMAP integrations from the UserIntegration model
- _pull_user_integration_imap()
-
- logger.info("Finished pull_all_inboxes")
-
- finally:
- release_lock()
-
-
-def _pull_user_imap_accounts() -> None:
- """Iterate over all active per-user IMAP accounts and pull their inboxes."""
- try:
- from app.models import UserImapAccount
-
- db = _get_db_session()
- try:
- accounts = db.query(UserImapAccount).filter(UserImapAccount.is_active.is_(True)).all()
- logger.info("Processing %d per-user IMAP account(s)", len(accounts))
- for acct in accounts:
- # Use a descriptive identifier for logging and processed-email cache keys
- account_identifier = f"user_{acct.owner_id}_{acct.id}"
- try:
- pull_inbox(
- mailbox_key=account_identifier,
- host=acct.host,
- port=acct.port,
- username=acct.username,
- password=_decrypt_imap_password(acct.password),
- use_ssl=acct.use_ssl,
- delete_after_process=acct.delete_after_process,
- owner_id=acct.owner_id,
- )
- # Record successful poll
- acct.last_checked_at = datetime.now(timezone.utc)
- acct.last_error = None
- db.commit()
- except Exception as exc: # noqa: BLE001
- error_msg = str(exc)[:_MAX_ERROR_LENGTH]
- logger.error(
- "Error pulling user IMAP account %d (%s@%s): %s",
- acct.id,
- acct.username,
- acct.host,
- error_msg,
- )
- try:
- acct.last_checked_at = datetime.now(timezone.utc)
- acct.last_error = error_msg
- db.commit()
- except Exception: # noqa: BLE001
- db.rollback()
- finally:
- db.close()
- except Exception as exc: # noqa: BLE001
- logger.error("Failed to process per-user IMAP accounts: %s", exc)
-
-
-def _pull_user_integration_imap() -> None:
- """Iterate over all active IMAP UserIntegrations and pull their inboxes.
-
- This polls the ``user_integrations`` table for records with
- ``integration_type='IMAP'``, ``direction='SOURCE'``, and ``is_active=True``.
- Each integration's config/credentials are decoded and passed to
- :func:`pull_inbox` with the owning user's ``owner_id`` so that ingested
- documents are correctly attributed.
-
- Individual connection failures are caught and recorded on the integration
- without crashing the polling loop.
- """
- try:
- import json as _json
-
- from app.models import IntegrationDirection, IntegrationType, UserIntegration
- from app.utils.encryption import decrypt_value
-
- db = _get_db_session()
- try:
- integrations = (
- db.query(UserIntegration)
- .filter(
- UserIntegration.integration_type == IntegrationType.IMAP,
- UserIntegration.direction == IntegrationDirection.SOURCE,
- UserIntegration.is_active.is_(True),
- )
- .all()
- )
- logger.info("Processing %d IMAP UserIntegration(s)", len(integrations))
- for integ in integrations:
- account_identifier = f"integration_{integ.owner_id}_{integ.id}"
- try:
- cfg = _json.loads(integ.config) if integ.config else {}
- raw_creds = decrypt_value(integ.credentials) if integ.credentials else None
- creds = _json.loads(raw_creds) if raw_creds else {}
-
- host = cfg.get("host")
- port = int(cfg.get("port", 993))
- username = cfg.get("username")
- password = creds.get("password")
- use_ssl = cfg.get("use_ssl", True)
- delete_after = cfg.get("delete_after_process", False)
-
- if not (host and username and password):
- logger.warning(
- "IMAP integration %d (owner %s) has incomplete config — skipping.",
- integ.id,
- integ.owner_id,
- )
- continue
-
- pull_inbox(
- mailbox_key=account_identifier,
- host=host,
- port=port,
- username=username,
- password=password,
- use_ssl=use_ssl,
- delete_after_process=delete_after,
- owner_id=integ.owner_id,
- )
- integ.last_used_at = datetime.now(timezone.utc)
- integ.last_error = None
- db.commit()
- except Exception as exc: # noqa: BLE001
- error_msg = str(exc)[:_MAX_ERROR_LENGTH]
- logger.error(
- "Error pulling IMAP integration %d (owner %s): %s",
- integ.id,
- integ.owner_id,
- error_msg,
- )
- try:
- integ.last_used_at = datetime.now(timezone.utc)
- integ.last_error = error_msg
- db.commit()
- except Exception: # noqa: BLE001
- db.rollback()
- finally:
- db.close()
- except Exception as exc: # noqa: BLE001
- logger.error("Failed to process IMAP UserIntegrations: %s", exc)
-
-
-def check_and_pull_mailbox(
- mailbox_key: str,
- host: str | None,
- port: int | None,
- username: str | None,
- password: str | None,
- use_ssl: bool,
- delete_after_process: bool,
-):
- """Validates config and invokes pulling from the mailbox if valid."""
- if not (host and port and username and password):
- logger.warning(f"Mailbox {mailbox_key} is missing config, skipping.")
- return
-
- logger.info(f"Checking mailbox: {mailbox_key}")
- pull_inbox(
- mailbox_key=mailbox_key,
- host=host,
- port=port,
- username=username,
- password=password,
- use_ssl=use_ssl,
- delete_after_process=delete_after_process,
- )
-
-
-def pull_inbox(mailbox_key, host, port, username, password, use_ssl, delete_after_process, owner_id=None):
- """
- Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
- and processes attachments while preserving the original unread status.
-
- For Gmail:
- - Attempts to select the localized All Mail folder.
- - Runs an X-GM-RAW query: "in:anywhere in:unread newer_than:3d has:attachment".
-
- For non-Gmail mailboxes, it falls back to selecting the INBOX with a SINCE/UNSEEN filter.
-
- Args:
- owner_id: Optional user identifier. When provided, ingested documents are
- attributed to this user via ``process_document`` / ``convert_to_pdf``.
- """
- logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
- processed_emails = load_processed_emails()
-
- try:
- mail = imaplib.IMAP4_SSL(host, port) if use_ssl else imaplib.IMAP4(host, port)
- mail.login(username, password)
-
- is_gmail_host = "gmail" in host.lower()
- if is_gmail_host:
- # For Gmail, try to select the localized All Mail folder.
- all_mail_folder = find_all_mail_folder(mail)
- if all_mail_folder:
- logger.info("Using Gmail All Mail folder: %s", all_mail_folder)
- mail.select(f'"{all_mail_folder}"')
- else:
- logger.warning("Gmail All Mail folder not found, falling back to INBOX.")
- mail.select("INBOX")
- # Use the X-GM-RAW query for Gmail.
- raw_query = "in:anywhere in:unread newer_than:3d has:attachment"
- status, search_data = mail.search(None, "X-GM-RAW", f'"{raw_query}"')
- else:
- # For non-Gmail, select INBOX and use SINCE/UNSEEN query.
- mail.select("INBOX")
- since_date = (datetime.now(timezone.utc) - timedelta(days=3)).strftime("%d-%b-%Y")
- status, search_data = mail.search(None, f"(SINCE {since_date} UNSEEN)")
-
- if status != "OK":
- logger.warning("Search failed on mailbox %s. Status=%s", mailbox_key, status)
- mail.close()
- mail.logout()
- return
-
- msg_numbers = search_data[0].split()
- logger.info("Found %d unread emails in %s.", len(msg_numbers), mailbox_key)
-
- for num in msg_numbers:
- status, msg_data = mail.fetch(num, "(RFC822)")
- if status != "OK":
- logger.warning("Failed to fetch message %s in %s. Status=%s", num, mailbox_key, status)
- continue
-
- raw_email = msg_data[0][1]
- email_message = email.message_from_bytes(raw_email)
- msg_id = email_message.get("Message-ID")
-
- if not msg_id:
- logger.warning("Skipping email without Message-ID in %s", mailbox_key)
- continue
-
- if msg_id in processed_emails:
- logger.info("Skipping already processed email %s in %s", msg_id, mailbox_key)
- continue
-
- # For Gmail, check if the email already has the "Ingested" label.
- if is_gmail_host:
- if email_already_has_label(mail, num, "Ingested"):
- logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
- continue
-
- # Process attachments (and convert non-PDF files).
- # We call the function without assigning its return value since it is not used.
- fetch_attachments_and_enqueue(email_message, owner_id=owner_id)
-
- if settings.imap_readonly_mode:
- logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
- else:
- if is_gmail_host:
- mark_as_processed_with_star(mail, num)
- mark_as_processed_with_label(mail, num, label="Ingested")
-
- if delete_after_process:
- logger.info("Deleting message %s from %s", num.decode(), mailbox_key)
- mail.store(num, "+FLAGS", "\\Deleted")
- else:
- mail.store(num, "-FLAGS", "\\Seen")
-
- processed_emails[msg_id] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
- save_processed_emails(processed_emails)
-
- if not settings.imap_readonly_mode and delete_after_process:
- mail.expunge()
-
- mail.close()
- mail.logout()
- logger.info("Finished processing mailbox %s", mailbox_key)
-
- except Exception as e:
- logger.exception("Error pulling mailbox %s: %s", mailbox_key, e)
-
-
-def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
- """
- Extracts attachments from the email and processes only allowed file types.
-
- Files are accepted if either:
- 1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR
- 2. They have a '.pdf' file extension (regardless of MIME type)
-
- Allowed file types include:
- - PDF: application/pdf or *.pdf extension
- - Microsoft Office files:
- - Word: application/msword,
- application/vnd.openxmlformats-officedocument.wordprocessingml.document
- - Excel: application/vnd.ms-excel,
- application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
- - PowerPoint: application/vnd.ms-powerpoint,
- application/vnd.openxmlformats-officedocument.presentationml.presentation
- - Other meaningful attachments:
- - Plain text: text/plain
- - CSV: text/csv
- - Rich Text Format: application/rtf, text/rtf
-
- If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
- any other allowed file is enqueued for conversion to PDF.
-
- Args:
- email_message: The parsed email message to extract attachments from.
- owner_id: Optional user identifier forwarded to ``process_document`` /
- ``convert_to_pdf`` for multi-tenant attribution.
-
- Returns True if at least one allowed attachment was processed.
- """
- has_attachment = False
- for part in email_message.walk():
- if part.get_content_maintype() == "multipart":
- continue
-
- filename = part.get_filename()
- if not filename:
- continue
-
- # Check if it's a PDF file by extension, regardless of MIME type
- is_pdf_by_extension = filename.lower().endswith(".pdf")
-
- mime_type = part.get_content_type()
- file_ext = os.path.splitext(filename)[1].lower()
- # Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
- if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension:
- logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
- continue
-
- file_path = os.path.join(settings.workdir, filename)
- with open(file_path, "wb") as f:
- f.write(part.get_payload(decode=True))
-
- # If it's a PDF by MIME type or extension, process it directly
- if mime_type == "application/pdf" or is_pdf_by_extension:
- process_document.delay(file_path, owner_id=owner_id)
- logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type)
- elif mime_type in ALLOWED_MIME_TYPES:
- # Other allowed files are sent for conversion
- convert_to_pdf.delay(file_path, owner_id=owner_id)
- logger.info("Enqueued file for conversion to PDF: %s", filename)
-
- has_attachment = True
- return has_attachment
-
-
-def email_already_has_label(mail, msg_id, label="Ingested"):
- """
- Checks if the given message (msg_id) has the specified Gmail label.
- Returns True if the label is found, False otherwise.
- """
- try:
- # Convert msg_id to bytes if it's an integer
- if isinstance(msg_id, int):
- msg_id = str(msg_id).encode()
-
- label_status, label_data = mail.fetch(msg_id, "(X-GM-LABELS)")
- if label_status == "OK" and label_data and len(label_data) > 0:
- raw_labels = label_data[0][1].decode("utf-8", errors="ignore")
- if label in raw_labels:
- return True
- except Exception as e:
- logger.error("Failed to fetch labels for msg_id=%s: %s", msg_id, e)
- return False
-
-
-def mark_as_processed_with_star(mail, msg_id):
- """Stars the email in Gmail."""
- try:
- mail.store(msg_id, "+FLAGS", "\\Flagged")
- logger.info("Email %s starred in Gmail.", msg_id)
- except Exception as e:
- logger.error("Failed to star email %s: %s", msg_id, e)
-
-
-def mark_as_processed_with_label(mail, msg_id, label="Ingested"):
- """Adds a custom label to the email in Gmail."""
- try:
- mail.store(msg_id, "+X-GM-LABELS", label)
- logger.info("Email %s labeled '%s' in Gmail.", msg_id, label)
- except Exception as e:
- logger.error("Failed to label email %s with %s: %s", msg_id, label, e)
-
-
-def find_all_mail_folder(mail):
- """
- Attempts to select the Gmail All Mail folder using known localized names.
- Falls back to using XLIST if needed.
- Returns the folder name if found, otherwise None.
- """
- COMMON_ALL_MAIL_NAMES = [
- "[Gmail]/Alle Nachrichten",
- "[Gmail]/All Mail",
- "[Gmail]/Todos",
- "[Gmail]/Tutte le mail",
- "[Gmail]/Tous les messages",
- ]
- for candidate in COMMON_ALL_MAIL_NAMES:
- status, _ = mail.select(f'"{candidate}"', readonly=True)
- if status == "OK":
- return candidate
-
- capabilities = get_capabilities(mail)
- if "XLIST" in capabilities:
- candidate = find_all_mail_xlist(mail)
- if candidate:
- return candidate
- return None
-
-
-def get_capabilities(mail):
- """Returns a list of capabilities supported by the IMAP server."""
- typ, data = mail.capability()
- if typ == "OK" and data:
- caps = data[0].decode("utf-8", errors="ignore").upper().split()
- return caps
- return []
-
-
-def find_all_mail_xlist(mail):
- """
- Uses XLIST to discover the mailbox flagged as All Mail.
- Returns the folder name if found, otherwise None.
- """
- tag = mail._new_tag().decode("ascii")
- command_str = f'{tag} XLIST "" "*"'
- mail.send((command_str + "\r\n").encode("utf-8"))
-
- all_mail_folder = None
- while True:
- line = mail.readline()
- if not line:
- break
- line_str = line.decode("utf-8", errors="ignore").strip()
- if line_str.upper().startswith("* XLIST ") and "\\ALLMAIL" in line_str.upper():
- match = re.search(r'"([^"]+)"$', line_str)
- if match:
- candidate = match.group(1)
- logger.info("Found All Mail folder via XLIST: %s", candidate)
- all_mail_folder = candidate
- if line_str.startswith(tag):
- break
- return all_mail_folder
+#!/usr/bin/env python3
+import email
+import imaplib
+import json
+import logging
+import os
+import re
+from datetime import datetime, timedelta, timezone
+
+import redis
+from celery import shared_task
+
+from app.config import settings
+from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
+from app.tasks.process_document import process_document # Updated import
+from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES
+
+# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
+_db_session_factory = None
+
+# Maximum length to store as last_error to prevent DB bloat
+_MAX_ERROR_LENGTH = 500
+
+
+def _get_db_session():
+ """Return a new SQLAlchemy session (lazy import to avoid startup issues)."""
+ global _db_session_factory # noqa: PLW0603
+ if _db_session_factory is None:
+ from app.database import SessionLocal
+
+ _db_session_factory = SessionLocal
+ return _db_session_factory()
+
+
+logger = logging.getLogger(__name__)
+
+# Initialize Redis connection using Celery's Redis settings
+redis_client = redis.StrictRedis.from_url(settings.redis_url, decode_responses=True)
+
+
+def _decrypt_imap_password(password: str | None) -> str | None:
+ """Decrypt an IMAP account password stored in the database.
+
+ Passwords are stored encrypted (Fernet, ``enc:`` prefix) for new records;
+ legacy plaintext records are returned unchanged so existing accounts
+ continue to work until they are next updated via the API.
+ """
+ from app.utils.encryption import decrypt_value
+
+ return decrypt_value(password)
+
+
+LOCK_KEY = "imap_lock" # Unique key for locking
+LOCK_EXPIRE = 300 # Lock expires in 5 minutes
+
+# Local cache file for tracking processed emails
+CACHE_FILE = os.path.join(settings.workdir, "processed_mails.json")
+
+
+def acquire_lock():
+ """Attempt to acquire a Redis-based lock. If acquired, set an expiration."""
+ lock_acquired = redis_client.setnx(LOCK_KEY, "locked")
+ if lock_acquired:
+ redis_client.expire(LOCK_KEY, LOCK_EXPIRE)
+ logger.info("Lock acquired for IMAP processing.")
+ return True
+ logger.warning("Lock already held. Skipping this cycle.")
+ return False
+
+
+def release_lock():
+ """Release the lock by deleting the Redis key."""
+ redis_client.delete(LOCK_KEY)
+ logger.info("Lock released.")
+
+
+def load_processed_emails():
+ """Load the list of already processed emails from a local JSON file."""
+ if os.path.exists(CACHE_FILE):
+ try:
+ with open(CACHE_FILE, "r") as f:
+ processed_emails = json.load(f)
+ processed_emails = cleanup_old_entries(processed_emails)
+ return processed_emails
+ except json.JSONDecodeError:
+ logger.warning("Failed to decode JSON, resetting processed emails cache.")
+ return {}
+ return {}
+
+
+def save_processed_emails(processed_emails):
+ """Save the processed email IDs to a local JSON file."""
+ with open(CACHE_FILE, "w") as f:
+ json.dump(processed_emails, f, indent=4)
+
+
+def cleanup_old_entries(processed_emails):
+ """Remove entries older than 7 days from the cache to avoid infinite growth."""
+ seven_days_ago = datetime.now(timezone.utc) - timedelta(days=7)
+ valid_emails = {}
+ for msg_id, date_str in processed_emails.items():
+ naive_dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S")
+ aware_dt = naive_dt.replace(tzinfo=timezone.utc)
+ if aware_dt > seven_days_ago:
+ valid_emails[msg_id] = date_str
+ return valid_emails
+
+
+@shared_task
+def pull_all_inboxes():
+ """
+ Periodic Celery task that checks all configured IMAP mailboxes
+ and fetches attachments from new emails.
+ Ensures only one instance runs at a time using Redis-based locking.
+
+ Processes:
+ 1. System-level mailboxes configured via environment variables (IMAP1, IMAP2).
+ 2. Per-user IMAP accounts stored in the ``user_imap_accounts`` database table.
+ """
+ if not acquire_lock():
+ logger.info("Skipping execution: Another instance is running.")
+ return
+
+ try:
+ logger.info("Starting pull_all_inboxes")
+
+ # Mailbox #1 (non-Gmail)
+ check_and_pull_mailbox(
+ mailbox_key="imap1",
+ host=settings.imap1_host,
+ port=settings.imap1_port,
+ username=settings.imap1_username,
+ password=settings.imap1_password,
+ use_ssl=settings.imap1_ssl,
+ delete_after_process=settings.imap1_delete_after_process,
+ )
+
+ # Mailbox #2 (Gmail)
+ check_and_pull_mailbox(
+ mailbox_key="imap2",
+ host=settings.imap2_host,
+ port=settings.imap2_port,
+ username=settings.imap2_username,
+ password=settings.imap2_password,
+ use_ssl=settings.imap2_ssl,
+ delete_after_process=settings.imap2_delete_after_process,
+ )
+
+ # Per-user IMAP accounts from the database
+ _pull_user_imap_accounts()
+
+ # Per-user IMAP integrations from the UserIntegration model
+ _pull_user_integration_imap()
+
+ logger.info("Finished pull_all_inboxes")
+
+ finally:
+ release_lock()
+
+
+def _pull_user_imap_accounts() -> None:
+ """Iterate over all active per-user IMAP accounts and pull their inboxes."""
+ try:
+ from app.models import UserImapAccount
+
+ db = _get_db_session()
+ try:
+ accounts = db.query(UserImapAccount).filter(UserImapAccount.is_active.is_(True)).all()
+ logger.info("Processing %d per-user IMAP account(s)", len(accounts))
+ for acct in accounts:
+ # Use a descriptive identifier for logging and processed-email cache keys
+ account_identifier = f"user_{acct.owner_id}_{acct.id}"
+ try:
+ pull_inbox(
+ mailbox_key=account_identifier,
+ host=acct.host,
+ port=acct.port,
+ username=acct.username,
+ password=_decrypt_imap_password(acct.password),
+ use_ssl=acct.use_ssl,
+ delete_after_process=acct.delete_after_process,
+ owner_id=acct.owner_id,
+ )
+ # Record successful poll
+ acct.last_checked_at = datetime.now(timezone.utc)
+ acct.last_error = None
+ db.commit()
+ except Exception as exc: # noqa: BLE001
+ error_msg = str(exc)[:_MAX_ERROR_LENGTH]
+ logger.error(
+ "Error pulling user IMAP account %d (%s@%s): %s",
+ acct.id,
+ acct.username,
+ acct.host,
+ error_msg,
+ )
+ try:
+ acct.last_checked_at = datetime.now(timezone.utc)
+ acct.last_error = error_msg
+ db.commit()
+ except Exception: # noqa: BLE001
+ db.rollback()
+ finally:
+ db.close()
+ except Exception as exc: # noqa: BLE001
+ logger.error("Failed to process per-user IMAP accounts: %s", exc)
+
+
+def _pull_user_integration_imap() -> None:
+ """Iterate over all active IMAP UserIntegrations and pull their inboxes.
+
+ This polls the ``user_integrations`` table for records with
+ ``integration_type='IMAP'``, ``direction='SOURCE'``, and ``is_active=True``.
+ Each integration's config/credentials are decoded and passed to
+ :func:`pull_inbox` with the owning user's ``owner_id`` so that ingested
+ documents are correctly attributed.
+
+ Individual connection failures are caught and recorded on the integration
+ without crashing the polling loop.
+ """
+ try:
+ import json as _json
+
+ from app.models import IntegrationDirection, IntegrationType, UserIntegration
+ from app.utils.encryption import decrypt_value
+
+ db = _get_db_session()
+ try:
+ integrations = (
+ db.query(UserIntegration)
+ .filter(
+ UserIntegration.integration_type == IntegrationType.IMAP,
+ UserIntegration.direction == IntegrationDirection.SOURCE,
+ UserIntegration.is_active.is_(True),
+ )
+ .all()
+ )
+ logger.info("Processing %d IMAP UserIntegration(s)", len(integrations))
+ for integ in integrations:
+ account_identifier = f"integration_{integ.owner_id}_{integ.id}"
+ try:
+ cfg = _json.loads(integ.config) if integ.config else {}
+ raw_creds = decrypt_value(integ.credentials) if integ.credentials else None
+ creds = _json.loads(raw_creds) if raw_creds else {}
+
+ host = cfg.get("host")
+ port = int(cfg.get("port", 993))
+ username = cfg.get("username")
+ password = creds.get("password")
+ use_ssl = cfg.get("use_ssl", True)
+ delete_after = cfg.get("delete_after_process", False)
+ gmail_labels = cfg.get("gmail_apply_labels", True)
+
+ if not (host and username and password):
+ logger.warning(
+ "IMAP integration %d (owner %s) has incomplete config — skipping.",
+ integ.id,
+ integ.owner_id,
+ )
+ continue
+
+ pull_inbox(
+ mailbox_key=account_identifier,
+ host=host,
+ port=port,
+ username=username,
+ password=password,
+ use_ssl=use_ssl,
+ delete_after_process=delete_after,
+ owner_id=integ.owner_id,
+ gmail_apply_labels=gmail_labels,
+ )
+ integ.last_used_at = datetime.now(timezone.utc)
+ integ.last_error = None
+ db.commit()
+ except Exception as exc: # noqa: BLE001
+ error_msg = str(exc)[:_MAX_ERROR_LENGTH]
+ logger.error(
+ "Error pulling IMAP integration %d (owner %s): %s",
+ integ.id,
+ integ.owner_id,
+ error_msg,
+ )
+ try:
+ integ.last_used_at = datetime.now(timezone.utc)
+ integ.last_error = error_msg
+ db.commit()
+ except Exception: # noqa: BLE001
+ db.rollback()
+ finally:
+ db.close()
+ except Exception as exc: # noqa: BLE001
+ logger.error("Failed to process IMAP UserIntegrations: %s", exc)
+
+
+def check_and_pull_mailbox(
+ mailbox_key: str,
+ host: str | None,
+ port: int | None,
+ username: str | None,
+ password: str | None,
+ use_ssl: bool,
+ delete_after_process: bool,
+):
+ """Validates config and invokes pulling from the mailbox if valid."""
+ if not (host and port and username and password):
+ logger.warning(f"Mailbox {mailbox_key} is missing config, skipping.")
+ return
+
+ logger.info(f"Checking mailbox: {mailbox_key}")
+ pull_inbox(
+ mailbox_key=mailbox_key,
+ host=host,
+ port=port,
+ username=username,
+ password=password,
+ use_ssl=use_ssl,
+ delete_after_process=delete_after_process,
+ )
+
+
+def pull_inbox(
+ mailbox_key,
+ host,
+ port,
+ username,
+ password,
+ use_ssl,
+ delete_after_process,
+ owner_id=None,
+ gmail_apply_labels=True,
+):
+ """
+ Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
+ and processes attachments while preserving the original unread status.
+
+ For Gmail:
+ - Attempts to select the localized All Mail folder.
+ - Runs an X-GM-RAW query: "in:anywhere in:unread newer_than:3d has:attachment".
+
+ For non-Gmail mailboxes, it falls back to selecting the INBOX with a SINCE/UNSEEN filter.
+
+ Args:
+ owner_id: Optional user identifier. When provided, ingested documents are
+ attributed to this user via ``process_document`` / ``convert_to_pdf``.
+ gmail_apply_labels: Whether to apply Gmail-specific labels and stars to
+ processed emails. Only relevant for Gmail hosts. Defaults to True.
+ """
+ logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
+ processed_emails = load_processed_emails()
+
+ try:
+ mail = imaplib.IMAP4_SSL(host, port) if use_ssl else imaplib.IMAP4(host, port)
+ mail.login(username, password)
+
+ is_gmail_host = "gmail" in host.lower()
+ if is_gmail_host:
+ # For Gmail, try to select the localized All Mail folder.
+ all_mail_folder = find_all_mail_folder(mail)
+ if all_mail_folder:
+ logger.info("Using Gmail All Mail folder: %s", all_mail_folder)
+ mail.select(f'"{all_mail_folder}"')
+ else:
+ logger.warning("Gmail All Mail folder not found, falling back to INBOX.")
+ mail.select("INBOX")
+ # Use the X-GM-RAW query for Gmail.
+ raw_query = "in:anywhere in:unread newer_than:3d has:attachment"
+ status, search_data = mail.search(None, "X-GM-RAW", f'"{raw_query}"')
+ else:
+ # For non-Gmail, select INBOX and use SINCE/UNSEEN query.
+ mail.select("INBOX")
+ since_date = (datetime.now(timezone.utc) - timedelta(days=3)).strftime("%d-%b-%Y")
+ status, search_data = mail.search(None, f"(SINCE {since_date} UNSEEN)")
+
+ if status != "OK":
+ logger.warning("Search failed on mailbox %s. Status=%s", mailbox_key, status)
+ mail.close()
+ mail.logout()
+ return
+
+ msg_numbers = search_data[0].split()
+ logger.info("Found %d unread emails in %s.", len(msg_numbers), mailbox_key)
+
+ for num in msg_numbers:
+ status, msg_data = mail.fetch(num, "(RFC822)")
+ if status != "OK":
+ logger.warning("Failed to fetch message %s in %s. Status=%s", num, mailbox_key, status)
+ continue
+
+ raw_email = msg_data[0][1]
+ email_message = email.message_from_bytes(raw_email)
+ msg_id = email_message.get("Message-ID")
+
+ if not msg_id:
+ logger.warning("Skipping email without Message-ID in %s", mailbox_key)
+ continue
+
+ if msg_id in processed_emails:
+ logger.info("Skipping already processed email %s in %s", msg_id, mailbox_key)
+ continue
+
+ # For Gmail, check if the email already has the "Ingested" label.
+ if is_gmail_host and gmail_apply_labels:
+ if email_already_has_label(mail, num, "Ingested"):
+ logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
+ continue
+
+ # Process attachments (and convert non-PDF files).
+ # We call the function without assigning its return value since it is not used.
+ fetch_attachments_and_enqueue(email_message, owner_id=owner_id)
+
+ if settings.imap_readonly_mode:
+ logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
+ else:
+ if is_gmail_host and gmail_apply_labels:
+ mark_as_processed_with_star(mail, num)
+ mark_as_processed_with_label(mail, num, label="Ingested")
+
+ if delete_after_process:
+ logger.info("Deleting message %s from %s", num.decode(), mailbox_key)
+ mail.store(num, "+FLAGS", "\\Deleted")
+ else:
+ mail.store(num, "-FLAGS", "\\Seen")
+
+ processed_emails[msg_id] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
+ save_processed_emails(processed_emails)
+
+ if not settings.imap_readonly_mode and delete_after_process:
+ mail.expunge()
+
+ mail.close()
+ mail.logout()
+ logger.info("Finished processing mailbox %s", mailbox_key)
+
+ except Exception as e:
+ logger.exception("Error pulling mailbox %s: %s", mailbox_key, e)
+
+
+def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
+ """
+ Extracts attachments from the email and processes only allowed file types.
+
+ Files are accepted if either:
+ 1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR
+ 2. They have a '.pdf' file extension (regardless of MIME type)
+
+ Allowed file types include:
+ - PDF: application/pdf or *.pdf extension
+ - Microsoft Office files:
+ - Word: application/msword,
+ application/vnd.openxmlformats-officedocument.wordprocessingml.document
+ - Excel: application/vnd.ms-excel,
+ application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+ - PowerPoint: application/vnd.ms-powerpoint,
+ application/vnd.openxmlformats-officedocument.presentationml.presentation
+ - Other meaningful attachments:
+ - Plain text: text/plain
+ - CSV: text/csv
+ - Rich Text Format: application/rtf, text/rtf
+
+ If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
+ any other allowed file is enqueued for conversion to PDF.
+
+ Args:
+ email_message: The parsed email message to extract attachments from.
+ owner_id: Optional user identifier forwarded to ``process_document`` /
+ ``convert_to_pdf`` for multi-tenant attribution.
+
+ Returns True if at least one allowed attachment was processed.
+ """
+ has_attachment = False
+ for part in email_message.walk():
+ if part.get_content_maintype() == "multipart":
+ continue
+
+ filename = part.get_filename()
+ if not filename:
+ continue
+
+ # Check if it's a PDF file by extension, regardless of MIME type
+ is_pdf_by_extension = filename.lower().endswith(".pdf")
+
+ mime_type = part.get_content_type()
+ file_ext = os.path.splitext(filename)[1].lower()
+ # Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
+ if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension:
+ logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
+ continue
+
+ file_path = os.path.join(settings.workdir, filename)
+ with open(file_path, "wb") as f:
+ f.write(part.get_payload(decode=True))
+
+ # If it's a PDF by MIME type or extension, process it directly
+ if mime_type == "application/pdf" or is_pdf_by_extension:
+ process_document.delay(file_path, owner_id=owner_id)
+ logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type)
+ elif mime_type in ALLOWED_MIME_TYPES:
+ # Other allowed files are sent for conversion
+ convert_to_pdf.delay(file_path, owner_id=owner_id)
+ logger.info("Enqueued file for conversion to PDF: %s", filename)
+
+ has_attachment = True
+ return has_attachment
+
+
+def email_already_has_label(mail, msg_id, label="Ingested"):
+ """
+ Checks if the given message (msg_id) has the specified Gmail label.
+ Returns True if the label is found, False otherwise.
+ """
+ try:
+ # Convert msg_id to bytes if it's an integer
+ if isinstance(msg_id, int):
+ msg_id = str(msg_id).encode()
+
+ label_status, label_data = mail.fetch(msg_id, "(X-GM-LABELS)")
+ if label_status == "OK" and label_data and len(label_data) > 0:
+ raw_labels = label_data[0][1].decode("utf-8", errors="ignore")
+ if label in raw_labels:
+ return True
+ except Exception as e:
+ logger.error("Failed to fetch labels for msg_id=%s: %s", msg_id, e)
+ return False
+
+
+def mark_as_processed_with_star(mail, msg_id):
+ """Stars the email in Gmail."""
+ try:
+ mail.store(msg_id, "+FLAGS", "\\Flagged")
+ logger.info("Email %s starred in Gmail.", msg_id)
+ except Exception as e:
+ logger.error("Failed to star email %s: %s", msg_id, e)
+
+
+def mark_as_processed_with_label(mail, msg_id, label="Ingested"):
+ """Adds a custom label to the email in Gmail."""
+ try:
+ mail.store(msg_id, "+X-GM-LABELS", label)
+ logger.info("Email %s labeled '%s' in Gmail.", msg_id, label)
+ except Exception as e:
+ logger.error("Failed to label email %s with %s: %s", msg_id, label, e)
+
+
+def find_all_mail_folder(mail):
+ """
+ Attempts to select the Gmail All Mail folder using known localized names.
+ Falls back to using XLIST if needed.
+ Returns the folder name if found, otherwise None.
+ """
+ COMMON_ALL_MAIL_NAMES = [
+ "[Gmail]/Alle Nachrichten",
+ "[Gmail]/All Mail",
+ "[Gmail]/Todos",
+ "[Gmail]/Tutte le mail",
+ "[Gmail]/Tous les messages",
+ ]
+ for candidate in COMMON_ALL_MAIL_NAMES:
+ status, _ = mail.select(f'"{candidate}"', readonly=True)
+ if status == "OK":
+ return candidate
+
+ capabilities = get_capabilities(mail)
+ if "XLIST" in capabilities:
+ candidate = find_all_mail_xlist(mail)
+ if candidate:
+ return candidate
+ return None
+
+
+def get_capabilities(mail):
+ """Returns a list of capabilities supported by the IMAP server."""
+ typ, data = mail.capability()
+ if typ == "OK" and data:
+ caps = data[0].decode("utf-8", errors="ignore").upper().split()
+ return caps
+ return []
+
+
+def find_all_mail_xlist(mail):
+ """
+ Uses XLIST to discover the mailbox flagged as All Mail.
+ Returns the folder name if found, otherwise None.
+ """
+ tag = mail._new_tag().decode("ascii")
+ command_str = f'{tag} XLIST "" "*"'
+ mail.send((command_str + "\r\n").encode("utf-8"))
+
+ all_mail_folder = None
+ while True:
+ line = mail.readline()
+ if not line:
+ break
+ line_str = line.decode("utf-8", errors="ignore").strip()
+ if line_str.upper().startswith("* XLIST ") and "\\ALLMAIL" in line_str.upper():
+ match = re.search(r'"([^"]+)"$', line_str)
+ if match:
+ candidate = match.group(1)
+ logger.info("Found All Mail folder via XLIST: %s", candidate)
+ all_mail_folder = candidate
+ if line_str.startswith(tag):
+ break
+ return all_mail_folder
diff --git a/app/tasks/watch_folder_tasks.py b/app/tasks/watch_folder_tasks.py
index 4b1dfef0..cbd4338a 100644
--- a/app/tasks/watch_folder_tasks.py
+++ b/app/tasks/watch_folder_tasks.py
@@ -16,6 +16,7 @@ import json
import logging
import os
from datetime import datetime, timedelta, timezone
+from typing import Any
import redis
from celery import shared_task
@@ -1424,16 +1425,724 @@ def _scan_user_watch_folder(
return count
+# ---------------------------------------------------------------------------
+# Per-user cloud source watch folder scanning
+# ---------------------------------------------------------------------------
+
+# Maps source_type values to their per-user scan functions.
+_USER_WF_CLOUD_HANDLERS: dict[str, Any] = {} # populated after function defs
+
+
+def _scan_user_s3_folder(
+ cfg: dict,
+ creds: dict,
+ cache: dict[str, str],
+ delete_after: bool,
+ owner_id: str,
+) -> int:
+ """Scan an S3 bucket prefix using per-user credentials.
+
+ Args:
+ cfg: Integration config with ``bucket``, ``region``, ``prefix``, ``endpoint_url``.
+ creds: Decrypted credentials with ``access_key_id``, ``secret_access_key``.
+ cache: In-memory dict of already-processed file keys.
+ delete_after: Whether to remove the source object after ingestion.
+ owner_id: The user to attribute ingested documents to.
+
+ Returns:
+ Number of files newly enqueued.
+ """
+ try:
+ import boto3
+ from botocore.exceptions import ClientError
+ except ImportError as exc:
+ logger.error("User S3 watch folder: boto3 not installed: %s", exc)
+ return 0
+
+ bucket = cfg.get("bucket", "")
+ prefix = cfg.get("prefix", "")
+ region = cfg.get("region", "us-east-1")
+ endpoint_url = cfg.get("endpoint_url") or None
+
+ if not bucket:
+ logger.warning("User S3 watch folder: bucket not configured.")
+ return 0
+
+ access_key = creds.get("access_key_id", "")
+ secret_key = creds.get("secret_access_key", "")
+ if not (access_key and secret_key):
+ logger.warning("User S3 watch folder: credentials incomplete.")
+ return 0
+
+ try:
+ client_kwargs: dict = {
+ "region_name": region,
+ "aws_access_key_id": access_key,
+ "aws_secret_access_key": secret_key,
+ }
+ if endpoint_url:
+ client_kwargs["endpoint_url"] = endpoint_url
+ s3 = boto3.client("s3", **client_kwargs)
+ except Exception as exc:
+ logger.error("User S3 watch folder: failed to create client: %s", exc)
+ return 0
+
+ count = 0
+ paginator = s3.get_paginator("list_objects_v2")
+
+ try:
+ pages = paginator.paginate(Bucket=bucket, Prefix=prefix)
+ except Exception as exc:
+ logger.error("User S3 watch folder: failed to list %s/%s: %s", bucket, prefix, exc)
+ return 0
+
+ for page in pages:
+ for obj in page.get("Contents", []):
+ key = obj["Key"]
+ filename = key.split("/")[-1]
+ if not filename or not _is_allowed_file(filename):
+ continue
+
+ cache_key = f"s3:{bucket}/{key}"
+ if cache_key in cache:
+ continue
+
+ dest_path = os.path.join(settings.workdir, f"uwf_s3_{owner_id}_{filename}")
+ if os.path.exists(dest_path):
+ base2, ext2 = os.path.splitext(f"uwf_s3_{owner_id}_{filename}")
+ dest_path = os.path.join(settings.workdir, f"{base2}_{int(datetime.now().timestamp())}{ext2}")
+
+ try:
+ s3.download_file(bucket, key, dest_path)
+ logger.info("User S3 watch folder: downloaded s3://%s/%s", bucket, key)
+ except ClientError as exc:
+ logger.error("User S3 watch folder: failed to download %s: %s", key, exc)
+ if os.path.exists(dest_path):
+ os.remove(dest_path)
+ continue
+
+ _enqueue_file(dest_path, filename=filename, owner_id=owner_id)
+ _mark_processed(cache, cache_key)
+ count += 1
+
+ if delete_after:
+ try:
+ s3.delete_object(Bucket=bucket, Key=key)
+ logger.info("User S3 watch folder: deleted s3://%s/%s", bucket, key)
+ except Exception as exc:
+ logger.warning("User S3 watch folder: could not delete %s: %s", key, exc)
+
+ return count
+
+
+def _scan_user_dropbox_folder(
+ cfg: dict,
+ creds: dict,
+ cache: dict[str, str],
+ delete_after: bool,
+ owner_id: str,
+) -> int:
+ """Scan a Dropbox folder using per-user credentials.
+
+ Args:
+ cfg: Integration config with ``folder_path``.
+ creds: Decrypted credentials with ``refresh_token``, ``app_key``, ``app_secret``.
+ cache: In-memory dict of already-processed file keys.
+ delete_after: Whether to remove the source file after ingestion.
+ owner_id: The user to attribute ingested documents to.
+
+ Returns:
+ Number of files newly enqueued.
+ """
+ try:
+ import dropbox as dropbox_module
+ except ImportError as exc:
+ logger.error("User Dropbox watch folder: dropbox SDK not installed: %s", exc)
+ return 0
+
+ refresh_token = creds.get("refresh_token", "")
+ app_key = creds.get("app_key", "")
+ app_secret = creds.get("app_secret", "")
+ if not (refresh_token and app_key and app_secret):
+ logger.warning("User Dropbox watch folder: credentials incomplete.")
+ return 0
+
+ try:
+ dbx = dropbox_module.Dropbox(
+ oauth2_refresh_token=refresh_token,
+ app_key=app_key,
+ app_secret=app_secret,
+ )
+ except Exception as exc:
+ logger.error("User Dropbox watch folder: auth failed: %s", exc)
+ return 0
+
+ folder_path = cfg.get("folder_path", "")
+ if not folder_path:
+ logger.warning("User Dropbox watch folder: folder_path not configured.")
+ return 0
+
+ count = 0
+ try:
+ result = dbx.files_list_folder(folder_path)
+ entries = list(result.entries)
+ while result.has_more:
+ result = dbx.files_list_folder_continue(result.cursor)
+ entries.extend(result.entries)
+ except Exception as exc:
+ logger.error("User Dropbox watch folder: cannot list %s: %s", folder_path, exc)
+ return 0
+
+ for entry in entries:
+ if not isinstance(entry, dropbox_module.files.FileMetadata):
+ continue
+
+ filename = entry.name
+ if not _is_allowed_file(filename):
+ continue
+
+ cache_key = f"dropbox:{entry.id}"
+ if cache_key in cache:
+ continue
+
+ dest_path = os.path.join(settings.workdir, f"uwf_dbx_{owner_id}_{filename}")
+ if os.path.exists(dest_path):
+ base2, ext2 = os.path.splitext(f"uwf_dbx_{owner_id}_{filename}")
+ dest_path = os.path.join(settings.workdir, f"{base2}_{int(datetime.now().timestamp())}{ext2}")
+
+ try:
+ _meta, response = dbx.files_download(entry.path_lower)
+ with open(dest_path, "wb") as f:
+ f.write(response.content)
+ logger.info("User Dropbox watch folder: downloaded %s", filename)
+ except Exception as exc:
+ logger.error("User Dropbox watch folder: failed to download %s: %s", filename, exc)
+ if os.path.exists(dest_path):
+ os.remove(dest_path)
+ continue
+
+ _enqueue_file(dest_path, filename=filename, owner_id=owner_id)
+ _mark_processed(cache, cache_key)
+ count += 1
+
+ if delete_after:
+ try:
+ dbx.files_delete_v2(entry.path_lower)
+ logger.info("User Dropbox watch folder: deleted %s", entry.path_lower)
+ except Exception as exc:
+ logger.warning("User Dropbox watch folder: could not delete %s: %s", entry.path_lower, exc)
+
+ return count
+
+
+def _scan_user_google_drive_folder(
+ cfg: dict,
+ creds: dict,
+ cache: dict[str, str],
+ delete_after: bool,
+ owner_id: str,
+) -> int:
+ """Scan a Google Drive folder using per-user service-account credentials.
+
+ Args:
+ cfg: Integration config with ``folder_id``.
+ creds: Decrypted credentials with ``credentials_json``.
+ cache: In-memory dict of already-processed file keys.
+ delete_after: Whether to remove the source file after ingestion.
+ owner_id: The user to attribute ingested documents to.
+
+ Returns:
+ Number of files newly enqueued.
+ """
+ try:
+ from google.oauth2 import service_account
+ from googleapiclient.discovery import build
+ except ImportError as exc:
+ logger.error("User Google Drive watch folder: SDK not installed: %s", exc)
+ return 0
+
+ creds_json = creds.get("credentials_json", "")
+ if not creds_json:
+ logger.warning("User Google Drive watch folder: credentials_json not provided.")
+ return 0
+
+ folder_id = cfg.get("folder_id", "")
+ if not folder_id:
+ logger.warning("User Google Drive watch folder: folder_id not configured.")
+ return 0
+
+ try:
+ import json as _json
+
+ info = _json.loads(creds_json) if isinstance(creds_json, str) else creds_json
+ credentials = service_account.Credentials.from_service_account_info(
+ info,
+ scopes=["https://www.googleapis.com/auth/drive"],
+ )
+ service = build("drive", "v3", credentials=credentials)
+ except Exception as exc:
+ logger.error("User Google Drive watch folder: auth failed: %s", exc)
+ return 0
+
+ count = 0
+ query = f"'{folder_id}' in parents and trashed = false and mimeType != 'application/vnd.google-apps.folder'"
+ page_token = None
+
+ while True:
+ try:
+ params: dict = {
+ "q": query,
+ "fields": "nextPageToken, files(id, name, mimeType)",
+ "pageSize": 100,
+ }
+ if page_token:
+ params["pageToken"] = page_token
+ response = service.files().list(**params).execute()
+ except Exception as exc:
+ logger.error("User Google Drive watch folder: listing %s failed: %s", folder_id, exc)
+ break
+
+ for file_meta in response.get("files", []):
+ file_id_gd = file_meta["id"]
+ filename = file_meta["name"]
+
+ if not _is_allowed_file(filename):
+ continue
+
+ cache_key = f"gdrive:{file_id_gd}"
+ if cache_key in cache:
+ continue
+
+ dest_path = os.path.join(settings.workdir, f"uwf_gd_{owner_id}_{filename}")
+ if os.path.exists(dest_path):
+ base2, ext2 = os.path.splitext(f"uwf_gd_{owner_id}_{filename}")
+ dest_path = os.path.join(settings.workdir, f"{base2}_{int(datetime.now().timestamp())}{ext2}")
+
+ try:
+ import io
+
+ from googleapiclient.http import MediaIoBaseDownload
+
+ request = service.files().get_media(fileId=file_id_gd)
+ buf = io.BytesIO()
+ downloader = MediaIoBaseDownload(buf, request)
+ done = False
+ while not done:
+ _, done = downloader.next_chunk()
+ with open(dest_path, "wb") as f:
+ f.write(buf.getvalue())
+ logger.info("User Google Drive watch folder: downloaded %s", filename)
+ except Exception as exc:
+ logger.error("User Google Drive watch folder: download %s failed: %s", filename, exc)
+ if os.path.exists(dest_path):
+ os.remove(dest_path)
+ continue
+
+ _enqueue_file(dest_path, filename=filename, owner_id=owner_id)
+ _mark_processed(cache, cache_key)
+ count += 1
+
+ if delete_after:
+ try:
+ service.files().delete(fileId=file_id_gd).execute()
+ logger.info("User Google Drive watch folder: deleted %s", filename)
+ except Exception as exc:
+ logger.warning("User Google Drive watch folder: could not delete %s: %s", filename, exc)
+
+ page_token = response.get("nextPageToken")
+ if not page_token:
+ break
+
+ return count
+
+
+def _scan_user_onedrive_folder(
+ cfg: dict,
+ creds: dict,
+ cache: dict[str, str],
+ delete_after: bool,
+ owner_id: str,
+) -> int:
+ """Scan a OneDrive folder using per-user OAuth credentials.
+
+ Args:
+ cfg: Integration config with ``folder_path``.
+ creds: Decrypted credentials with ``refresh_token``, ``client_id``, ``client_secret``.
+ cache: In-memory dict of already-processed file keys.
+ delete_after: Whether to remove the source file after ingestion.
+ owner_id: The user to attribute ingested documents to.
+
+ Returns:
+ Number of files newly enqueued.
+ """
+ import requests as req_lib
+
+ refresh_token = creds.get("refresh_token", "")
+ client_id = creds.get("client_id", "")
+ client_secret = creds.get("client_secret", "")
+ if not (refresh_token and client_id and client_secret):
+ logger.warning("User OneDrive watch folder: credentials incomplete.")
+ return 0
+
+ folder_path = cfg.get("folder_path", "")
+ if not folder_path:
+ logger.warning("User OneDrive watch folder: folder_path not configured.")
+ return 0
+
+ # Exchange refresh token for an access token
+ try:
+ token_resp = req_lib.post(
+ "https://login.microsoftonline.com/common/oauth2/v2.0/token",
+ data={
+ "grant_type": "refresh_token",
+ "refresh_token": refresh_token,
+ "client_id": client_id,
+ "client_secret": client_secret,
+ "scope": "https://graph.microsoft.com/.default",
+ },
+ timeout=getattr(settings, "http_request_timeout", 120),
+ )
+ token_resp.raise_for_status()
+ access_token = token_resp.json()["access_token"]
+ except Exception as exc:
+ logger.error("User OneDrive watch folder: token exchange failed: %s", exc)
+ return 0
+
+ headers = {"Authorization": f"Bearer {access_token}"}
+ import urllib.parse
+
+ encoded_path = urllib.parse.quote(folder_path.lstrip("/"))
+ list_url: str | None = f"https://graph.microsoft.com/v1.0/me/drive/root:/{encoded_path}:/children"
+
+ count = 0
+ timeout = getattr(settings, "http_request_timeout", 120)
+
+ while list_url:
+ try:
+ resp = req_lib.get(list_url, headers=headers, timeout=timeout)
+ resp.raise_for_status()
+ data = resp.json()
+ except Exception as exc:
+ logger.error("User OneDrive watch folder: listing %s failed: %s", folder_path, exc)
+ break
+
+ for item in data.get("value", []):
+ if "folder" in item:
+ continue
+
+ filename = item["name"]
+ item_id = item["id"]
+
+ if not _is_allowed_file(filename):
+ continue
+
+ cache_key = f"onedrive:{item_id}"
+ if cache_key in cache:
+ continue
+
+ download_url = item.get("@microsoft.graph.downloadUrl")
+ if not download_url:
+ continue
+
+ dest_path = os.path.join(settings.workdir, f"uwf_od_{owner_id}_{filename}")
+ if os.path.exists(dest_path):
+ base2, ext2 = os.path.splitext(f"uwf_od_{owner_id}_{filename}")
+ dest_path = os.path.join(settings.workdir, f"{base2}_{int(datetime.now().timestamp())}{ext2}")
+
+ try:
+ dl_resp = req_lib.get(download_url, headers=headers, timeout=timeout)
+ dl_resp.raise_for_status()
+ with open(dest_path, "wb") as f:
+ f.write(dl_resp.content)
+ logger.info("User OneDrive watch folder: downloaded %s", filename)
+ except Exception as exc:
+ logger.error("User OneDrive watch folder: download %s failed: %s", filename, exc)
+ if os.path.exists(dest_path):
+ os.remove(dest_path)
+ continue
+
+ _enqueue_file(dest_path, filename=filename, owner_id=owner_id)
+ _mark_processed(cache, cache_key)
+ count += 1
+
+ if delete_after:
+ try:
+ del_resp = req_lib.delete(
+ f"https://graph.microsoft.com/v1.0/me/drive/items/{item_id}",
+ headers=headers,
+ timeout=timeout,
+ )
+ del_resp.raise_for_status()
+ logger.info("User OneDrive watch folder: deleted %s", filename)
+ except Exception as exc:
+ logger.warning("User OneDrive watch folder: could not delete %s: %s", filename, exc)
+
+ list_url = data.get("@odata.nextLink")
+
+ return count
+
+
+def _scan_user_nextcloud_folder(
+ cfg: dict,
+ creds: dict,
+ cache: dict[str, str],
+ delete_after: bool,
+ owner_id: str,
+) -> int:
+ """Scan a Nextcloud folder using per-user WebDAV credentials.
+
+ Args:
+ cfg: Integration config with ``url``, ``folder_path``.
+ creds: Decrypted credentials with ``username``, ``password``.
+ cache: In-memory dict of already-processed file keys.
+ delete_after: Whether to remove the source file after ingestion.
+ owner_id: The user to attribute ingested documents to.
+
+ Returns:
+ Number of files newly enqueued.
+ """
+ import defusedxml.ElementTree as ET
+ import requests as req_lib
+ from requests.auth import HTTPBasicAuth
+
+ nc_url = cfg.get("url", "")
+ folder_path = cfg.get("folder_path", "")
+ nc_user = creds.get("username", "")
+ nc_pass = creds.get("password", "")
+
+ if not (nc_url and nc_user and nc_pass):
+ logger.warning("User Nextcloud watch folder: connection settings incomplete.")
+ return 0
+
+ auth = HTTPBasicAuth(nc_user, nc_pass)
+ timeout = getattr(settings, "http_request_timeout", 120)
+
+ base = nc_url.rstrip("/")
+ folder = folder_path.strip("/")
+ propfind_url = f"{base}/{folder}/" if folder else f"{base}/"
+
+ try:
+ resp = req_lib.request(
+ "PROPFIND",
+ propfind_url,
+ auth=auth,
+ headers={"Depth": "1", "Content-Type": "application/xml"},
+ timeout=timeout,
+ )
+ resp.raise_for_status()
+ except Exception as exc:
+ logger.error("User Nextcloud watch folder: PROPFIND on %s failed: %s", propfind_url, exc)
+ return 0
+
+ count = 0
+ try:
+ root = ET.fromstring(resp.text) # noqa: S314 — defusedxml is safe
+ except Exception as exc:
+ logger.error("User Nextcloud watch folder: failed to parse response: %s", exc)
+ return 0
+
+ ns = {"d": "DAV:"}
+ for response_el in root.findall("d:response", ns):
+ href_el = response_el.find("d:href", ns)
+ if href_el is None or href_el.text is None:
+ continue
+
+ href = href_el.text
+ if href.rstrip("/").endswith(folder.rstrip("/")):
+ continue
+
+ import urllib.parse
+
+ filename = urllib.parse.unquote(href.rstrip("/").split("/")[-1])
+ if not _is_allowed_file(filename):
+ continue
+
+ cache_key = f"nextcloud:{href}"
+ if cache_key in cache:
+ continue
+
+ if href.startswith("http"):
+ file_url = href
+ else:
+ from urllib.parse import urlparse
+
+ parsed = urlparse(nc_url)
+ file_url = f"{parsed.scheme}://{parsed.netloc}{href}"
+
+ dest_path = os.path.join(settings.workdir, f"uwf_nc_{owner_id}_{filename}")
+ if os.path.exists(dest_path):
+ base_name, ext2 = os.path.splitext(f"uwf_nc_{owner_id}_{filename}")
+ dest_path = os.path.join(settings.workdir, f"{base_name}_{int(datetime.now().timestamp())}{ext2}")
+
+ try:
+ dl = req_lib.get(file_url, auth=auth, timeout=timeout)
+ dl.raise_for_status()
+ with open(dest_path, "wb") as f:
+ f.write(dl.content)
+ logger.info("User Nextcloud watch folder: downloaded %s", filename)
+ except Exception as exc:
+ logger.error("User Nextcloud watch folder: download %s failed: %s", filename, exc)
+ if os.path.exists(dest_path):
+ os.remove(dest_path)
+ continue
+
+ _enqueue_file(dest_path, filename=filename, owner_id=owner_id)
+ _mark_processed(cache, cache_key)
+ count += 1
+
+ if delete_after:
+ try:
+ del_resp = req_lib.request("DELETE", file_url, auth=auth, timeout=timeout)
+ del_resp.raise_for_status()
+ logger.info("User Nextcloud watch folder: deleted %s", filename)
+ except Exception as exc:
+ logger.warning("User Nextcloud watch folder: could not delete %s: %s", filename, exc)
+
+ return count
+
+
+def _scan_user_webdav_folder(
+ cfg: dict,
+ creds: dict,
+ cache: dict[str, str],
+ delete_after: bool,
+ owner_id: str,
+) -> int:
+ """Scan a WebDAV folder using per-user credentials.
+
+ Args:
+ cfg: Integration config with ``url``, ``folder_path``.
+ creds: Decrypted credentials with ``username``, ``password``.
+ cache: In-memory dict of already-processed file keys.
+ delete_after: Whether to remove the source file after ingestion.
+ owner_id: The user to attribute ingested documents to.
+
+ Returns:
+ Number of files newly enqueued.
+ """
+ import defusedxml.ElementTree as ET
+ import requests as req_lib
+ from requests.auth import HTTPBasicAuth
+
+ webdav_url = cfg.get("url", "")
+ folder_path = cfg.get("folder_path", "")
+ dav_user = creds.get("username", "")
+ dav_pass = creds.get("password", "")
+
+ if not webdav_url:
+ logger.warning("User WebDAV watch folder: URL not configured.")
+ return 0
+
+ base = webdav_url.rstrip("/")
+ folder = folder_path.strip("/")
+ propfind_url = f"{base}/{folder}/" if folder else f"{base}/"
+
+ auth = HTTPBasicAuth(dav_user, dav_pass) if dav_user else None
+ timeout = getattr(settings, "http_request_timeout", 120)
+
+ try:
+ resp = req_lib.request(
+ "PROPFIND",
+ propfind_url,
+ auth=auth,
+ headers={"Depth": "1"},
+ timeout=timeout,
+ )
+ resp.raise_for_status()
+ except Exception as exc:
+ logger.error("User WebDAV watch folder: PROPFIND on %s failed: %s", propfind_url, exc)
+ return 0
+
+ count = 0
+ try:
+ root = ET.fromstring(resp.text) # noqa: S314 — defusedxml is safe
+ except Exception as exc:
+ logger.error("User WebDAV watch folder: failed to parse response: %s", exc)
+ return 0
+
+ from urllib.parse import unquote, urlparse
+
+ ns = {"d": "DAV:"}
+ for response_el in root.findall("d:response", ns):
+ href_el = response_el.find("d:href", ns)
+ if href_el is None or href_el.text is None:
+ continue
+
+ href = href_el.text
+ if href.endswith("/"):
+ continue
+
+ filename = unquote(href.split("/")[-1])
+ if not _is_allowed_file(filename):
+ continue
+
+ cache_key = f"webdav:{href}"
+ if cache_key in cache:
+ continue
+
+ if href.startswith("http"):
+ file_url = href
+ else:
+ parsed = urlparse(webdav_url)
+ file_url = f"{parsed.scheme}://{parsed.netloc}{href}"
+
+ dest_path = os.path.join(settings.workdir, f"uwf_dav_{owner_id}_{filename}")
+ if os.path.exists(dest_path):
+ base2, ext2 = os.path.splitext(f"uwf_dav_{owner_id}_{filename}")
+ dest_path = os.path.join(settings.workdir, f"{base2}_{int(datetime.now().timestamp())}{ext2}")
+
+ try:
+ dl = req_lib.get(file_url, auth=auth, timeout=timeout)
+ dl.raise_for_status()
+ with open(dest_path, "wb") as f:
+ f.write(dl.content)
+ logger.info("User WebDAV watch folder: downloaded %s", filename)
+ except Exception as exc:
+ logger.error("User WebDAV watch folder: download %s failed: %s", filename, exc)
+ if os.path.exists(dest_path):
+ os.remove(dest_path)
+ continue
+
+ _enqueue_file(dest_path, filename=filename, owner_id=owner_id)
+ _mark_processed(cache, cache_key)
+ count += 1
+
+ if delete_after:
+ try:
+ del_resp = req_lib.request("DELETE", file_url, auth=auth, timeout=timeout)
+ del_resp.raise_for_status()
+ logger.info("User WebDAV watch folder: deleted %s", filename)
+ except Exception as exc:
+ logger.warning("User WebDAV watch folder: could not delete %s: %s", filename, exc)
+
+ return count
+
+
+# Populate the cloud handler dispatch table
+_USER_WF_CLOUD_HANDLERS.update(
+ {
+ "s3": _scan_user_s3_folder,
+ "dropbox": _scan_user_dropbox_folder,
+ "google_drive": _scan_user_google_drive_folder,
+ "onedrive": _scan_user_onedrive_folder,
+ "nextcloud": _scan_user_nextcloud_folder,
+ "webdav": _scan_user_webdav_folder,
+ }
+)
+
+
def _pull_user_integration_watch_folders() -> dict:
- """Iterate over all active WATCH_FOLDER UserIntegrations and scan their paths.
+ """Iterate over all active WATCH_FOLDER UserIntegrations and scan their sources.
Polls the ``user_integrations`` table for records with
``integration_type='WATCH_FOLDER'``, ``direction='SOURCE'``, and
``is_active=True``. Each integration's config is decoded and the
- configured ``folder_path`` is scanned for new files, which are enqueued
+ configured source is scanned for new files, which are enqueued
with the owning user's ``owner_id``.
- Path traversal protection is enforced on the configured path.
+ Path traversal protection is enforced on local filesystem paths.
+ Cloud source types (S3, Dropbox, Google Drive, OneDrive, Nextcloud, WebDAV)
+ are dispatched to their per-user scanning helpers.
Individual integration failures are caught and recorded without crashing
the polling loop.
@@ -1445,6 +2154,7 @@ def _pull_user_integration_watch_folders() -> dict:
import json as _json
from app.models import IntegrationDirection, IntegrationType, UserIntegration
+ from app.utils.encryption import decrypt_value
db = _get_db_session()
try:
@@ -1462,32 +2172,51 @@ def _pull_user_integration_watch_folders() -> dict:
for integ in integrations:
try:
cfg = _json.loads(integ.config) if integ.config else {}
- folder_path = cfg.get("folder_path", "")
delete_after = cfg.get("delete_after_process", False)
-
- if not folder_path:
- logger.warning(
- "Watch folder integration %d (owner %s) has no folder_path — skipping.",
- integ.id,
- integ.owner_id,
- )
- continue
-
- if not _is_safe_watch_path(folder_path):
- error_msg = f"Unsafe watch folder path rejected: {folder_path}"
- logger.error(
- "Watch folder integration %d (owner %s): %s",
- integ.id,
- integ.owner_id,
- error_msg,
- )
- integ.last_error = error_msg[:_MAX_ERROR_LENGTH]
- db.commit()
- continue
+ source_type = cfg.get("source_type", "local")
cache_file = f"{_USER_WF_CACHE_PREFIX}{integ.id}.json"
cache = _load_cache(cache_file)
- n = _scan_user_watch_folder(folder_path, cache, delete_after, integ.owner_id)
+
+ if source_type == "local":
+ # Local filesystem watch folder (original behaviour)
+ folder_path = cfg.get("folder_path", "")
+ if not folder_path:
+ logger.warning(
+ "Watch folder integration %d (owner %s) has no folder_path — skipping.",
+ integ.id,
+ integ.owner_id,
+ )
+ continue
+
+ if not _is_safe_watch_path(folder_path):
+ error_msg = f"Unsafe watch folder path rejected: {folder_path}"
+ logger.error(
+ "Watch folder integration %d (owner %s): %s",
+ integ.id,
+ integ.owner_id,
+ error_msg,
+ )
+ integ.last_error = error_msg[:_MAX_ERROR_LENGTH]
+ db.commit()
+ continue
+
+ n = _scan_user_watch_folder(folder_path, cache, delete_after, integ.owner_id)
+ elif source_type in _USER_WF_CLOUD_HANDLERS:
+ # Cloud source — decrypt per-user credentials and delegate
+ raw_creds = decrypt_value(integ.credentials) if integ.credentials else None
+ creds = _json.loads(raw_creds) if raw_creds else {}
+ handler = _USER_WF_CLOUD_HANDLERS[source_type]
+ n = handler(cfg, creds, cache, delete_after, integ.owner_id)
+ else:
+ logger.warning(
+ "Watch folder integration %d (owner %s): unknown source_type '%s' — skipping.",
+ integ.id,
+ integ.owner_id,
+ source_type,
+ )
+ continue
+
_save_cache(cache_file, cache)
total_files += n
@@ -1495,11 +2224,11 @@ def _pull_user_integration_watch_folders() -> dict:
integ.last_error = None
db.commit()
logger.info(
- "Watch folder integration %d (owner %s): %d file(s) enqueued from %s",
+ "Watch folder integration %d (owner %s): %d file(s) enqueued (source=%s)",
integ.id,
integ.owner_id,
n,
- folder_path,
+ source_type,
)
except Exception as exc: # noqa: BLE001
error_msg = str(exc)[:_MAX_ERROR_LENGTH]
diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md
index 424fa173..009dc5a9 100644
--- a/docs/ConfigurationGuide.md
+++ b/docs/ConfigurationGuide.md
@@ -272,10 +272,16 @@ DocuElevate can poll a WebDAV folder for new files. It reuses the existing WebDA
In addition to system-level watch folders, each user can configure personal watch folder sources through the **Integrations** dashboard (`/integrations`). Documents ingested from per-user watch folder integrations are automatically attributed to the owning user's `owner_id`.
Per-user watch folder integrations are stored in the `user_integrations` table with `integration_type='WATCH_FOLDER'` and `direction='SOURCE'`. The `config` JSON field stores:
-- `folder_path` — absolute path to the directory to scan
+- `source_type` — the type of source to scan (`local`, `s3`, `dropbox`, `google_drive`, `onedrive`, `nextcloud`, `webdav`; default: `local`)
+- `folder_path` — path to the directory/folder to scan (used by local, Dropbox, OneDrive, Nextcloud, WebDAV)
- `delete_after_process` — whether to remove source files after ingestion (default: `false`)
-> **Security**: Path traversal protection is enforced on user-configured watch folder paths. Relative paths, `..` components, and symlink escapes are rejected to prevent access to files outside the intended directory.
+Additional type-specific config fields:
+- **S3**: `bucket`, `region`, `prefix`, `endpoint_url`
+- **Google Drive**: `folder_id`
+- **Nextcloud / WebDAV**: `url`, `folder_path`
+
+> **Security**: Path traversal protection is enforced on local watch folder paths. Relative paths, `..` components, and symlink escapes are rejected. Cloud source types use per-user encrypted credentials instead.
- Individual scan failures are handled gracefully and recorded on the integration's `last_error` field without interrupting the scanning of other integrations.
- The scan runs alongside the system-level watch folder polling cycle.
@@ -302,7 +308,15 @@ DocuElevate can automatically pull document attachments from IMAP mailboxes —
In addition to system-level mailboxes, each user can configure personal IMAP sources through the **Integrations** dashboard (`/integrations`). Documents ingested from per-user IMAP integrations are automatically attributed to the owning user's `owner_id`.
-Per-user IMAP integrations are stored in the `user_integrations` table with `integration_type='IMAP'` and `direction='SOURCE'`. Credentials are encrypted at rest using Fernet encryption.
+Per-user IMAP integrations are stored in the `user_integrations` table with `integration_type='IMAP'` and `direction='SOURCE'`. The `config` JSON field stores:
+- `host` — IMAP server hostname (required)
+- `port` — IMAP server port (default: `993`)
+- `username` — IMAP login username (required)
+- `use_ssl` — whether to use SSL/TLS (default: `true`)
+- `delete_after_process` — whether to delete emails from the mailbox after processing (default: `false`)
+- `gmail_apply_labels` — whether to apply Gmail-specific labels and stars to processed emails (default: `true`). When enabled, processed emails are starred and tagged with an "Ingested" label. Only applies to Gmail hosts.
+
+Credentials are encrypted at rest using Fernet encryption.
- Individual connection failures are handled gracefully and recorded on the integration's `last_error` field without interrupting the polling of other integrations.
- The polling loop runs every minute and processes all active IMAP sources (system-level and per-user) in sequence.
diff --git a/docs/UserGuide.md b/docs/UserGuide.md
index fbdf972f..9906a89c 100644
--- a/docs/UserGuide.md
+++ b/docs/UserGuide.md
@@ -157,13 +157,13 @@ The **Integrations** page (`/integrations`) provides a unified view of all your
2. Choose a **Direction** — Source (ingestion) or Destination (storage).
3. Choose an **Integration Type** (e.g. IMAP, S3, Dropbox, WebDAV).
4. Fill in the type-specific fields — the form adapts dynamically based on your choice:
- - **IMAP** — host, port, username, password, SSL toggle
+ - **IMAP** — host, port, username, password, SSL toggle, delete after processing, Gmail labels & star toggle
- **S3** — bucket, region, access key, secret key
- **WebDAV / Nextcloud** — URL, folder, username, password
- **FTP / SFTP** — host, port, remote path, username, password
- **Dropbox / Google Drive / OneDrive** — folder path, with a link to the OAuth setup page
- **Email Forward** — recipient email address
- - **Watch Folder** — folder path
+ - **Watch Folder** — source type (Local, S3, Dropbox, Google Drive, OneDrive, Nextcloud, WebDAV), per-type config fields, delete after processing toggle
- **Paperless NGX** — URL and API token
- **Webhook** — no configuration needed; the form shows a quick-start guide with sample `curl` and Python snippets for uploading documents via the API
5. Click **Test Connection** to verify the settings before saving.
diff --git a/frontend/templates/integrations_dashboard.html b/frontend/templates/integrations_dashboard.html
index 5b71aaa7..ea530404 100644
--- a/frontend/templates/integrations_dashboard.html
+++ b/frontend/templates/integrations_dashboard.html
@@ -461,16 +461,28 @@
-
@@ -603,8 +615,182 @@
Watch Folder Settings
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -995,7 +1181,7 @@ function integrationsDashboard() {
this.testResult = null;
// Set sensible defaults
if (this.form.integration_type === 'IMAP') {
- this.form.config = { host: '', port: 993, username: '', use_ssl: true };
+ this.form.config = { host: '', port: 993, username: '', use_ssl: true, delete_after_process: false, gmail_apply_labels: true };
this.form.credentials = { password: '' };
} else if (this.form.integration_type === 'S3') {
this.form.config = { bucket: '', region: 'us-east-1', folder_prefix: '', endpoint_url: '' };
@@ -1013,7 +1199,7 @@ function integrationsDashboard() {
this.form.config = { recipient: '' };
this.form.credentials = {};
} else if (this.form.integration_type === 'WATCH_FOLDER') {
- this.form.config = { path: '' };
+ this.form.config = { source_type: 'local', folder_path: '', delete_after_process: false };
this.form.credentials = {};
} else if (this.form.integration_type === 'PAPERLESS') {
this.form.config = { url: '' };
@@ -1021,6 +1207,33 @@ function integrationsDashboard() {
}
},
+ onWatchFolderSourceTypeChange() {
+ const st = this.form.config.source_type;
+ const dap = this.form.config.delete_after_process || false;
+ this.form.credentials = {};
+ if (st === 'local') {
+ this.form.config = { source_type: st, folder_path: '', delete_after_process: dap };
+ } else if (st === 's3') {
+ this.form.config = { source_type: st, bucket: '', region: 'us-east-1', prefix: '', endpoint_url: '', delete_after_process: dap };
+ this.form.credentials = { access_key_id: '', secret_access_key: '' };
+ } else if (st === 'dropbox') {
+ this.form.config = { source_type: st, folder_path: '', delete_after_process: dap };
+ this.form.credentials = { refresh_token: '', app_key: '', app_secret: '' };
+ } else if (st === 'google_drive') {
+ this.form.config = { source_type: st, folder_id: '', delete_after_process: dap };
+ this.form.credentials = { credentials_json: '' };
+ } else if (st === 'onedrive') {
+ this.form.config = { source_type: st, folder_path: '', delete_after_process: dap };
+ this.form.credentials = { refresh_token: '', client_id: '', client_secret: '' };
+ } else if (st === 'nextcloud') {
+ this.form.config = { source_type: st, url: '', folder_path: '', delete_after_process: dap };
+ this.form.credentials = { username: '', password: '' };
+ } else if (st === 'webdav') {
+ this.form.config = { source_type: st, url: '', folder_path: '', delete_after_process: dap };
+ this.form.credentials = { username: '', password: '' };
+ }
+ },
+
openCreateModal() {
this.editingIntegration = null;
this.form = {
diff --git a/tests/test_imap_tasks.py b/tests/test_imap_tasks.py
index 3fb7b7f9..b35363bc 100644
--- a/tests/test_imap_tasks.py
+++ b/tests/test_imap_tasks.py
@@ -687,6 +687,59 @@ class TestPullInbox:
mock_star.assert_called_once_with(mock_mail, b"1")
mock_label.assert_called_once_with(mock_mail, b"1", label="Ingested")
+ @patch("app.tasks.imap_tasks.email_already_has_label")
+ @patch("app.tasks.imap_tasks.mark_as_processed_with_label")
+ @patch("app.tasks.imap_tasks.mark_as_processed_with_star")
+ @patch("app.tasks.imap_tasks.fetch_attachments_and_enqueue")
+ @patch("app.tasks.imap_tasks.imaplib.IMAP4_SSL")
+ @patch("app.tasks.imap_tasks.load_processed_emails")
+ @patch("app.tasks.imap_tasks.save_processed_emails")
+ @patch("app.tasks.imap_tasks.settings")
+ def test_gmail_labels_disabled_when_gmail_apply_labels_false(
+ self,
+ mock_settings,
+ mock_save,
+ mock_load,
+ mock_imap_class,
+ mock_fetch,
+ mock_star,
+ mock_label,
+ mock_has_label,
+ ):
+ """Gmail star/label operations should be skipped when gmail_apply_labels=False."""
+ mock_settings.workdir = "/tmp"
+ mock_settings.imap_readonly_mode = False
+ mock_load.return_value = {}
+ mock_mail = MagicMock()
+ mock_imap_class.return_value = mock_mail
+
+ import email
+
+ msg = email.message.EmailMessage()
+ msg["Message-ID"] = ""
+ raw_email = msg.as_bytes()
+
+ mock_mail.login.return_value = ("OK", [])
+ mock_mail.select.return_value = ("OK", [])
+ mock_mail.search.return_value = ("OK", [b"1"])
+ mock_mail.fetch.return_value = ("OK", [[None, raw_email]])
+
+ pull_inbox(
+ mailbox_key="imap2",
+ host="imap.gmail.com",
+ port=993,
+ username="user@gmail.com",
+ password=_TEST_CREDENTIAL,
+ use_ssl=True,
+ delete_after_process=False,
+ gmail_apply_labels=False,
+ )
+
+ mock_star.assert_not_called()
+ mock_label.assert_not_called()
+ mock_has_label.assert_not_called()
+ mock_fetch.assert_called()
+
@patch("app.tasks.imap_tasks.email_already_has_label")
@patch("app.tasks.imap_tasks.imaplib.IMAP4_SSL")
@patch("app.tasks.imap_tasks.load_processed_emails")
@@ -1531,6 +1584,55 @@ class TestPullUserIntegrationImap:
assert mock_integ.last_error is None
assert mock_integ.last_used_at is not None
+ @patch("app.tasks.imap_tasks._get_db_session")
+ @patch("app.tasks.imap_tasks.pull_inbox")
+ def test_passes_gmail_apply_labels_config_to_pull_inbox(self, mock_pull, mock_session_factory):
+ """gmail_apply_labels config should be forwarded to pull_inbox."""
+ from app.tasks.imap_tasks import _pull_user_integration_imap
+
+ mock_integ = MagicMock()
+ mock_integ.id = 14
+ mock_integ.owner_id = "owner-gmail"
+ mock_integ.config = (
+ '{"host": "imap.gmail.com", "port": 993, "username": "u@gmail.com",'
+ ' "use_ssl": true, "gmail_apply_labels": false}'
+ )
+ mock_integ.credentials = "enc:encrypted"
+
+ mock_db = MagicMock()
+ mock_db.query.return_value.filter.return_value.all.return_value = [mock_integ]
+ mock_session_factory.return_value = mock_db
+
+ with patch("app.utils.encryption.decrypt_value", return_value='{"password": "p"}'):
+ _pull_user_integration_imap()
+
+ mock_pull.assert_called_once()
+ call_kwargs = mock_pull.call_args
+ assert call_kwargs.kwargs.get("gmail_apply_labels") is False
+
+ @patch("app.tasks.imap_tasks._get_db_session")
+ @patch("app.tasks.imap_tasks.pull_inbox")
+ def test_gmail_apply_labels_defaults_to_true(self, mock_pull, mock_session_factory):
+ """Config without gmail_apply_labels should default to True."""
+ from app.tasks.imap_tasks import _pull_user_integration_imap
+
+ mock_integ = MagicMock()
+ mock_integ.id = 15
+ mock_integ.owner_id = "owner-gmail2"
+ mock_integ.config = '{"host": "imap.gmail.com", "port": 993, "username": "u@gmail.com", "use_ssl": true}'
+ mock_integ.credentials = "enc:encrypted"
+
+ mock_db = MagicMock()
+ mock_db.query.return_value.filter.return_value.all.return_value = [mock_integ]
+ mock_session_factory.return_value = mock_db
+
+ with patch("app.utils.encryption.decrypt_value", return_value='{"password": "p"}'):
+ _pull_user_integration_imap()
+
+ mock_pull.assert_called_once()
+ call_kwargs = mock_pull.call_args
+ assert call_kwargs.kwargs.get("gmail_apply_labels") is True
+
@pytest.mark.unit
class TestPullAllInboxesCallsIntegrations:
diff --git a/tests/test_watch_folder_tasks.py b/tests/test_watch_folder_tasks.py
index fccef1ec..46aa5444 100644
--- a/tests/test_watch_folder_tasks.py
+++ b/tests/test_watch_folder_tasks.py
@@ -3843,6 +3843,1987 @@ class TestPullUserIntegrationWatchFolders:
result = _pull_user_integration_watch_folders()
assert result["status"] == "ok"
+ @patch("app.tasks.watch_folder_tasks._get_db_session")
+ @patch("app.tasks.watch_folder_tasks._load_cache", return_value={})
+ @patch("app.tasks.watch_folder_tasks._save_cache")
+ def test_dispatches_s3_source_type(self, mock_save, mock_load, mock_session_factory):
+ """WATCH_FOLDER with source_type 's3' should dispatch to _scan_user_s3_folder."""
+ from app.tasks.watch_folder_tasks import (
+ _USER_WF_CLOUD_HANDLERS,
+ _pull_user_integration_watch_folders,
+ )
+
+ mock_integ = MagicMock()
+ mock_integ.id = 30
+ mock_integ.owner_id = "owner-s3"
+ mock_integ.config = (
+ '{"source_type": "s3", "bucket": "test-bucket", "prefix": "inbox/", "delete_after_process": false}'
+ )
+ mock_integ.is_active = True
+ mock_integ.credentials = "encrypted-s3-creds"
+
+ mock_db = MagicMock()
+ mock_db.query.return_value.filter.return_value.all.return_value = [mock_integ]
+ mock_session_factory.return_value = mock_db
+
+ mock_s3_handler = MagicMock(return_value=3)
+ original_handler = _USER_WF_CLOUD_HANDLERS.get("s3")
+ _USER_WF_CLOUD_HANDLERS["s3"] = mock_s3_handler
+ try:
+ with patch(
+ "app.utils.encryption.decrypt_value",
+ return_value='{"access_key_id": "AKI", "secret_access_key": "SK"}',
+ ):
+ result = _pull_user_integration_watch_folders()
+
+ assert result["status"] == "ok"
+ assert result["files_enqueued"] == 3
+ mock_s3_handler.assert_called_once()
+ args = mock_s3_handler.call_args
+ assert args[0][0]["source_type"] == "s3"
+ assert args[0][4] == "owner-s3"
+ finally:
+ if original_handler is not None:
+ _USER_WF_CLOUD_HANDLERS["s3"] = original_handler
+
+ @patch("app.tasks.watch_folder_tasks._get_db_session")
+ @patch("app.tasks.watch_folder_tasks._load_cache", return_value={})
+ @patch("app.tasks.watch_folder_tasks._save_cache")
+ def test_dispatches_dropbox_source_type(self, mock_save, mock_load, mock_session_factory):
+ """WATCH_FOLDER with source_type 'dropbox' should dispatch to _scan_user_dropbox_folder."""
+ from app.tasks.watch_folder_tasks import (
+ _USER_WF_CLOUD_HANDLERS,
+ _pull_user_integration_watch_folders,
+ )
+
+ mock_integ = MagicMock()
+ mock_integ.id = 31
+ mock_integ.owner_id = "owner-dbx"
+ mock_integ.config = '{"source_type": "dropbox", "folder_path": "/Inbox", "delete_after_process": false}'
+ mock_integ.is_active = True
+ mock_integ.credentials = "encrypted-dbx-creds"
+
+ mock_db = MagicMock()
+ mock_db.query.return_value.filter.return_value.all.return_value = [mock_integ]
+ mock_session_factory.return_value = mock_db
+
+ mock_dbx_handler = MagicMock(return_value=5)
+ original_handler = _USER_WF_CLOUD_HANDLERS.get("dropbox")
+ _USER_WF_CLOUD_HANDLERS["dropbox"] = mock_dbx_handler
+ try:
+ with patch(
+ "app.utils.encryption.decrypt_value",
+ return_value='{"refresh_token": "tok", "app_key": "ak", "app_secret": "as"}',
+ ):
+ result = _pull_user_integration_watch_folders()
+
+ assert result["status"] == "ok"
+ assert result["files_enqueued"] == 5
+ mock_dbx_handler.assert_called_once()
+ args = mock_dbx_handler.call_args
+ assert args[0][0]["source_type"] == "dropbox"
+ assert args[0][4] == "owner-dbx"
+ finally:
+ if original_handler is not None:
+ _USER_WF_CLOUD_HANDLERS["dropbox"] = original_handler
+
+ @patch("app.tasks.watch_folder_tasks._get_db_session")
+ def test_unknown_source_type_skipped(self, mock_session_factory):
+ """Unknown source_type should be skipped gracefully."""
+ from app.tasks.watch_folder_tasks import _pull_user_integration_watch_folders
+
+ mock_integ = MagicMock()
+ mock_integ.id = 32
+ mock_integ.owner_id = "owner-unknown"
+ mock_integ.config = '{"source_type": "unknown_provider", "delete_after_process": false}'
+ mock_integ.is_active = True
+
+ mock_db = MagicMock()
+ mock_db.query.return_value.filter.return_value.all.return_value = [mock_integ]
+ mock_session_factory.return_value = mock_db
+
+ result = _pull_user_integration_watch_folders()
+ assert result["status"] == "ok"
+ assert result["files_enqueued"] == 0
+
+ @patch("app.tasks.watch_folder_tasks._get_db_session")
+ @patch("app.tasks.watch_folder_tasks._scan_user_watch_folder", return_value=4)
+ @patch("app.tasks.watch_folder_tasks._load_cache", return_value={})
+ @patch("app.tasks.watch_folder_tasks._save_cache")
+ def test_local_source_type_dispatches_to_local_scanner(
+ self, mock_save, mock_load, mock_scan_local, mock_session_factory
+ ):
+ """Explicit source_type 'local' should use the local filesystem scanner."""
+ from app.tasks.watch_folder_tasks import _pull_user_integration_watch_folders
+
+ mock_integ = MagicMock()
+ mock_integ.id = 33
+ mock_integ.owner_id = "owner-local"
+ mock_integ.config = '{"source_type": "local", "folder_path": "/data/scans", "delete_after_process": false}'
+ mock_integ.is_active = True
+
+ mock_db = MagicMock()
+ mock_db.query.return_value.filter.return_value.all.return_value = [mock_integ]
+ mock_session_factory.return_value = mock_db
+
+ result = _pull_user_integration_watch_folders()
+ assert result["status"] == "ok"
+ assert result["files_enqueued"] == 4
+ mock_scan_local.assert_called_once()
+ assert mock_scan_local.call_args[0][0] == "/data/scans"
+ assert mock_scan_local.call_args[0][3] == "owner-local"
+
+ @patch("app.tasks.watch_folder_tasks._get_db_session")
+ @patch("app.tasks.watch_folder_tasks._load_cache", return_value={})
+ @patch("app.tasks.watch_folder_tasks._save_cache")
+ def test_dispatches_google_drive_source_type(self, mock_save, mock_load, mock_session_factory):
+ """WATCH_FOLDER with source_type 'google_drive' should dispatch to handler."""
+ from app.tasks.watch_folder_tasks import (
+ _USER_WF_CLOUD_HANDLERS,
+ _pull_user_integration_watch_folders,
+ )
+
+ mock_integ = MagicMock()
+ mock_integ.id = 34
+ mock_integ.owner_id = "owner-gd"
+ mock_integ.config = '{"source_type": "google_drive", "folder_id": "abc123", "delete_after_process": false}'
+ mock_integ.is_active = True
+ mock_integ.credentials = "encrypted-gd-creds"
+
+ mock_db = MagicMock()
+ mock_db.query.return_value.filter.return_value.all.return_value = [mock_integ]
+ mock_session_factory.return_value = mock_db
+
+ mock_gd_handler = MagicMock(return_value=2)
+ original_handler = _USER_WF_CLOUD_HANDLERS.get("google_drive")
+ _USER_WF_CLOUD_HANDLERS["google_drive"] = mock_gd_handler
+ try:
+ with patch(
+ "app.utils.encryption.decrypt_value",
+ return_value='{"credentials_json": "{}"}',
+ ):
+ result = _pull_user_integration_watch_folders()
+
+ assert result["status"] == "ok"
+ assert result["files_enqueued"] == 2
+ mock_gd_handler.assert_called_once()
+ args = mock_gd_handler.call_args
+ assert args[0][0]["source_type"] == "google_drive"
+ assert args[0][4] == "owner-gd"
+ finally:
+ if original_handler is not None:
+ _USER_WF_CLOUD_HANDLERS["google_drive"] = original_handler
+
+ @patch("app.tasks.watch_folder_tasks._get_db_session")
+ @patch("app.tasks.watch_folder_tasks._load_cache", return_value={})
+ @patch("app.tasks.watch_folder_tasks._save_cache")
+ def test_dispatches_onedrive_source_type(self, mock_save, mock_load, mock_session_factory):
+ """WATCH_FOLDER with source_type 'onedrive' should dispatch to handler."""
+ from app.tasks.watch_folder_tasks import (
+ _USER_WF_CLOUD_HANDLERS,
+ _pull_user_integration_watch_folders,
+ )
+
+ mock_integ = MagicMock()
+ mock_integ.id = 35
+ mock_integ.owner_id = "owner-od"
+ mock_integ.config = '{"source_type": "onedrive", "folder_path": "/Documents", "delete_after_process": false}'
+ mock_integ.is_active = True
+ mock_integ.credentials = "encrypted-od-creds"
+
+ mock_db = MagicMock()
+ mock_db.query.return_value.filter.return_value.all.return_value = [mock_integ]
+ mock_session_factory.return_value = mock_db
+
+ mock_od_handler = MagicMock(return_value=7)
+ original_handler = _USER_WF_CLOUD_HANDLERS.get("onedrive")
+ _USER_WF_CLOUD_HANDLERS["onedrive"] = mock_od_handler
+ try:
+ with patch(
+ "app.utils.encryption.decrypt_value",
+ return_value='{"refresh_token": "rt", "client_id": "ci", "client_secret": "cs"}',
+ ):
+ result = _pull_user_integration_watch_folders()
+
+ assert result["status"] == "ok"
+ assert result["files_enqueued"] == 7
+ mock_od_handler.assert_called_once()
+ args = mock_od_handler.call_args
+ assert args[0][0]["source_type"] == "onedrive"
+ assert args[0][4] == "owner-od"
+ finally:
+ if original_handler is not None:
+ _USER_WF_CLOUD_HANDLERS["onedrive"] = original_handler
+
+ @patch("app.tasks.watch_folder_tasks._get_db_session")
+ @patch("app.tasks.watch_folder_tasks._load_cache", return_value={})
+ @patch("app.tasks.watch_folder_tasks._save_cache")
+ def test_dispatches_nextcloud_source_type(self, mock_save, mock_load, mock_session_factory):
+ """WATCH_FOLDER with source_type 'nextcloud' should dispatch to handler."""
+ from app.tasks.watch_folder_tasks import (
+ _USER_WF_CLOUD_HANDLERS,
+ _pull_user_integration_watch_folders,
+ )
+
+ mock_integ = MagicMock()
+ mock_integ.id = 36
+ mock_integ.owner_id = "owner-nc"
+ mock_integ.config = (
+ '{"source_type": "nextcloud", "url": "https://nc.example.com",'
+ ' "folder_path": "/inbox", "delete_after_process": false}'
+ )
+ mock_integ.is_active = True
+ mock_integ.credentials = "encrypted-nc-creds"
+
+ mock_db = MagicMock()
+ mock_db.query.return_value.filter.return_value.all.return_value = [mock_integ]
+ mock_session_factory.return_value = mock_db
+
+ mock_nc_handler = MagicMock(return_value=1)
+ original_handler = _USER_WF_CLOUD_HANDLERS.get("nextcloud")
+ _USER_WF_CLOUD_HANDLERS["nextcloud"] = mock_nc_handler
+ try:
+ with patch(
+ "app.utils.encryption.decrypt_value",
+ return_value='{"username": "u", "password": "p"}',
+ ):
+ result = _pull_user_integration_watch_folders()
+
+ assert result["status"] == "ok"
+ assert result["files_enqueued"] == 1
+ mock_nc_handler.assert_called_once()
+ args = mock_nc_handler.call_args
+ assert args[0][0]["source_type"] == "nextcloud"
+ assert args[0][4] == "owner-nc"
+ finally:
+ if original_handler is not None:
+ _USER_WF_CLOUD_HANDLERS["nextcloud"] = original_handler
+
+ @patch("app.tasks.watch_folder_tasks._get_db_session")
+ @patch("app.tasks.watch_folder_tasks._load_cache", return_value={})
+ @patch("app.tasks.watch_folder_tasks._save_cache")
+ def test_dispatches_webdav_source_type(self, mock_save, mock_load, mock_session_factory):
+ """WATCH_FOLDER with source_type 'webdav' should dispatch to handler."""
+ from app.tasks.watch_folder_tasks import (
+ _USER_WF_CLOUD_HANDLERS,
+ _pull_user_integration_watch_folders,
+ )
+
+ mock_integ = MagicMock()
+ mock_integ.id = 37
+ mock_integ.owner_id = "owner-dav"
+ mock_integ.config = (
+ '{"source_type": "webdav", "url": "https://dav.example.com",'
+ ' "folder_path": "/inbox", "delete_after_process": false}'
+ )
+ mock_integ.is_active = True
+ mock_integ.credentials = "encrypted-dav-creds"
+
+ mock_db = MagicMock()
+ mock_db.query.return_value.filter.return_value.all.return_value = [mock_integ]
+ mock_session_factory.return_value = mock_db
+
+ mock_dav_handler = MagicMock(return_value=6)
+ original_handler = _USER_WF_CLOUD_HANDLERS.get("webdav")
+ _USER_WF_CLOUD_HANDLERS["webdav"] = mock_dav_handler
+ try:
+ with patch(
+ "app.utils.encryption.decrypt_value",
+ return_value='{"username": "u", "password": "p"}',
+ ):
+ result = _pull_user_integration_watch_folders()
+
+ assert result["status"] == "ok"
+ assert result["files_enqueued"] == 6
+ mock_dav_handler.assert_called_once()
+ args = mock_dav_handler.call_args
+ assert args[0][0]["source_type"] == "webdav"
+ assert args[0][4] == "owner-dav"
+ finally:
+ if original_handler is not None:
+ _USER_WF_CLOUD_HANDLERS["webdav"] = original_handler
+
+
+@pytest.mark.unit
+class TestScanUserS3Folder:
+ """Tests for _scan_user_s3_folder per-user S3 scanning."""
+
+ def test_returns_zero_when_bucket_not_configured(self):
+ """Empty bucket config should return 0 immediately."""
+ mock_boto3 = MagicMock()
+ mock_botocore = MagicMock()
+ mock_botocore_exc = MagicMock()
+ with patch.dict(
+ "sys.modules", {"boto3": mock_boto3, "botocore": mock_botocore, "botocore.exceptions": mock_botocore_exc}
+ ):
+ from app.tasks.watch_folder_tasks import _scan_user_s3_folder
+
+ result = _scan_user_s3_folder(
+ cfg={"prefix": "inbox/"},
+ creds={"access_key_id": "AKI", "secret_access_key": "SK"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_credentials_incomplete(self):
+ """Missing access_key_id or secret_access_key should return 0."""
+ mock_boto3 = MagicMock()
+ mock_botocore = MagicMock()
+ mock_botocore_exc = MagicMock()
+ with patch.dict(
+ "sys.modules", {"boto3": mock_boto3, "botocore": mock_botocore, "botocore.exceptions": mock_botocore_exc}
+ ):
+ from app.tasks.watch_folder_tasks import _scan_user_s3_folder
+
+ result = _scan_user_s3_folder(
+ cfg={"bucket": "my-bucket"},
+ creds={"access_key_id": "AKI"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_client_creation_fails(self):
+ """boto3.client raising should return 0."""
+ mock_boto3 = MagicMock()
+ mock_boto3.client.side_effect = Exception("Bad credentials")
+ mock_botocore = MagicMock()
+ mock_botocore_exc = MagicMock()
+ with patch.dict(
+ "sys.modules", {"boto3": mock_boto3, "botocore": mock_botocore, "botocore.exceptions": mock_botocore_exc}
+ ):
+ from app.tasks.watch_folder_tasks import _scan_user_s3_folder
+
+ result = _scan_user_s3_folder(
+ cfg={"bucket": "my-bucket"},
+ creds={"access_key_id": "AKI", "secret_access_key": "SK"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_pagination_fails(self):
+ """Paginator raising should return 0."""
+ mock_boto3 = MagicMock()
+ mock_s3 = MagicMock()
+ mock_boto3.client.return_value = mock_s3
+ mock_paginator = MagicMock()
+ mock_s3.get_paginator.return_value = mock_paginator
+ mock_paginator.paginate.side_effect = Exception("Access denied")
+ mock_botocore = MagicMock()
+ mock_botocore_exc = MagicMock()
+ with patch.dict(
+ "sys.modules", {"boto3": mock_boto3, "botocore": mock_botocore, "botocore.exceptions": mock_botocore_exc}
+ ):
+ from app.tasks.watch_folder_tasks import _scan_user_s3_folder
+
+ result = _scan_user_s3_folder(
+ cfg={"bucket": "my-bucket"},
+ creds={"access_key_id": "AKI", "secret_access_key": "SK"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_downloads_and_enqueues_new_file(self, tmp_path):
+ """Successful S3 download should enqueue file and return 1."""
+ mock_boto3 = MagicMock()
+ mock_s3 = MagicMock()
+ mock_boto3.client.return_value = mock_s3
+ mock_paginator = MagicMock()
+ mock_s3.get_paginator.return_value = mock_paginator
+ mock_paginator.paginate.return_value = [{"Contents": [{"Key": "inbox/doc.pdf"}]}]
+ mock_botocore = MagicMock()
+ mock_botocore_exc = MagicMock()
+ mock_botocore_exc.ClientError = type("ClientError", (Exception,), {})
+
+ with (
+ patch.dict(
+ "sys.modules",
+ {"boto3": mock_boto3, "botocore": mock_botocore, "botocore.exceptions": mock_botocore_exc},
+ ),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._mark_processed") as mock_mark,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_s3_folder
+
+ result = _scan_user_s3_folder(
+ cfg={"bucket": "my-bucket", "prefix": "inbox/"},
+ creds={"access_key_id": "AKI", "secret_access_key": "SK"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert result == 1
+ mock_enqueue.assert_called_once()
+ mock_mark.assert_called_once()
+
+ def test_skips_cached_file(self, tmp_path):
+ """Files already in cache should be skipped."""
+ mock_boto3 = MagicMock()
+ mock_s3 = MagicMock()
+ mock_boto3.client.return_value = mock_s3
+ mock_paginator = MagicMock()
+ mock_s3.get_paginator.return_value = mock_paginator
+ mock_paginator.paginate.return_value = [{"Contents": [{"Key": "inbox/doc.pdf"}]}]
+ mock_botocore = MagicMock()
+ mock_botocore_exc = MagicMock()
+
+ with (
+ patch.dict(
+ "sys.modules",
+ {"boto3": mock_boto3, "botocore": mock_botocore, "botocore.exceptions": mock_botocore_exc},
+ ),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_s3_folder
+
+ result = _scan_user_s3_folder(
+ cfg={"bucket": "my-bucket", "prefix": "inbox/"},
+ creds={"access_key_id": "AKI", "secret_access_key": "SK"},
+ cache={"s3:my-bucket/inbox/doc.pdf": "done"},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert result == 0
+ mock_enqueue.assert_not_called()
+
+ def test_skips_disallowed_file_type(self, tmp_path):
+ """Non-allowed files (e.g. .exe) should be skipped."""
+ mock_boto3 = MagicMock()
+ mock_s3 = MagicMock()
+ mock_boto3.client.return_value = mock_s3
+ mock_paginator = MagicMock()
+ mock_s3.get_paginator.return_value = mock_paginator
+ mock_paginator.paginate.return_value = [{"Contents": [{"Key": "inbox/virus.exe"}]}]
+ mock_botocore = MagicMock()
+ mock_botocore_exc = MagicMock()
+
+ with (
+ patch.dict(
+ "sys.modules",
+ {"boto3": mock_boto3, "botocore": mock_botocore, "botocore.exceptions": mock_botocore_exc},
+ ),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=False),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_s3_folder
+
+ result = _scan_user_s3_folder(
+ cfg={"bucket": "my-bucket", "prefix": "inbox/"},
+ creds={"access_key_id": "AKI", "secret_access_key": "SK"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert result == 0
+ mock_enqueue.assert_not_called()
+
+ def test_deletes_after_download(self, tmp_path):
+ """delete_after=True should call delete_object on the S3 key."""
+ mock_boto3 = MagicMock()
+ mock_s3 = MagicMock()
+ mock_boto3.client.return_value = mock_s3
+ mock_paginator = MagicMock()
+ mock_s3.get_paginator.return_value = mock_paginator
+ mock_paginator.paginate.return_value = [{"Contents": [{"Key": "inbox/doc.pdf"}]}]
+ mock_botocore = MagicMock()
+ mock_botocore_exc = MagicMock()
+ mock_botocore_exc.ClientError = type("ClientError", (Exception,), {})
+
+ with (
+ patch.dict(
+ "sys.modules",
+ {"boto3": mock_boto3, "botocore": mock_botocore, "botocore.exceptions": mock_botocore_exc},
+ ),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file"),
+ patch("app.tasks.watch_folder_tasks._mark_processed"),
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_s3_folder
+
+ result = _scan_user_s3_folder(
+ cfg={"bucket": "my-bucket", "prefix": "inbox/"},
+ creds={"access_key_id": "AKI", "secret_access_key": "SK"},
+ cache={},
+ delete_after=True,
+ owner_id="u1",
+ )
+
+ assert result == 1
+ mock_s3.delete_object.assert_called_once_with(Bucket="my-bucket", Key="inbox/doc.pdf")
+
+ def test_handles_download_failure(self, tmp_path):
+ """ClientError on download_file should skip the file and continue."""
+ mock_boto3 = MagicMock()
+ mock_s3 = MagicMock()
+ mock_boto3.client.return_value = mock_s3
+ mock_paginator = MagicMock()
+ mock_s3.get_paginator.return_value = mock_paginator
+ mock_paginator.paginate.return_value = [{"Contents": [{"Key": "inbox/doc.pdf"}]}]
+
+ mock_botocore = MagicMock()
+ mock_botocore_exc = MagicMock()
+ FakeClientError = type("ClientError", (Exception,), {})
+ mock_botocore_exc.ClientError = FakeClientError
+ mock_s3.download_file.side_effect = FakeClientError("403 Forbidden")
+
+ with (
+ patch.dict(
+ "sys.modules",
+ {"boto3": mock_boto3, "botocore": mock_botocore, "botocore.exceptions": mock_botocore_exc},
+ ),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_s3_folder
+
+ result = _scan_user_s3_folder(
+ cfg={"bucket": "my-bucket", "prefix": "inbox/"},
+ creds={"access_key_id": "AKI", "secret_access_key": "SK"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert result == 0
+ mock_enqueue.assert_not_called()
+
+ def test_handles_delete_failure_gracefully(self, tmp_path):
+ """delete_object raising should log warning but not crash."""
+ mock_boto3 = MagicMock()
+ mock_s3 = MagicMock()
+ mock_boto3.client.return_value = mock_s3
+ mock_paginator = MagicMock()
+ mock_s3.get_paginator.return_value = mock_paginator
+ mock_paginator.paginate.return_value = [{"Contents": [{"Key": "inbox/doc.pdf"}]}]
+ mock_s3.delete_object.side_effect = Exception("Delete denied")
+ mock_botocore = MagicMock()
+ mock_botocore_exc = MagicMock()
+ mock_botocore_exc.ClientError = type("ClientError", (Exception,), {})
+
+ with (
+ patch.dict(
+ "sys.modules",
+ {"boto3": mock_boto3, "botocore": mock_botocore, "botocore.exceptions": mock_botocore_exc},
+ ),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file"),
+ patch("app.tasks.watch_folder_tasks._mark_processed"),
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_s3_folder
+
+ result = _scan_user_s3_folder(
+ cfg={"bucket": "my-bucket", "prefix": "inbox/"},
+ creds={"access_key_id": "AKI", "secret_access_key": "SK"},
+ cache={},
+ delete_after=True,
+ owner_id="u1",
+ )
+
+ assert result == 1
+
+
+@pytest.mark.unit
+class TestScanUserDropboxFolder:
+ """Tests for _scan_user_dropbox_folder per-user Dropbox scanning."""
+
+ def _make_dropbox_mocks(self):
+ """Create standard dropbox mock module and helper types."""
+ mock_dropbox_mod = MagicMock()
+ FakeFileMeta = type("FileMetadata", (), {})
+ FakeFolderMeta = type("FolderMetadata", (), {})
+ mock_dropbox_mod.files.FileMetadata = FakeFileMeta
+ mock_dropbox_mod.files.FolderMetadata = FakeFolderMeta
+ return mock_dropbox_mod, FakeFileMeta, FakeFolderMeta
+
+ def test_returns_zero_when_credentials_incomplete(self):
+ """Missing refresh_token/app_key/app_secret should return 0."""
+ mock_dropbox_mod, _, _ = self._make_dropbox_mocks()
+ with patch.dict("sys.modules", {"dropbox": mock_dropbox_mod}):
+ from app.tasks.watch_folder_tasks import _scan_user_dropbox_folder
+
+ result = _scan_user_dropbox_folder(
+ cfg={"folder_path": "/inbox"},
+ creds={"refresh_token": "tok", "app_key": ""},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_folder_path_empty(self):
+ """Empty folder_path should return 0."""
+ mock_dropbox_mod, _, _ = self._make_dropbox_mocks()
+ with patch.dict("sys.modules", {"dropbox": mock_dropbox_mod}):
+ from app.tasks.watch_folder_tasks import _scan_user_dropbox_folder
+
+ result = _scan_user_dropbox_folder(
+ cfg={"folder_path": ""},
+ creds={"refresh_token": "tok", "app_key": "ak", "app_secret": "as"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_auth_fails(self):
+ """Dropbox constructor raising should return 0."""
+ mock_dropbox_mod, _, _ = self._make_dropbox_mocks()
+ mock_dropbox_mod.Dropbox.side_effect = Exception("Auth error")
+ with patch.dict("sys.modules", {"dropbox": mock_dropbox_mod}):
+ from app.tasks.watch_folder_tasks import _scan_user_dropbox_folder
+
+ result = _scan_user_dropbox_folder(
+ cfg={"folder_path": "/inbox"},
+ creds={"refresh_token": "tok", "app_key": "ak", "app_secret": "as"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_listing_fails(self):
+ """files_list_folder raising should return 0."""
+ mock_dropbox_mod, _, _ = self._make_dropbox_mocks()
+ mock_dbx_client = MagicMock()
+ mock_dropbox_mod.Dropbox.return_value = mock_dbx_client
+ mock_dbx_client.files_list_folder.side_effect = Exception("API error")
+
+ with patch.dict("sys.modules", {"dropbox": mock_dropbox_mod}):
+ from app.tasks.watch_folder_tasks import _scan_user_dropbox_folder
+
+ result = _scan_user_dropbox_folder(
+ cfg={"folder_path": "/inbox"},
+ creds={"refresh_token": "tok", "app_key": "ak", "app_secret": "as"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_downloads_and_enqueues_new_file(self, tmp_path):
+ """Successful Dropbox download should enqueue file and return 1."""
+ mock_dropbox_mod, FakeFileMeta, _ = self._make_dropbox_mocks()
+ entry = FakeFileMeta()
+ entry.name = "doc.pdf"
+ entry.id = "id:abc123"
+ entry.path_lower = "/inbox/doc.pdf"
+
+ mock_dbx_client = MagicMock()
+ mock_dropbox_mod.Dropbox.return_value = mock_dbx_client
+ result_obj = MagicMock()
+ result_obj.entries = [entry]
+ result_obj.has_more = False
+ mock_dbx_client.files_list_folder.return_value = result_obj
+
+ mock_response = MagicMock()
+ mock_response.content = b"%PDF-1.4 test"
+ mock_dbx_client.files_download.return_value = (MagicMock(), mock_response)
+
+ with (
+ patch.dict("sys.modules", {"dropbox": mock_dropbox_mod}),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._mark_processed") as mock_mark,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_dropbox_folder
+
+ count = _scan_user_dropbox_folder(
+ cfg={"folder_path": "/inbox"},
+ creds={"refresh_token": "tok", "app_key": "ak", "app_secret": "as"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 1
+ mock_enqueue.assert_called_once()
+ mock_mark.assert_called_once()
+
+ def test_skips_cached_file(self, tmp_path):
+ """Files already in cache should be skipped."""
+ mock_dropbox_mod, FakeFileMeta, _ = self._make_dropbox_mocks()
+ entry = FakeFileMeta()
+ entry.name = "doc.pdf"
+ entry.id = "id:abc123"
+ entry.path_lower = "/inbox/doc.pdf"
+
+ mock_dbx_client = MagicMock()
+ mock_dropbox_mod.Dropbox.return_value = mock_dbx_client
+ result_obj = MagicMock()
+ result_obj.entries = [entry]
+ result_obj.has_more = False
+ mock_dbx_client.files_list_folder.return_value = result_obj
+
+ with (
+ patch.dict("sys.modules", {"dropbox": mock_dropbox_mod}),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_dropbox_folder
+
+ count = _scan_user_dropbox_folder(
+ cfg={"folder_path": "/inbox"},
+ creds={"refresh_token": "tok", "app_key": "ak", "app_secret": "as"},
+ cache={"dropbox:id:abc123": "done"},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_skips_non_file_entries(self, tmp_path):
+ """FolderMetadata entries should be skipped."""
+ mock_dropbox_mod, _, FakeFolderMeta = self._make_dropbox_mocks()
+ folder_entry = FakeFolderMeta()
+ folder_entry.name = "subfolder"
+
+ mock_dbx_client = MagicMock()
+ mock_dropbox_mod.Dropbox.return_value = mock_dbx_client
+ result_obj = MagicMock()
+ result_obj.entries = [folder_entry]
+ result_obj.has_more = False
+ mock_dbx_client.files_list_folder.return_value = result_obj
+
+ with (
+ patch.dict("sys.modules", {"dropbox": mock_dropbox_mod}),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_dropbox_folder
+
+ count = _scan_user_dropbox_folder(
+ cfg={"folder_path": "/inbox"},
+ creds={"refresh_token": "tok", "app_key": "ak", "app_secret": "as"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_deletes_after_download(self, tmp_path):
+ """delete_after=True should call files_delete_v2."""
+ mock_dropbox_mod, FakeFileMeta, _ = self._make_dropbox_mocks()
+ entry = FakeFileMeta()
+ entry.name = "doc.pdf"
+ entry.id = "id:abc123"
+ entry.path_lower = "/inbox/doc.pdf"
+
+ mock_dbx_client = MagicMock()
+ mock_dropbox_mod.Dropbox.return_value = mock_dbx_client
+ result_obj = MagicMock()
+ result_obj.entries = [entry]
+ result_obj.has_more = False
+ mock_dbx_client.files_list_folder.return_value = result_obj
+
+ mock_response = MagicMock()
+ mock_response.content = b"%PDF-1.4 test"
+ mock_dbx_client.files_download.return_value = (MagicMock(), mock_response)
+
+ with (
+ patch.dict("sys.modules", {"dropbox": mock_dropbox_mod}),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file"),
+ patch("app.tasks.watch_folder_tasks._mark_processed"),
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_dropbox_folder
+
+ count = _scan_user_dropbox_folder(
+ cfg={"folder_path": "/inbox"},
+ creds={"refresh_token": "tok", "app_key": "ak", "app_secret": "as"},
+ cache={},
+ delete_after=True,
+ owner_id="u1",
+ )
+
+ assert count == 1
+ mock_dbx_client.files_delete_v2.assert_called_once_with("/inbox/doc.pdf")
+
+ def test_handles_download_failure(self, tmp_path):
+ """Download failure should skip the file and continue."""
+ mock_dropbox_mod, FakeFileMeta, _ = self._make_dropbox_mocks()
+ entry = FakeFileMeta()
+ entry.name = "doc.pdf"
+ entry.id = "id:abc123"
+ entry.path_lower = "/inbox/doc.pdf"
+
+ mock_dbx_client = MagicMock()
+ mock_dropbox_mod.Dropbox.return_value = mock_dbx_client
+ result_obj = MagicMock()
+ result_obj.entries = [entry]
+ result_obj.has_more = False
+ mock_dbx_client.files_list_folder.return_value = result_obj
+ mock_dbx_client.files_download.side_effect = Exception("Download failed")
+
+ with (
+ patch.dict("sys.modules", {"dropbox": mock_dropbox_mod}),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_dropbox_folder
+
+ count = _scan_user_dropbox_folder(
+ cfg={"folder_path": "/inbox"},
+ creds={"refresh_token": "tok", "app_key": "ak", "app_secret": "as"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+
+@pytest.mark.unit
+class TestScanUserGoogleDriveFolder:
+ """Tests for _scan_user_google_drive_folder per-user Google Drive scanning."""
+
+ def _make_google_mocks(self):
+ """Create Google API mock modules for sys.modules patching."""
+ mock_sa_mod = MagicMock()
+ mock_gapi_disc = MagicMock()
+ mock_gapi_http = MagicMock()
+
+ mock_service = MagicMock()
+ mock_gapi_disc.build.return_value = mock_service
+ mock_sa_mod.Credentials.from_service_account_info.return_value = MagicMock()
+
+ # Wire parent-child module attributes so `from X.Y import Z` resolves correctly
+ mock_google_oauth2 = MagicMock()
+ mock_google_oauth2.service_account = mock_sa_mod
+ mock_google = MagicMock()
+ mock_google.oauth2 = mock_google_oauth2
+
+ mock_gapi = MagicMock()
+ mock_gapi.discovery = mock_gapi_disc
+ mock_gapi.http = mock_gapi_http
+
+ modules = {
+ "google": mock_google,
+ "google.oauth2": mock_google_oauth2,
+ "google.oauth2.service_account": mock_sa_mod,
+ "googleapiclient": mock_gapi,
+ "googleapiclient.discovery": mock_gapi_disc,
+ "googleapiclient.http": mock_gapi_http,
+ }
+ return modules, mock_service, mock_gapi_http, mock_sa_mod
+
+ def test_returns_zero_when_credentials_json_empty(self):
+ """Empty credentials_json should return 0."""
+ modules, _, _, _ = self._make_google_mocks()
+ with patch.dict("sys.modules", modules):
+ from app.tasks.watch_folder_tasks import _scan_user_google_drive_folder
+
+ result = _scan_user_google_drive_folder(
+ cfg={"folder_id": "abc"},
+ creds={"credentials_json": ""},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_folder_id_empty(self):
+ """Empty folder_id should return 0."""
+ modules, _, _, _ = self._make_google_mocks()
+ with patch.dict("sys.modules", modules):
+ from app.tasks.watch_folder_tasks import _scan_user_google_drive_folder
+
+ result = _scan_user_google_drive_folder(
+ cfg={"folder_id": ""},
+ creds={"credentials_json": '{"type": "service_account"}'},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_auth_fails(self):
+ """service_account.Credentials raising should return 0."""
+ modules, _, _, mock_sa_mod = self._make_google_mocks()
+ mock_sa_mod.Credentials.from_service_account_info.side_effect = Exception("Bad SA key")
+ with patch.dict("sys.modules", modules):
+ from app.tasks.watch_folder_tasks import _scan_user_google_drive_folder
+
+ result = _scan_user_google_drive_folder(
+ cfg={"folder_id": "abc"},
+ creds={"credentials_json": '{"type": "service_account"}'},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_downloads_and_enqueues_new_file(self, tmp_path):
+ """Successful Google Drive download should enqueue file and return 1."""
+ modules, mock_service, mock_gapi_http, _ = self._make_google_mocks()
+
+ mock_service.files.return_value.list.return_value.execute.return_value = {
+ "files": [{"id": "f1", "name": "doc.pdf", "mimeType": "application/pdf"}],
+ }
+
+ mock_downloader = MagicMock()
+ mock_downloader.next_chunk.return_value = (None, True)
+ mock_gapi_http.MediaIoBaseDownload.return_value = mock_downloader
+
+ with (
+ patch.dict("sys.modules", modules),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._mark_processed") as mock_mark,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_google_drive_folder
+
+ count = _scan_user_google_drive_folder(
+ cfg={"folder_id": "abc"},
+ creds={"credentials_json": '{"type": "service_account"}'},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 1
+ mock_enqueue.assert_called_once()
+ mock_mark.assert_called_once()
+
+ def test_skips_cached_file(self, tmp_path):
+ """Files already in cache should be skipped."""
+ modules, mock_service, _, _ = self._make_google_mocks()
+
+ mock_service.files.return_value.list.return_value.execute.return_value = {
+ "files": [{"id": "f1", "name": "doc.pdf", "mimeType": "application/pdf"}],
+ }
+
+ with (
+ patch.dict("sys.modules", modules),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_google_drive_folder
+
+ count = _scan_user_google_drive_folder(
+ cfg={"folder_id": "abc"},
+ creds={"credentials_json": '{"type": "service_account"}'},
+ cache={"gdrive:f1": "done"},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_deletes_after_download(self, tmp_path):
+ """delete_after=True should call files().delete()."""
+ modules, mock_service, mock_gapi_http, _ = self._make_google_mocks()
+
+ mock_service.files.return_value.list.return_value.execute.return_value = {
+ "files": [{"id": "f1", "name": "doc.pdf", "mimeType": "application/pdf"}],
+ }
+ mock_downloader = MagicMock()
+ mock_downloader.next_chunk.return_value = (None, True)
+ mock_gapi_http.MediaIoBaseDownload.return_value = mock_downloader
+
+ with (
+ patch.dict("sys.modules", modules),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file"),
+ patch("app.tasks.watch_folder_tasks._mark_processed"),
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_google_drive_folder
+
+ count = _scan_user_google_drive_folder(
+ cfg={"folder_id": "abc"},
+ creds={"credentials_json": '{"type": "service_account"}'},
+ cache={},
+ delete_after=True,
+ owner_id="u1",
+ )
+
+ assert count == 1
+ mock_service.files.return_value.delete.assert_called_once_with(fileId="f1")
+
+ def test_handles_download_failure(self, tmp_path):
+ """Download failure should skip the file and continue."""
+ modules, mock_service, mock_gapi_http, _ = self._make_google_mocks()
+
+ mock_service.files.return_value.list.return_value.execute.return_value = {
+ "files": [{"id": "f1", "name": "doc.pdf", "mimeType": "application/pdf"}],
+ }
+ mock_gapi_http.MediaIoBaseDownload.side_effect = Exception("Download failed")
+
+ with (
+ patch.dict("sys.modules", modules),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_google_drive_folder
+
+ count = _scan_user_google_drive_folder(
+ cfg={"folder_id": "abc"},
+ creds={"credentials_json": '{"type": "service_account"}'},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_handles_listing_failure(self, tmp_path):
+ """Listing failure should break and return 0."""
+ modules, mock_service, _, _ = self._make_google_mocks()
+
+ mock_service.files.return_value.list.return_value.execute.side_effect = Exception("API error")
+
+ with (
+ patch.dict("sys.modules", modules),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ from app.tasks.watch_folder_tasks import _scan_user_google_drive_folder
+
+ count = _scan_user_google_drive_folder(
+ cfg={"folder_id": "abc"},
+ creds={"credentials_json": '{"type": "service_account"}'},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+
+@pytest.mark.unit
+class TestScanUserOneDriveFolder:
+ """Tests for _scan_user_onedrive_folder per-user OneDrive scanning."""
+
+ def test_returns_zero_when_credentials_incomplete(self):
+ """Missing refresh_token/client_id/client_secret should return 0."""
+ from app.tasks.watch_folder_tasks import _scan_user_onedrive_folder
+
+ result = _scan_user_onedrive_folder(
+ cfg={"folder_path": "/Documents"},
+ creds={"refresh_token": "tok", "client_id": ""},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_folder_path_empty(self):
+ """Empty folder_path should return 0."""
+ from app.tasks.watch_folder_tasks import _scan_user_onedrive_folder
+
+ result = _scan_user_onedrive_folder(
+ cfg={"folder_path": ""},
+ creds={"refresh_token": "tok", "client_id": "ci", "client_secret": "cs"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_token_exchange_fails(self):
+ """Token exchange failure should return 0."""
+ from app.tasks.watch_folder_tasks import _scan_user_onedrive_folder
+
+ with patch("requests.post") as mock_post:
+ mock_post.side_effect = Exception("Token error")
+
+ result = _scan_user_onedrive_folder(
+ cfg={"folder_path": "/Documents"},
+ creds={"refresh_token": "tok", "client_id": "ci", "client_secret": "cs"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_downloads_and_enqueues_new_file(self, tmp_path):
+ """Successful OneDrive download should enqueue file and return 1."""
+ from app.tasks.watch_folder_tasks import _scan_user_onedrive_folder
+
+ mock_token_resp = MagicMock()
+ mock_token_resp.json.return_value = {"access_token": "tok123"}
+
+ mock_list_resp = MagicMock()
+ mock_list_resp.json.return_value = {
+ "value": [
+ {
+ "name": "doc.pdf",
+ "id": "item1",
+ "@microsoft.graph.downloadUrl": "https://dl.example.com/doc.pdf",
+ }
+ ],
+ }
+
+ mock_dl_resp = MagicMock()
+ mock_dl_resp.content = b"%PDF-1.4"
+
+ with (
+ patch("requests.post", return_value=mock_token_resp),
+ patch("requests.get", side_effect=[mock_list_resp, mock_dl_resp]),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._mark_processed") as mock_mark,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_onedrive_folder(
+ cfg={"folder_path": "/Documents"},
+ creds={"refresh_token": "tok", "client_id": "ci", "client_secret": "cs"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 1
+ mock_enqueue.assert_called_once()
+ mock_mark.assert_called_once()
+
+ def test_skips_folders(self, tmp_path):
+ """Items with 'folder' key should be skipped."""
+ from app.tasks.watch_folder_tasks import _scan_user_onedrive_folder
+
+ mock_token_resp = MagicMock()
+ mock_token_resp.json.return_value = {"access_token": "tok123"}
+
+ mock_list_resp = MagicMock()
+ mock_list_resp.json.return_value = {
+ "value": [{"name": "subfolder", "id": "fold1", "folder": {"childCount": 3}}],
+ }
+
+ with (
+ patch("requests.post", return_value=mock_token_resp),
+ patch("requests.get", return_value=mock_list_resp),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_onedrive_folder(
+ cfg={"folder_path": "/Documents"},
+ creds={"refresh_token": "tok", "client_id": "ci", "client_secret": "cs"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_skips_cached_file(self, tmp_path):
+ """Files already in cache should be skipped."""
+ from app.tasks.watch_folder_tasks import _scan_user_onedrive_folder
+
+ mock_token_resp = MagicMock()
+ mock_token_resp.json.return_value = {"access_token": "tok123"}
+
+ mock_list_resp = MagicMock()
+ mock_list_resp.json.return_value = {
+ "value": [
+ {
+ "name": "doc.pdf",
+ "id": "item1",
+ "@microsoft.graph.downloadUrl": "https://dl.example.com/doc.pdf",
+ }
+ ],
+ }
+
+ with (
+ patch("requests.post", return_value=mock_token_resp),
+ patch("requests.get", return_value=mock_list_resp),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_onedrive_folder(
+ cfg={"folder_path": "/Documents"},
+ creds={"refresh_token": "tok", "client_id": "ci", "client_secret": "cs"},
+ cache={"onedrive:item1": "done"},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_skips_items_without_download_url(self, tmp_path):
+ """Items missing @microsoft.graph.downloadUrl should be skipped."""
+ from app.tasks.watch_folder_tasks import _scan_user_onedrive_folder
+
+ mock_token_resp = MagicMock()
+ mock_token_resp.json.return_value = {"access_token": "tok123"}
+
+ mock_list_resp = MagicMock()
+ mock_list_resp.json.return_value = {
+ "value": [{"name": "doc.pdf", "id": "item1"}],
+ }
+
+ with (
+ patch("requests.post", return_value=mock_token_resp),
+ patch("requests.get", return_value=mock_list_resp),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_onedrive_folder(
+ cfg={"folder_path": "/Documents"},
+ creds={"refresh_token": "tok", "client_id": "ci", "client_secret": "cs"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_deletes_after_download(self, tmp_path):
+ """delete_after=True should call requests.delete on the item."""
+ from app.tasks.watch_folder_tasks import _scan_user_onedrive_folder
+
+ mock_token_resp = MagicMock()
+ mock_token_resp.json.return_value = {"access_token": "tok123"}
+
+ mock_list_resp = MagicMock()
+ mock_list_resp.json.return_value = {
+ "value": [
+ {
+ "name": "doc.pdf",
+ "id": "item1",
+ "@microsoft.graph.downloadUrl": "https://dl.example.com/doc.pdf",
+ }
+ ],
+ }
+
+ mock_dl_resp = MagicMock()
+ mock_dl_resp.content = b"%PDF-1.4"
+ mock_del_resp = MagicMock()
+
+ with (
+ patch("requests.post", return_value=mock_token_resp),
+ patch("requests.get", side_effect=[mock_list_resp, mock_dl_resp]),
+ patch("requests.delete", return_value=mock_del_resp) as mock_delete,
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file"),
+ patch("app.tasks.watch_folder_tasks._mark_processed"),
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_onedrive_folder(
+ cfg={"folder_path": "/Documents"},
+ creds={"refresh_token": "tok", "client_id": "ci", "client_secret": "cs"},
+ cache={},
+ delete_after=True,
+ owner_id="u1",
+ )
+
+ assert count == 1
+ mock_delete.assert_called_once()
+
+ def test_handles_download_failure(self, tmp_path):
+ """Download failure should skip the file and continue."""
+ from app.tasks.watch_folder_tasks import _scan_user_onedrive_folder
+
+ mock_token_resp = MagicMock()
+ mock_token_resp.json.return_value = {"access_token": "tok123"}
+
+ mock_list_resp = MagicMock()
+ mock_list_resp.json.return_value = {
+ "value": [
+ {
+ "name": "doc.pdf",
+ "id": "item1",
+ "@microsoft.graph.downloadUrl": "https://dl.example.com/doc.pdf",
+ }
+ ],
+ }
+
+ mock_dl_resp = MagicMock()
+ mock_dl_resp.raise_for_status.side_effect = Exception("Download failed")
+
+ with (
+ patch("requests.post", return_value=mock_token_resp),
+ patch("requests.get", side_effect=[mock_list_resp, mock_dl_resp]),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_onedrive_folder(
+ cfg={"folder_path": "/Documents"},
+ creds={"refresh_token": "tok", "client_id": "ci", "client_secret": "cs"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_handles_listing_failure(self, tmp_path):
+ """Listing failure should break and return 0."""
+ from app.tasks.watch_folder_tasks import _scan_user_onedrive_folder
+
+ mock_token_resp = MagicMock()
+ mock_token_resp.json.return_value = {"access_token": "tok123"}
+
+ mock_list_resp = MagicMock()
+ mock_list_resp.raise_for_status.side_effect = Exception("Listing failed")
+
+ with (
+ patch("requests.post", return_value=mock_token_resp),
+ patch("requests.get", return_value=mock_list_resp),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_onedrive_folder(
+ cfg={"folder_path": "/Documents"},
+ creds={"refresh_token": "tok", "client_id": "ci", "client_secret": "cs"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+
+def _build_propfind_xml(href_paths: list[str]) -> str:
+ """Build a minimal PROPFIND multi-status XML response for testing."""
+ responses = ""
+ for href in href_paths:
+ responses += f"""
+
+ {href}
+
+ 1024
+ HTTP/1.1 200 OK
+
+ """
+ return f"""
+
+ {responses}
+ """
+
+
+@pytest.mark.unit
+class TestScanUserNextcloudFolder:
+ """Tests for _scan_user_nextcloud_folder per-user Nextcloud scanning."""
+
+ def test_returns_zero_when_settings_incomplete(self):
+ """Missing url, username, or password should return 0."""
+ from app.tasks.watch_folder_tasks import _scan_user_nextcloud_folder
+
+ result = _scan_user_nextcloud_folder(
+ cfg={"url": "", "folder_path": "/inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_propfind_fails(self):
+ """PROPFIND failure should return 0."""
+ from app.tasks.watch_folder_tasks import _scan_user_nextcloud_folder
+
+ with patch("requests.request") as mock_request:
+ mock_request.side_effect = Exception("Connection refused")
+
+ result = _scan_user_nextcloud_folder(
+ cfg={"url": "https://nc.example.com/remote.php/dav/files/user", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_xml_parse_fails(self):
+ """Invalid XML response should return 0."""
+ from app.tasks.watch_folder_tasks import _scan_user_nextcloud_folder
+
+ mock_resp = MagicMock()
+ mock_resp.text = "not valid xml <<<"
+
+ with patch("requests.request", return_value=mock_resp):
+ result = _scan_user_nextcloud_folder(
+ cfg={"url": "https://nc.example.com/remote.php/dav/files/user", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_downloads_and_enqueues_new_file(self, tmp_path):
+ """Successful Nextcloud download should enqueue file and return 1."""
+ from app.tasks.watch_folder_tasks import _scan_user_nextcloud_folder
+
+ propfind_xml = _build_propfind_xml(
+ [
+ "/remote.php/dav/files/user/inbox/",
+ "/remote.php/dav/files/user/inbox/doc.pdf",
+ ]
+ )
+
+ mock_propfind_resp = MagicMock()
+ mock_propfind_resp.text = propfind_xml
+
+ mock_dl_resp = MagicMock()
+ mock_dl_resp.content = b"%PDF-1.4"
+
+ with (
+ patch("requests.request", return_value=mock_propfind_resp),
+ patch("requests.get", return_value=mock_dl_resp),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._mark_processed") as mock_mark,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_nextcloud_folder(
+ cfg={"url": "https://nc.example.com/remote.php/dav/files/user", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 1
+ mock_enqueue.assert_called_once()
+ mock_mark.assert_called_once()
+
+ def test_skips_folder_self_entry(self, tmp_path):
+ """The folder itself (self entry) should be skipped."""
+ from app.tasks.watch_folder_tasks import _scan_user_nextcloud_folder
+
+ propfind_xml = _build_propfind_xml(["/remote.php/dav/files/user/inbox/"])
+
+ mock_propfind_resp = MagicMock()
+ mock_propfind_resp.text = propfind_xml
+
+ with (
+ patch("requests.request", return_value=mock_propfind_resp),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_nextcloud_folder(
+ cfg={"url": "https://nc.example.com/remote.php/dav/files/user", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_skips_cached_file(self, tmp_path):
+ """Files already in cache should be skipped."""
+ from app.tasks.watch_folder_tasks import _scan_user_nextcloud_folder
+
+ propfind_xml = _build_propfind_xml(
+ [
+ "/remote.php/dav/files/user/inbox/",
+ "/remote.php/dav/files/user/inbox/doc.pdf",
+ ]
+ )
+
+ mock_propfind_resp = MagicMock()
+ mock_propfind_resp.text = propfind_xml
+
+ with (
+ patch("requests.request", return_value=mock_propfind_resp),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_nextcloud_folder(
+ cfg={"url": "https://nc.example.com/remote.php/dav/files/user", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={"nextcloud:/remote.php/dav/files/user/inbox/doc.pdf": "done"},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_deletes_after_download(self, tmp_path):
+ """delete_after=True should send DELETE request."""
+ from app.tasks.watch_folder_tasks import _scan_user_nextcloud_folder
+
+ propfind_xml = _build_propfind_xml(
+ [
+ "/remote.php/dav/files/user/inbox/",
+ "/remote.php/dav/files/user/inbox/doc.pdf",
+ ]
+ )
+
+ mock_propfind_resp = MagicMock()
+ mock_propfind_resp.text = propfind_xml
+
+ mock_dl_resp = MagicMock()
+ mock_dl_resp.content = b"%PDF-1.4"
+
+ mock_del_resp = MagicMock()
+
+ with (
+ patch("requests.request", side_effect=[mock_propfind_resp, mock_del_resp]) as mock_request,
+ patch("requests.get", return_value=mock_dl_resp),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file"),
+ patch("app.tasks.watch_folder_tasks._mark_processed"),
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_nextcloud_folder(
+ cfg={"url": "https://nc.example.com/remote.php/dav/files/user", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=True,
+ owner_id="u1",
+ )
+
+ assert count == 1
+ assert mock_request.call_count == 2
+ delete_call = mock_request.call_args_list[1]
+ assert delete_call[0][0] == "DELETE"
+
+ def test_handles_download_failure(self, tmp_path):
+ """Download failure should skip the file and continue."""
+ from app.tasks.watch_folder_tasks import _scan_user_nextcloud_folder
+
+ propfind_xml = _build_propfind_xml(
+ [
+ "/remote.php/dav/files/user/inbox/",
+ "/remote.php/dav/files/user/inbox/doc.pdf",
+ ]
+ )
+
+ mock_propfind_resp = MagicMock()
+ mock_propfind_resp.text = propfind_xml
+
+ with (
+ patch("requests.request", return_value=mock_propfind_resp),
+ patch("requests.get") as mock_get,
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+ mock_get.side_effect = Exception("Download failed")
+
+ count = _scan_user_nextcloud_folder(
+ cfg={"url": "https://nc.example.com/remote.php/dav/files/user", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_handles_absolute_href_url(self, tmp_path):
+ """Absolute href URLs should be used directly for download."""
+ from app.tasks.watch_folder_tasks import _scan_user_nextcloud_folder
+
+ propfind_xml = _build_propfind_xml(
+ [
+ "/remote.php/dav/files/user/inbox/",
+ "https://nc.example.com/remote.php/dav/files/user/inbox/doc.pdf",
+ ]
+ )
+
+ mock_propfind_resp = MagicMock()
+ mock_propfind_resp.text = propfind_xml
+
+ mock_dl_resp = MagicMock()
+ mock_dl_resp.content = b"%PDF-1.4"
+
+ with (
+ patch("requests.request", return_value=mock_propfind_resp),
+ patch("requests.get", return_value=mock_dl_resp) as mock_get,
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._mark_processed"),
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_nextcloud_folder(
+ cfg={"url": "https://nc.example.com/remote.php/dav/files/user", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 1
+ mock_enqueue.assert_called_once()
+ download_url = mock_get.call_args[0][0]
+ assert download_url.startswith("https://")
+
+
+@pytest.mark.unit
+class TestScanUserWebdavFolder:
+ """Tests for _scan_user_webdav_folder per-user WebDAV scanning."""
+
+ def test_returns_zero_when_url_not_configured(self):
+ """Missing URL should return 0."""
+ from app.tasks.watch_folder_tasks import _scan_user_webdav_folder
+
+ result = _scan_user_webdav_folder(
+ cfg={"url": "", "folder_path": "/inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_propfind_fails(self):
+ """PROPFIND failure should return 0."""
+ from app.tasks.watch_folder_tasks import _scan_user_webdav_folder
+
+ with patch("requests.request") as mock_request:
+ mock_request.side_effect = Exception("Connection refused")
+
+ result = _scan_user_webdav_folder(
+ cfg={"url": "https://dav.example.com", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_returns_zero_when_xml_parse_fails(self):
+ """Invalid XML response should return 0."""
+ from app.tasks.watch_folder_tasks import _scan_user_webdav_folder
+
+ mock_resp = MagicMock()
+ mock_resp.text = "not valid xml <<<"
+
+ with patch("requests.request", return_value=mock_resp):
+ result = _scan_user_webdav_folder(
+ cfg={"url": "https://dav.example.com", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+ assert result == 0
+
+ def test_downloads_and_enqueues_new_file(self, tmp_path):
+ """Successful WebDAV download should enqueue file and return 1."""
+ from app.tasks.watch_folder_tasks import _scan_user_webdav_folder
+
+ propfind_xml = _build_propfind_xml(
+ [
+ "/inbox/",
+ "/inbox/doc.pdf",
+ ]
+ )
+
+ mock_propfind_resp = MagicMock()
+ mock_propfind_resp.text = propfind_xml
+
+ mock_dl_resp = MagicMock()
+ mock_dl_resp.content = b"%PDF-1.4"
+
+ with (
+ patch("requests.request", return_value=mock_propfind_resp),
+ patch("requests.get", return_value=mock_dl_resp),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._mark_processed") as mock_mark,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_webdav_folder(
+ cfg={"url": "https://dav.example.com", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 1
+ mock_enqueue.assert_called_once()
+ mock_mark.assert_called_once()
+
+ def test_skips_directory_entries(self, tmp_path):
+ """Entries ending with / should be skipped (directories)."""
+ from app.tasks.watch_folder_tasks import _scan_user_webdav_folder
+
+ propfind_xml = _build_propfind_xml(
+ [
+ "/inbox/",
+ "/inbox/subdir/",
+ ]
+ )
+
+ mock_propfind_resp = MagicMock()
+ mock_propfind_resp.text = propfind_xml
+
+ with (
+ patch("requests.request", return_value=mock_propfind_resp),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_webdav_folder(
+ cfg={"url": "https://dav.example.com", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_skips_cached_file(self, tmp_path):
+ """Files already in cache should be skipped."""
+ from app.tasks.watch_folder_tasks import _scan_user_webdav_folder
+
+ propfind_xml = _build_propfind_xml(
+ [
+ "/inbox/",
+ "/inbox/doc.pdf",
+ ]
+ )
+
+ mock_propfind_resp = MagicMock()
+ mock_propfind_resp.text = propfind_xml
+
+ with (
+ patch("requests.request", return_value=mock_propfind_resp),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_webdav_folder(
+ cfg={"url": "https://dav.example.com", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={"webdav:/inbox/doc.pdf": "done"},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_deletes_after_download(self, tmp_path):
+ """delete_after=True should send DELETE request."""
+ from app.tasks.watch_folder_tasks import _scan_user_webdav_folder
+
+ propfind_xml = _build_propfind_xml(
+ [
+ "/inbox/",
+ "/inbox/doc.pdf",
+ ]
+ )
+
+ mock_propfind_resp = MagicMock()
+ mock_propfind_resp.text = propfind_xml
+
+ mock_dl_resp = MagicMock()
+ mock_dl_resp.content = b"%PDF-1.4"
+
+ mock_del_resp = MagicMock()
+
+ with (
+ patch("requests.request", side_effect=[mock_propfind_resp, mock_del_resp]) as mock_request,
+ patch("requests.get", return_value=mock_dl_resp),
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file"),
+ patch("app.tasks.watch_folder_tasks._mark_processed"),
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_webdav_folder(
+ cfg={"url": "https://dav.example.com", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=True,
+ owner_id="u1",
+ )
+
+ assert count == 1
+ assert mock_request.call_count == 2
+ delete_call = mock_request.call_args_list[1]
+ assert delete_call[0][0] == "DELETE"
+
+ def test_handles_download_failure(self, tmp_path):
+ """Download failure should skip the file and continue."""
+ from app.tasks.watch_folder_tasks import _scan_user_webdav_folder
+
+ propfind_xml = _build_propfind_xml(
+ [
+ "/inbox/",
+ "/inbox/doc.pdf",
+ ]
+ )
+
+ mock_propfind_resp = MagicMock()
+ mock_propfind_resp.text = propfind_xml
+
+ with (
+ patch("requests.request", return_value=mock_propfind_resp),
+ patch("requests.get") as mock_get,
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+ mock_get.side_effect = Exception("Download failed")
+
+ count = _scan_user_webdav_folder(
+ cfg={"url": "https://dav.example.com", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 0
+ mock_enqueue.assert_not_called()
+
+ def test_handles_absolute_href_url(self, tmp_path):
+ """Absolute href URLs should be used directly for download."""
+ from app.tasks.watch_folder_tasks import _scan_user_webdav_folder
+
+ propfind_xml = _build_propfind_xml(
+ [
+ "/inbox/",
+ "https://dav.example.com/inbox/doc.pdf",
+ ]
+ )
+
+ mock_propfind_resp = MagicMock()
+ mock_propfind_resp.text = propfind_xml
+
+ mock_dl_resp = MagicMock()
+ mock_dl_resp.content = b"%PDF-1.4"
+
+ with (
+ patch("requests.request", return_value=mock_propfind_resp),
+ patch("requests.get", return_value=mock_dl_resp) as mock_get,
+ patch("app.tasks.watch_folder_tasks.settings") as mock_settings,
+ patch("app.tasks.watch_folder_tasks._enqueue_file") as mock_enqueue,
+ patch("app.tasks.watch_folder_tasks._mark_processed"),
+ patch("app.tasks.watch_folder_tasks._is_allowed_file", return_value=True),
+ ):
+ mock_settings.workdir = str(tmp_path)
+ mock_settings.http_request_timeout = 30
+
+ count = _scan_user_webdav_folder(
+ cfg={"url": "https://dav.example.com", "folder_path": "inbox"},
+ creds={"username": "u", "password": "p"},
+ cache={},
+ delete_after=False,
+ owner_id="u1",
+ )
+
+ assert count == 1
+ mock_enqueue.assert_called_once()
+ download_url = mock_get.call_args[0][0]
+ assert download_url == "https://dav.example.com/inbox/doc.pdf"
+
@pytest.mark.unit
class TestScanAllWatchFoldersIncludesUserIntegrations: