feat(tasks): refactor IMAP and watch folder polling to support multi-tenant user attribution

- Add owner_id parameter to pull_inbox() and fetch_attachments_and_enqueue()
  to attribute ingested documents to the correct user
- Add _pull_user_integration_imap() to poll IMAP sources from UserIntegration model
- Add _pull_user_integration_watch_folders() to scan watch folders from UserIntegration model
- Add _is_safe_watch_path() for path traversal security on user-configured paths
- Add _scan_user_watch_folder() that passes owner_id to _enqueue_file()
- Update _enqueue_file() to forward owner_id to process_document/convert_to_pdf
- Update celery beat schedule to always enable IMAP and watch folder polling
  (user integrations can exist without system-level config)
- Ensure individual connection failures don't crash the polling loop
- Update existing tests for new function signatures

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-08 18:38:22 +00:00
parent 322aef98ae
commit 019807d0f5
5 changed files with 366 additions and 44 deletions
+15 -29
View File
@@ -63,15 +63,13 @@ def test_task():
check_credentials.apply_async(countdown=10) # Run 10 seconds after worker starts
celery.conf.beat_schedule = {
"poll-inboxes-every-minute": (
{
"task": "app.tasks.imap_tasks.pull_all_inboxes",
"schedule": crontab(minute="*/1"), # every 1 minute
"options": {"expires": 55}, # Ensure tasks don't pile up
}
if (settings.imap1_host or settings.imap2_host)
else None
),
# IMAP polling — always enabled because per-user IMAP integrations may
# exist in the database even when no system-level IMAP hosts are configured.
"poll-inboxes-every-minute": {
"task": "app.tasks.imap_tasks.pull_all_inboxes",
"schedule": crontab(minute="*/1"), # every 1 minute
"options": {"expires": 55}, # Ensure tasks don't pile up
},
# Add Uptime Kuma ping task if configured
"ping-uptime-kuma": (
{
@@ -100,27 +98,15 @@ celery.conf.beat_schedule = {
"schedule": crontab(minute="*/1"), # Every minute
"options": {"expires": 55}, # Must complete within 55 seconds
},
# Watch folder scanning — polls local paths, FTP, SFTP, and cloud ingest folders.
# Watch folder scanning — always enabled because per-user WATCH_FOLDER
# integrations may exist in the database even when no system-level watch
# folder settings are configured.
# Schedule is controlled by WATCH_FOLDER_POLL_INTERVAL (default: 1 minute).
"scan-watch-folders": (
{
"task": "app.tasks.watch_folder_tasks.scan_all_watch_folders",
"schedule": crontab(minute=f"*/{max(1, settings.watch_folder_poll_interval)}"),
"options": {"expires": 55},
}
if (
settings.watch_folders
or settings.ftp_ingest_enabled
or settings.sftp_ingest_enabled
or settings.dropbox_ingest_enabled
or settings.google_drive_ingest_enabled
or settings.onedrive_ingest_enabled
or settings.nextcloud_ingest_enabled
or settings.s3_ingest_enabled
or settings.webdav_ingest_enabled
)
else None
),
"scan-watch-folders": {
"task": "app.tasks.watch_folder_tasks.scan_all_watch_folders",
"schedule": crontab(minute=f"*/{max(1, settings.watch_folder_poll_interval)}"),
"options": {"expires": 55},
},
# Backfill embeddings for files that were processed before the
# embedding pipeline was enabled, or where the embedding task failed.
"backfill-missing-embeddings": {