feat(tasks): refactor IMAP and watch folder polling to support multi-tenant user attribution
- Add owner_id parameter to pull_inbox() and fetch_attachments_and_enqueue() to attribute ingested documents to the correct user - Add _pull_user_integration_imap() to poll IMAP sources from UserIntegration model - Add _pull_user_integration_watch_folders() to scan watch folders from UserIntegration model - Add _is_safe_watch_path() for path traversal security on user-configured paths - Add _scan_user_watch_folder() that passes owner_id to _enqueue_file() - Update _enqueue_file() to forward owner_id to process_document/convert_to_pdf - Update celery beat schedule to always enable IMAP and watch folder polling (user integrations can exist without system-level config) - Ensure individual connection failures don't crash the polling loop - Update existing tests for new function signatures Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
+15
-29
@@ -63,15 +63,13 @@ def test_task():
|
||||
check_credentials.apply_async(countdown=10) # Run 10 seconds after worker starts
|
||||
|
||||
celery.conf.beat_schedule = {
|
||||
"poll-inboxes-every-minute": (
|
||||
{
|
||||
"task": "app.tasks.imap_tasks.pull_all_inboxes",
|
||||
"schedule": crontab(minute="*/1"), # every 1 minute
|
||||
"options": {"expires": 55}, # Ensure tasks don't pile up
|
||||
}
|
||||
if (settings.imap1_host or settings.imap2_host)
|
||||
else None
|
||||
),
|
||||
# IMAP polling — always enabled because per-user IMAP integrations may
|
||||
# exist in the database even when no system-level IMAP hosts are configured.
|
||||
"poll-inboxes-every-minute": {
|
||||
"task": "app.tasks.imap_tasks.pull_all_inboxes",
|
||||
"schedule": crontab(minute="*/1"), # every 1 minute
|
||||
"options": {"expires": 55}, # Ensure tasks don't pile up
|
||||
},
|
||||
# Add Uptime Kuma ping task if configured
|
||||
"ping-uptime-kuma": (
|
||||
{
|
||||
@@ -100,27 +98,15 @@ celery.conf.beat_schedule = {
|
||||
"schedule": crontab(minute="*/1"), # Every minute
|
||||
"options": {"expires": 55}, # Must complete within 55 seconds
|
||||
},
|
||||
# Watch folder scanning — polls local paths, FTP, SFTP, and cloud ingest folders.
|
||||
# Watch folder scanning — always enabled because per-user WATCH_FOLDER
|
||||
# integrations may exist in the database even when no system-level watch
|
||||
# folder settings are configured.
|
||||
# Schedule is controlled by WATCH_FOLDER_POLL_INTERVAL (default: 1 minute).
|
||||
"scan-watch-folders": (
|
||||
{
|
||||
"task": "app.tasks.watch_folder_tasks.scan_all_watch_folders",
|
||||
"schedule": crontab(minute=f"*/{max(1, settings.watch_folder_poll_interval)}"),
|
||||
"options": {"expires": 55},
|
||||
}
|
||||
if (
|
||||
settings.watch_folders
|
||||
or settings.ftp_ingest_enabled
|
||||
or settings.sftp_ingest_enabled
|
||||
or settings.dropbox_ingest_enabled
|
||||
or settings.google_drive_ingest_enabled
|
||||
or settings.onedrive_ingest_enabled
|
||||
or settings.nextcloud_ingest_enabled
|
||||
or settings.s3_ingest_enabled
|
||||
or settings.webdav_ingest_enabled
|
||||
)
|
||||
else None
|
||||
),
|
||||
"scan-watch-folders": {
|
||||
"task": "app.tasks.watch_folder_tasks.scan_all_watch_folders",
|
||||
"schedule": crontab(minute=f"*/{max(1, settings.watch_folder_poll_interval)}"),
|
||||
"options": {"expires": 55},
|
||||
},
|
||||
# Backfill embeddings for files that were processed before the
|
||||
# embedding pipeline was enabled, or where the embedding task failed.
|
||||
"backfill-missing-embeddings": {
|
||||
|
||||
Reference in New Issue
Block a user