Merge pull request #504 from christianlouis/copilot/add-watch-folder-support
This commit is contained in:
Christian Krakau-Louis
@@ -41,6 +41,7 @@ from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401
|
||||
from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401
|
||||
from app.tasks.upload_with_rclone import send_to_all_rclone_destinations, upload_with_rclone # noqa: F401
|
||||
from app.tasks.uptime_kuma_tasks import ping_uptime_kuma # noqa: F401
|
||||
from app.tasks.watch_folder_tasks import scan_all_watch_folders # noqa: F401
|
||||
from app.tasks.webhook_tasks import deliver_webhook_task # noqa: F401
|
||||
|
||||
# Register the settings reload signal handler so workers pick up config changes
|
||||
@@ -99,6 +100,27 @@ celery.conf.beat_schedule = {
|
||||
"schedule": crontab(minute="*/1"), # Every minute
|
||||
"options": {"expires": 55}, # Must complete within 55 seconds
|
||||
},
|
||||
# Watch folder scanning — polls local paths, FTP, SFTP, and cloud ingest folders.
|
||||
# Schedule is controlled by WATCH_FOLDER_POLL_INTERVAL (default: 1 minute).
|
||||
"scan-watch-folders": (
|
||||
{
|
||||
"task": "app.tasks.watch_folder_tasks.scan_all_watch_folders",
|
||||
"schedule": crontab(minute=f"*/{max(1, settings.watch_folder_poll_interval)}"),
|
||||
"options": {"expires": 55},
|
||||
}
|
||||
if (
|
||||
settings.watch_folders
|
||||
or settings.ftp_ingest_enabled
|
||||
or settings.sftp_ingest_enabled
|
||||
or settings.dropbox_ingest_enabled
|
||||
or settings.google_drive_ingest_enabled
|
||||
or settings.onedrive_ingest_enabled
|
||||
or settings.nextcloud_ingest_enabled
|
||||
or settings.s3_ingest_enabled
|
||||
or settings.webdav_ingest_enabled
|
||||
)
|
||||
else None
|
||||
),
|
||||
# Backfill embeddings for files that were processed before the
|
||||
# embedding pipeline was enabled, or where the embedding task failed.
|
||||
"backfill-missing-embeddings": {
|
||||
|
||||
+185
@@ -190,6 +190,191 @@ class Settings(BaseSettings):
|
||||
stripe_success_url: Optional[str] = None # e.g. https://app.example.com/billing/success
|
||||
stripe_cancel_url: Optional[str] = None # e.g. https://app.example.com/pricing
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Watch Folder Ingestion
|
||||
# ---------------------------------------------------------------------------
|
||||
# Local filesystem watch folders (comma-separated list of absolute paths).
|
||||
# DocuElevate will poll each path for new files and automatically ingest them.
|
||||
# Works with any mounted path, including SMB/CIFS (via system mount), NFS, etc.
|
||||
# Example: /watchfolders/scanner,/mnt/shared/inbox
|
||||
watch_folders: Optional[str] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Comma-separated list of local filesystem paths (absolute) that DocuElevate will "
|
||||
"poll for new files to ingest. Each file found is enqueued for document processing. "
|
||||
"Works with any mounted path including SMB/CIFS (mounted via system) and NFS. "
|
||||
"Example: /watchfolders/scanner,/mnt/shared/inbox"
|
||||
),
|
||||
)
|
||||
watch_folder_poll_interval: int = Field(
|
||||
default=1,
|
||||
description=("Poll interval in minutes for local watch folder scanning. Default: 1 minute."),
|
||||
)
|
||||
watch_folder_delete_after_process: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"Delete files from local watch folders after they have been successfully enqueued "
|
||||
"for processing. When False (default), files are left in place and tracked via a "
|
||||
"cache file to avoid re-ingesting them."
|
||||
),
|
||||
)
|
||||
|
||||
# FTP Ingest / Watch Folder
|
||||
# Uses the existing FTP credentials (ftp_host, ftp_username, ftp_password) to poll
|
||||
# a source folder on the FTP server for new files to ingest.
|
||||
ftp_ingest_folder: Optional[str] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"FTP folder path to monitor for new files to ingest. "
|
||||
"Uses the existing FTP connection settings (FTP_HOST, FTP_USERNAME, FTP_PASSWORD). "
|
||||
"When set, DocuElevate will periodically poll this folder and download new files for processing."
|
||||
),
|
||||
)
|
||||
ftp_ingest_enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable FTP watch folder ingestion. Requires FTP_INGEST_FOLDER and FTP connection settings.",
|
||||
)
|
||||
ftp_ingest_delete_after_process: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"Delete files from the FTP ingest folder after they have been successfully downloaded "
|
||||
"and enqueued for processing. Default: False (files are left in place)."
|
||||
),
|
||||
)
|
||||
|
||||
# SFTP Ingest / Watch Folder
|
||||
# Uses the existing SFTP credentials (sftp_host, sftp_username, sftp_password/sftp_private_key)
|
||||
# to poll a source folder on the SFTP server for new files to ingest.
|
||||
sftp_ingest_folder: Optional[str] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"SFTP folder path to monitor for new files to ingest. "
|
||||
"Uses the existing SFTP connection settings (SFTP_HOST, SFTP_USERNAME, SFTP_PASSWORD/SFTP_PRIVATE_KEY). "
|
||||
"When set, DocuElevate will periodically poll this folder and download new files for processing."
|
||||
),
|
||||
)
|
||||
sftp_ingest_enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable SFTP watch folder ingestion. Requires SFTP_INGEST_FOLDER and SFTP connection settings.",
|
||||
)
|
||||
sftp_ingest_delete_after_process: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"Delete files from the SFTP ingest folder after they have been successfully downloaded "
|
||||
"and enqueued for processing. Default: False (files are left in place)."
|
||||
),
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cloud Provider Watch Folders
|
||||
# ---------------------------------------------------------------------------
|
||||
# Each cloud provider has three settings:
|
||||
# <provider>_ingest_enabled — enable the watch-folder for this provider
|
||||
# <provider>_ingest_folder — the remote path / folder ID to poll
|
||||
# <provider>_ingest_delete_after_process — delete from cloud after download
|
||||
|
||||
# Dropbox ingest — reuses existing Dropbox OAuth credentials
|
||||
dropbox_ingest_enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable Dropbox watch folder ingestion. Requires Dropbox OAuth credentials.",
|
||||
)
|
||||
dropbox_ingest_folder: Optional[str] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Dropbox folder path to poll for new files to ingest (e.g. /Inbox/Scanner). "
|
||||
"Uses the existing Dropbox OAuth credentials."
|
||||
),
|
||||
)
|
||||
dropbox_ingest_delete_after_process: bool = Field(
|
||||
default=False,
|
||||
description="Delete files from Dropbox ingest folder after download and enqueue.",
|
||||
)
|
||||
|
||||
# Google Drive ingest — reuses existing Google Drive credentials
|
||||
google_drive_ingest_enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable Google Drive watch folder ingestion. Requires Google Drive credentials.",
|
||||
)
|
||||
google_drive_ingest_folder_id: Optional[str] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Google Drive folder ID to poll for new files to ingest. "
|
||||
"Uses the existing Google Drive service-account or OAuth credentials."
|
||||
),
|
||||
)
|
||||
google_drive_ingest_delete_after_process: bool = Field(
|
||||
default=False,
|
||||
description="Delete files from Google Drive ingest folder after download and enqueue.",
|
||||
)
|
||||
|
||||
# OneDrive ingest — reuses existing OneDrive MSAL credentials
|
||||
onedrive_ingest_enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable OneDrive watch folder ingestion. Requires OneDrive MSAL credentials.",
|
||||
)
|
||||
onedrive_ingest_folder_path: Optional[str] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"OneDrive folder path to poll for new files to ingest (e.g. /Inbox/Scanner). "
|
||||
"Uses the existing OneDrive client credentials."
|
||||
),
|
||||
)
|
||||
onedrive_ingest_delete_after_process: bool = Field(
|
||||
default=False,
|
||||
description="Delete files from OneDrive ingest folder after download and enqueue.",
|
||||
)
|
||||
|
||||
# Nextcloud ingest — reuses existing Nextcloud WebDAV credentials
|
||||
nextcloud_ingest_enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable Nextcloud watch folder ingestion. Requires Nextcloud WebDAV credentials.",
|
||||
)
|
||||
nextcloud_ingest_folder: Optional[str] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Nextcloud folder path to poll for new files to ingest (e.g. /Scans/Inbox). "
|
||||
"Uses the existing Nextcloud upload URL and credentials."
|
||||
),
|
||||
)
|
||||
nextcloud_ingest_delete_after_process: bool = Field(
|
||||
default=False,
|
||||
description="Delete files from Nextcloud ingest folder after download and enqueue.",
|
||||
)
|
||||
|
||||
# S3 ingest — reuses existing AWS/S3 credentials
|
||||
s3_ingest_enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable Amazon S3 watch folder (prefix) ingestion. Requires S3 credentials.",
|
||||
)
|
||||
s3_ingest_prefix: Optional[str] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"S3 key prefix to poll for new objects to ingest (e.g. inbox/scanner/). "
|
||||
"Uses the existing S3 bucket and AWS credentials."
|
||||
),
|
||||
)
|
||||
s3_ingest_delete_after_process: bool = Field(
|
||||
default=False,
|
||||
description="Delete objects from S3 ingest prefix after download and enqueue.",
|
||||
)
|
||||
|
||||
# WebDAV ingest — reuses existing WebDAV credentials
|
||||
webdav_ingest_enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable WebDAV watch folder ingestion. Requires WebDAV URL and credentials.",
|
||||
)
|
||||
webdav_ingest_folder: Optional[str] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"WebDAV folder path to poll for new files to ingest (e.g. /remote.php/webdav/Inbox). "
|
||||
"Uses the existing WebDAV URL and credentials."
|
||||
),
|
||||
)
|
||||
webdav_ingest_delete_after_process: bool = Field(
|
||||
default=False,
|
||||
description="Delete files from WebDAV ingest folder after download and enqueue.",
|
||||
)
|
||||
|
||||
# IMAP 1
|
||||
imap1_host: Optional[str] = None
|
||||
imap1_port: Optional[int] = 993
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -106,6 +106,35 @@ def get_settings_for_display(show_values: bool = False) -> dict[str, list[dict[s
|
||||
"email_sender",
|
||||
"email_default_recipient",
|
||||
],
|
||||
"Watch Folders": [
|
||||
"watch_folders",
|
||||
"watch_folder_poll_interval",
|
||||
"watch_folder_delete_after_process",
|
||||
"ftp_ingest_enabled",
|
||||
"ftp_ingest_folder",
|
||||
"ftp_ingest_delete_after_process",
|
||||
"sftp_ingest_enabled",
|
||||
"sftp_ingest_folder",
|
||||
"sftp_ingest_delete_after_process",
|
||||
"dropbox_ingest_enabled",
|
||||
"dropbox_ingest_folder",
|
||||
"dropbox_ingest_delete_after_process",
|
||||
"google_drive_ingest_enabled",
|
||||
"google_drive_ingest_folder_id",
|
||||
"google_drive_ingest_delete_after_process",
|
||||
"onedrive_ingest_enabled",
|
||||
"onedrive_ingest_folder_path",
|
||||
"onedrive_ingest_delete_after_process",
|
||||
"nextcloud_ingest_enabled",
|
||||
"nextcloud_ingest_folder",
|
||||
"nextcloud_ingest_delete_after_process",
|
||||
"s3_ingest_enabled",
|
||||
"s3_ingest_prefix",
|
||||
"s3_ingest_delete_after_process",
|
||||
"webdav_ingest_enabled",
|
||||
"webdav_ingest_folder",
|
||||
"webdav_ingest_delete_after_process",
|
||||
],
|
||||
"IMAP": [
|
||||
"imap1_host",
|
||||
"imap1_port",
|
||||
|
||||
@@ -971,6 +971,244 @@ SETTING_METADATA = {
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Watch Folder / Ingest Settings
|
||||
"watch_folders": {
|
||||
"category": "Watch Folders",
|
||||
"description": (
|
||||
"Comma-separated list of absolute local filesystem paths that DocuElevate will "
|
||||
"poll for new files to ingest. Works with any mounted path (SMB/CIFS, NFS, etc.). "
|
||||
"Example: /watchfolders/scanner,/mnt/shared/inbox"
|
||||
),
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"watch_folder_poll_interval": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Poll interval in minutes for local watch folder scanning (default: 1)",
|
||||
"type": "integer",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"watch_folder_delete_after_process": {
|
||||
"category": "Watch Folders",
|
||||
"description": (
|
||||
"Delete files from local watch folders after they have been enqueued for processing. "
|
||||
"When False (default), processed files are tracked via cache to avoid re-ingestion."
|
||||
),
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# FTP Ingest
|
||||
"ftp_ingest_enabled": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Enable FTP watch folder ingestion (requires FTP_INGEST_FOLDER and FTP connection settings)",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"ftp_ingest_folder": {
|
||||
"category": "Watch Folders",
|
||||
"description": (
|
||||
"FTP folder path to poll for new documents to ingest. "
|
||||
"Uses the existing FTP connection settings (FTP_HOST, FTP_USERNAME, FTP_PASSWORD)."
|
||||
),
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"ftp_ingest_delete_after_process": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Delete files from the FTP ingest folder after they have been downloaded and enqueued",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# SFTP Ingest
|
||||
"sftp_ingest_enabled": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Enable SFTP watch folder ingestion (requires SFTP_INGEST_FOLDER and SFTP connection settings)",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sftp_ingest_folder": {
|
||||
"category": "Watch Folders",
|
||||
"description": (
|
||||
"SFTP folder path to poll for new documents to ingest. "
|
||||
"Uses the existing SFTP connection settings (SFTP_HOST, SFTP_USERNAME, SFTP_PASSWORD/SFTP_PRIVATE_KEY)."
|
||||
),
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"sftp_ingest_delete_after_process": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Delete files from the SFTP ingest folder after they have been downloaded and enqueued",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Cloud Provider Watch Folders — Dropbox
|
||||
"dropbox_ingest_enabled": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Enable Dropbox watch folder ingestion (requires Dropbox OAuth credentials)",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"dropbox_ingest_folder": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Dropbox folder path to poll for new documents (e.g. /Inbox/Scanner). Uses existing Dropbox credentials.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"dropbox_ingest_delete_after_process": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Delete files from Dropbox ingest folder after download and enqueue",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Cloud Provider Watch Folders — Google Drive
|
||||
"google_drive_ingest_enabled": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Enable Google Drive watch folder ingestion (requires Google Drive credentials)",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"google_drive_ingest_folder_id": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Google Drive folder ID to poll for new documents. Uses existing Google Drive credentials.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"google_drive_ingest_delete_after_process": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Delete files from Google Drive ingest folder after download and enqueue",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Cloud Provider Watch Folders — OneDrive
|
||||
"onedrive_ingest_enabled": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Enable OneDrive watch folder ingestion (requires OneDrive MSAL credentials)",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"onedrive_ingest_folder_path": {
|
||||
"category": "Watch Folders",
|
||||
"description": "OneDrive folder path to poll for new documents (e.g. /Inbox/Scanner). Uses existing OneDrive credentials.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"onedrive_ingest_delete_after_process": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Delete files from OneDrive ingest folder after download and enqueue",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Cloud Provider Watch Folders — Nextcloud
|
||||
"nextcloud_ingest_enabled": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Enable Nextcloud watch folder ingestion (requires Nextcloud WebDAV credentials)",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"nextcloud_ingest_folder": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Nextcloud folder path to poll for new documents (e.g. /Scans/Inbox). Uses existing Nextcloud credentials.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"nextcloud_ingest_delete_after_process": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Delete files from Nextcloud ingest folder after download and enqueue",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Cloud Provider Watch Folders — S3
|
||||
"s3_ingest_enabled": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Enable Amazon S3 prefix (watch folder) ingestion (requires S3/AWS credentials)",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"s3_ingest_prefix": {
|
||||
"category": "Watch Folders",
|
||||
"description": "S3 key prefix to poll for new objects to ingest (e.g. inbox/scanner/). Uses existing S3 credentials.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"s3_ingest_delete_after_process": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Delete objects from S3 ingest prefix after download and enqueue",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# Cloud Provider Watch Folders — WebDAV
|
||||
"webdav_ingest_enabled": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Enable WebDAV watch folder ingestion (requires WebDAV URL and credentials)",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"webdav_ingest_folder": {
|
||||
"category": "Watch Folders",
|
||||
"description": "WebDAV folder path to poll for new documents. Uses existing WebDAV URL and credentials.",
|
||||
"type": "string",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
"webdav_ingest_delete_after_process": {
|
||||
"category": "Watch Folders",
|
||||
"description": "Delete files from WebDAV ingest folder after download and enqueue",
|
||||
"type": "boolean",
|
||||
"sensitive": False,
|
||||
"required": False,
|
||||
"restart_required": False,
|
||||
},
|
||||
# IMAP Settings - Account 1
|
||||
"imap1_host": {
|
||||
"category": "IMAP",
|
||||
|
||||
Reference in New Issue
Block a user