feat: implement Google Drive integration and enhance documentation for new features

app/api.py

@@ -1,12 +1,20 @@
 # app/api.py
-from fastapi import APIRouter, Request, HTTPException, status, Depends
+from fastapi import APIRouter, Request, HTTPException, status, Depends, UploadFile, File
 from hashlib import md5
 from sqlalchemy.orm import Session
 from typing import List
+import os
 
-from app.auth import require_login
+from app.auth import require_login, get_current_user
 from app.database import SessionLocal
 from app.models import FileRecord
+from app.config import settings
+from app.tasks.process_document import process_document
+from app.tasks.upload_to_dropbox import upload_to_dropbox
+from app.tasks.upload_to_paperless import upload_to_paperless
+from app.tasks.upload_to_nextcloud import upload_to_nextcloud
+from app.tasks.upload_to_google_drive import upload_to_google_drive
+from app.tasks.send_to_all import send_to_all_destinations
 
 router = APIRouter()
 
@@ -74,3 +82,171 @@ def list_files_api(request: Request, db: Session = Depends(get_db)):
             "created_at": f.created_at.isoformat() if f.created_at else None
         })
     return result
+
+# API endpoints
+@router.get("/diagnostic/settings")
+@require_login
+async def diagnostic_settings(current_user: dict = Depends(get_current_user)):
+    """
+    API endpoint to dump settings to the log and view basic config information
+    This endpoint doesn't expose sensitive information like passwords or tokens
+    """
+    from app.utils.config_validator import dump_all_settings
+    # Dump full settings to log for admin to see
+    dump_all_settings()
+    
+    # Return safe subset of settings for API response
+    safe_settings = {
+        "workdir": settings.workdir,
+        "external_hostname": settings.external_hostname,
+        "configured_services": {
+            "email": bool(getattr(settings, 'email_host', None)),
+            "s3": bool(getattr(settings, 's3_bucket_name', None)),
+            "dropbox": bool(getattr(settings, 'dropbox_refresh_token', None)),
+            "nextcloud": bool(getattr(settings, 'nextcloud_upload_url', None)),
+            "sftp": bool(getattr(settings, 'sftp_host', None)),
+            "paperless": bool(getattr(settings, 'paperless_host', None)),
+            "google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)),
+        },
+        "imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)),
+    }
+    
+    return {
+        "status": "success",
+        "settings": safe_settings,
+        "message": "Full settings have been dumped to application logs"
+    }
+
+@router.post("/process/")
+@require_login
+def process(file_path: str):
+    """API Endpoint to start document processing."""
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, file_path)
+
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+
+    task = process_document.delay(file_path)
+    return {"task_id": task.id, "status": "queued"}
+
+@router.post("/send_to_dropbox/")
+@require_login
+def send_to_dropbox(file_path: str):
+    """Send a document to Dropbox."""
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+    task = upload_to_dropbox.delay(file_path)
+    return {"task_id": task.id, "status": "queued"}
+
+@router.post("/send_to_paperless/")
+@require_login
+def send_to_paperless(file_path: str):
+    """Send a document to Paperless-ngx."""
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+    task = upload_to_paperless.delay(file_path)
+    return {"task_id": task.id, "status": "queued"}
+
+@router.post("/send_to_nextcloud/")
+@require_login
+def send_to_nextcloud(file_path: str):
+    """Send a document to NextCloud."""
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+    task = upload_to_nextcloud.delay(file_path)
+    return {"task_id": task.id, "status": "queued"}
+
+@router.post("/send_to_google_drive/")
+@require_login
+def send_to_google_drive(file_path: str):
+    """Send a document to Google Drive."""
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+    task = upload_to_google_drive.delay(file_path)
+    return {"task_id": task.id, "status": "queued"}
+
+@router.post("/send_to_all_destinations/")
+@require_login
+def send_to_all_destinations_endpoint(file_path: str):
+    """Call the aggregator task that sends this file to all configured destinations."""
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+
+    task = send_to_all_destinations.delay(file_path)
+    return {"task_id": task.id, "status": "queued", "file_path": file_path}
+
+@router.post("/processall")
+@require_login
+def process_all_pdfs_in_workdir():
+    """Finds all .pdf files in <workdir> and enqueues them for processing."""
+    target_dir = settings.workdir
+    if not os.path.exists(target_dir):
+        raise HTTPException(
+            status_code=400, detail=f"Directory {target_dir} does not exist."
+        )
+
+    pdf_files = []
+    for filename in os.listdir(target_dir):
+        if filename.lower().endswith(".pdf"):
+            pdf_files.append(filename)
+
+    if not pdf_files:
+        return {"message": "No PDF files found in that directory."}
+
+    task_ids = []
+    for pdf in pdf_files:
+        file_path = os.path.join(target_dir, pdf)
+        task = process_document.delay(file_path)
+        task_ids.append(task.id)
+
+    return {
+        "message": f"Enqueued {len(pdf_files)} PDFs to upload_to_s3",
+        "pdf_files": pdf_files,
+        "task_ids": task_ids
+    }
+
+@router.post("/ui-upload")
+@require_login
+async def ui_upload(file: UploadFile = File(...)):
+    """Endpoint to accept a user-uploaded file and enqueue it for processing."""
+    workdir = "/workdir"
+    target_path = os.path.join(workdir, file.filename)
+    try:
+        with open(target_path, "wb") as f:
+            content = await file.read()
+            f.write(content)
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to save file: {e}"
+        )
+
+    task = process_document.delay(target_path)
+    return {"task_id": task.id, "status": "queued"}
+
+# Note: The api/router.py is now a submodule organization, 
+# but we're keeping this file for compatibility until we've fully migrated

app/api/router.py

@@ -0,0 +1,179 @@
+from fastapi import APIRouter, HTTPException, UploadFile, File, Depends
+import os
+
+from app.config import settings
+from app.auth import get_current_user, require_login
+from app.tasks.process_document import process_document
+from app.tasks.upload_to_dropbox import upload_to_dropbox
+from app.tasks.upload_to_paperless import upload_to_paperless
+from app.tasks.upload_to_nextcloud import upload_to_nextcloud
+from app.tasks.upload_to_google_drive import upload_to_google_drive
+from app.tasks.send_to_all import send_to_all_destinations
+
+router = APIRouter()
+
+# Diagnostic endpoints
+@router.get("/diagnostic/settings")
+@require_login
+async def diagnostic_settings(current_user: dict = Depends(get_current_user)):
+    """
+    API endpoint to dump settings to the log and view basic config information
+    This endpoint doesn't expose sensitive information like passwords or tokens
+    """
+    from app.utils.config_validator import dump_all_settings
+    # Dump full settings to log for admin to see
+    dump_all_settings()
+    
+    # Return safe subset of settings for API response
+    safe_settings = {
+        "workdir": settings.workdir,
+        "external_hostname": settings.external_hostname,
+        "configured_services": {
+            "email": bool(getattr(settings, 'email_host', None)),
+            "s3": bool(getattr(settings, 's3_bucket_name', None)),
+            "dropbox": bool(getattr(settings, 'dropbox_refresh_token', None)),
+            "nextcloud": bool(getattr(settings, 'nextcloud_upload_url', None)),
+            "sftp": bool(getattr(settings, 'sftp_host', None)),
+            "paperless": bool(getattr(settings, 'paperless_host', None)),
+            "google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)),
+        },
+        "imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)),
+    }
+    
+    return {
+        "status": "success",
+        "settings": safe_settings,
+        "message": "Full settings have been dumped to application logs"
+    }
+
+# File processing endpoints
+@router.post("/process/")
+@require_login
+def process(file_path: str):
+    """API Endpoint to start document processing."""
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, file_path)
+
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+
+    task = process_document.delay(file_path)
+    return {"task_id": task.id, "status": "queued"}
+
+@router.post("/send_to_dropbox/")
+@require_login
+def send_to_dropbox(file_path: str):
+    """Send a document to Dropbox."""
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+    task = upload_to_dropbox.delay(file_path)
+    return {"task_id": task.id, "status": "queued"}
+
+@router.post("/send_to_paperless/")
+@require_login
+def send_to_paperless(file_path: str):
+    """Send a document to Paperless-ngx."""
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+    task = upload_to_paperless.delay(file_path)
+    return {"task_id": task.id, "status": "queued"}
+
+@router.post("/send_to_nextcloud/")
+@require_login
+def send_to_nextcloud(file_path: str):
+    """Send a document to NextCloud."""
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+    task = upload_to_nextcloud.delay(file_path)
+    return {"task_id": task.id, "status": "queued"}
+
+@router.post("/send_to_google_drive/")
+@require_login
+def send_to_google_drive(file_path: str):
+    """Send a document to Google Drive."""
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+    task = upload_to_google_drive.delay(file_path)
+    return {"task_id": task.id, "status": "queued"}
+
+@router.post("/send_to_all_destinations/")
+@require_login
+def send_to_all_destinations_endpoint(file_path: str):
+    """Call the aggregator task that sends this file to all configured destinations."""
+    if not os.path.isabs(file_path):
+        file_path = os.path.join(settings.workdir, 'processed', file_path)
+
+    if not os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400, detail=f"File {file_path} not found."
+        )
+
+    task = send_to_all_destinations.delay(file_path)
+    return {"task_id": task.id, "status": "queued", "file_path": file_path}
+
+@router.post("/processall")
+@require_login
+def process_all_pdfs_in_workdir():
+    """Finds all .pdf files in <workdir> and enqueues them for processing."""
+    target_dir = settings.workdir
+    if not os.path.exists(target_dir):
+        raise HTTPException(
+            status_code=400, detail=f"Directory {target_dir} does not exist."
+        )
+
+    pdf_files = []
+    for filename in os.listdir(target_dir):
+        if filename.lower().endswith(".pdf"):
+            pdf_files.append(filename)
+
+    if not pdf_files:
+        return {"message": "No PDF files found in that directory."}
+
+    task_ids = []
+    for pdf in pdf_files:
+        file_path = os.path.join(target_dir, pdf)
+        task = process_document.delay(file_path)
+        task_ids.append(task.id)
+
+    return {
+        "message": f"Enqueued {len(pdf_files)} PDFs to upload_to_s3",
+        "pdf_files": pdf_files,
+        "task_ids": task_ids
+    }
+
+@router.post("/ui-upload")
+@require_login
+async def ui_upload(file: UploadFile = File(...)):
+    """Endpoint to accept a user-uploaded file and enqueue it for processing."""
+    workdir = "/workdir"
+    target_path = os.path.join(workdir, file.filename)
+    try:
+        with open(target_path, "wb") as f:
+            content = await file.read()
+            f.write(content)
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to save file: {e}"
+        )
+
+    task = process_document.delay(target_path)
+    return {"task_id": task.id, "status": "queued"}

app/celery_worker.py

@@ -20,6 +20,14 @@ from app.tasks.convert_to_pdf import convert_to_pdf
 from app.tasks.upload_to_dropbox import upload_to_dropbox
 from app.tasks.upload_to_paperless import upload_to_paperless
 from app.tasks.upload_to_nextcloud import upload_to_nextcloud
+from app.tasks.upload_to_google_drive import upload_to_google_drive
+from app.tasks.upload_to_webdav import upload_to_webdav
+from app.tasks.upload_to_s3 import upload_to_s3
+from app.tasks.upload_to_onedrive import upload_to_onedrive
+from app.tasks.upload_to_ftp import upload_to_ftp
+from app.tasks.upload_to_sftp import upload_to_sftp 
+from app.tasks.upload_to_email import upload_to_email
+
 from app.tasks.imap_tasks import pull_all_inboxes
 from app.tasks.send_to_all import send_to_all_destinations
 

app/config.py

@@ -4,34 +4,35 @@ from pydantic_settings import BaseSettings
 from typing import Optional
 
 class Settings(BaseSettings):
-    admin_username: str
-    admin_password: str
-    aws_access_key_id: str
-    aws_secret_access_key: str
-    aws_region: str
     database_url: str
     redis_url: str
-    s3_bucket_name: str
     openai_api_key: str
     openai_base_url: str = "https://api.openai.com/v1"  # Default to OpenAI's endpoint
     openai_model: str = "gpt-4o-mini"  # Default model
     workdir: str
-    dropbox_app_key: str
-    dropbox_app_secret: str
-    dropbox_folder: str
-    dropbox_refresh_token: str
-    nextcloud_upload_url: str
-    nextcloud_username: str
-    nextcloud_password: str
-    nextcloud_folder: str
-    paperless_ngx_api_token: str
-    paperless_host: str
+    
+    # Making Dropbox optional
+    dropbox_app_key: Optional[str] = None
+    dropbox_app_secret: Optional[str] = None
+    dropbox_folder: Optional[str] = None
+    dropbox_refresh_token: Optional[str] = None
+    
+    # Making Nextcloud optional
+    nextcloud_upload_url: Optional[str] = None
+    nextcloud_username: Optional[str] = None
+    nextcloud_password: Optional[str] = None
+    nextcloud_folder: Optional[str] = None
+    
+    # Making Paperless optional
+    paperless_ngx_api_token: Optional[str] = None
+    paperless_host: Optional[str] = None
+    
     azure_ai_key: str
     azure_region: str
     azure_endpoint: str
     gotenberg_url: str
     external_hostname: str = "localhost"  # Default to localhost
-    
+
     # Authentik
     authentik_client_id: Optional[str] = None
     authentik_client_secret: Optional[str] = None
@@ -56,6 +57,59 @@ class Settings(BaseSettings):
     imap2_poll_interval_minutes: int = 10
     imap2_delete_after_process: bool = False
 
+    # Google Drive settings
+    google_drive_credentials_json: str = ""
+    google_drive_folder_id: str = ""
+    google_drive_delegate_to: str = ""  # Optional delegated user email
+
+    # WebDAV settings
+    webdav_url: Optional[str] = None
+    webdav_username: Optional[str] = None
+    webdav_password: Optional[str] = None
+    webdav_folder: Optional[str] = None
+    webdav_verify_ssl: bool = True
+
+    # FTP settings
+    ftp_host: Optional[str] = None
+    ftp_port: Optional[int] = 21
+    ftp_username: Optional[str] = None
+    ftp_password: Optional[str] = None
+    ftp_folder: Optional[str] = None
+
+    # SFTP settings
+    sftp_host: Optional[str] = None
+    sftp_port: Optional[int] = 22
+    sftp_username: Optional[str] = None
+    sftp_password: Optional[str] = None
+    sftp_folder: Optional[str] = None
+    sftp_private_key: Optional[str] = None
+    sftp_private_key_passphrase: Optional[str] = None
+
+    # Email settings
+    email_host: Optional[str] = None
+    email_port: Optional[int] = 587
+    email_username: Optional[str] = None
+    email_password: Optional[str] = None
+    email_use_tls: bool = True
+    email_sender: Optional[str] = None  # From address, defaults to email_username if not set
+    email_default_recipient: Optional[str] = None
+
+    # OneDrive settings
+    onedrive_client_id: Optional[str] = None
+    onedrive_client_secret: Optional[str] = None
+    onedrive_tenant_id: Optional[str] = "common"  # Default to "common" for personal accounts
+    onedrive_refresh_token: Optional[str] = None  # Required for personal accounts
+    onedrive_folder_path: Optional[str] = None
+
+    # AWS S3 settings
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[str] = None
+    aws_region: Optional[str] = "us-east-1"  # Default region
+    s3_bucket_name: Optional[str] = None
+    s3_folder_prefix: Optional[str] = ""  # Optional folder prefix (e.g. "uploads/")
+    s3_storage_class: Optional[str] = "STANDARD"  # Default storage class
+    s3_acl: Optional[str] = "private"  # Default ACL
+
     class Config:
         env_file = ".env"
 

app/frontend.py

@@ -1,15 +1,17 @@
 # app/frontend.py
-import os
-from pathlib import Path
-from fastapi import APIRouter, Request, Depends
+from fastapi import APIRouter, Request, Depends, HTTPException
 from fastapi.templating import Jinja2Templates
+from pathlib import Path
 from sqlalchemy.orm import Session
+from datetime import datetime
 
 from app.auth import require_login
 from app.database import SessionLocal
+from app.config import settings
 
 router = APIRouter()
 
+# Set up Jinja2 templates
 templates_dir = Path(__file__).parent.parent / "frontend" / "templates"
 templates = Jinja2Templates(directory=str(templates_dir))
 
@@ -29,24 +31,69 @@ def files_page(request: Request):
     """
     return templates.TemplateResponse("files.html", {"request": request})
 
-# ... existing routes for /, /upload, /about, etc. ...
-
-
 @router.get("/", include_in_schema=False)
 async def serve_index(request: Request):
-     return templates.TemplateResponse("index.html", {"request": request})
- 
+    return templates.TemplateResponse("index.html", {"request": request})
+
 @router.get("/about", include_in_schema=False)
 async def serve_about(request: Request):
-     return templates.TemplateResponse("about.html", {"request": request})
- 
+    return templates.TemplateResponse("about.html", {"request": request})
+
 @router.get("/upload", include_in_schema=False)
 @require_login
 async def serve_upload(request: Request):
-     return templates.TemplateResponse("upload.html", {"request": request})
-  
+    return templates.TemplateResponse("upload.html", {"request": request})
+
 @router.get("/favicon.ico", include_in_schema=False)
 def favicon():
     # If you have a real favicon in `frontend/static/favicon.ico`:
     favicon_path = Path(__file__).parent.parent / "frontend" / "static" / "favicon.ico"
     return str(favicon_path)
+
+@router.get("/status")
+@require_login
+async def status_dashboard(request: Request):
+    """
+    Status dashboard showing all configured integration targets
+    """
+    from app.utils.config_validator import get_provider_status
+    
+    # Get provider status
+    providers = get_provider_status()
+    
+    return templates.TemplateResponse(
+        "status_dashboard.html",
+        {
+            "request": request, 
+            "providers": providers,
+            "debug_enabled": getattr(settings, 'debug', False),
+            "last_check": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        }
+    )
+
+@router.get("/env")
+@require_login
+async def env_debug(request: Request):
+    """
+    Debug endpoint to view environment variables and settings
+    Only shows values when DEBUG is True
+    """
+    # Default DEBUG to True for this route
+    debug_enabled = True
+    
+    # Get settings data
+    from app.utils.config_validator import get_settings_for_display
+    settings_data = get_settings_for_display(show_values=debug_enabled)
+    
+    return templates.TemplateResponse(
+        "env_debug.html",
+        {
+            "request": request, 
+            "settings": settings_data,
+            "debug_enabled": debug_enabled,
+            "app_version": getattr(settings, 'version', 'Unknown')
+        }
+    )
+
+# Note: The frontend/router.py is now a submodule organization, 
+# but we're keeping this file for compatibility until we've fully migrated

app/frontend/router.py

@@ -0,0 +1,90 @@
+from fastapi import APIRouter, Request, Depends, HTTPException
+from fastapi.templating import Jinja2Templates
+from pathlib import Path
+from datetime import datetime
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.auth import require_login
+from app.database import SessionLocal
+
+router = APIRouter()
+
+# Set up Jinja2 templates
+templates_dir = Path(__file__).parent.parent.parent / "frontend" / "templates"
+templates = Jinja2Templates(directory=str(templates_dir))
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+
+@router.get("/", include_in_schema=False)
+async def serve_index(request: Request):
+    return templates.TemplateResponse("index.html", {"request": request})
+
+@router.get("/about", include_in_schema=False)
+async def serve_about(request: Request):
+    return templates.TemplateResponse("about.html", {"request": request})
+
+@router.get("/upload", include_in_schema=False)
+@require_login
+async def serve_upload(request: Request):
+    return templates.TemplateResponse("upload.html", {"request": request})
+
+@router.get("/files")
+@require_login
+def files_page(request: Request, db: Session = Depends(get_db)):
+    return templates.TemplateResponse("files.html", {"request": request})
+
+@router.get("/favicon.ico", include_in_schema=False)
+def favicon():
+    favicon_path = Path(__file__).parent.parent.parent / "frontend" / "static" / "favicon.ico"
+    return str(favicon_path)
+
+@router.get("/status")
+@require_login
+async def status_dashboard(request: Request):
+    """
+    Status dashboard showing all configured integration targets
+    """
+    from app.utils.config_validator import get_provider_status
+    
+    # Get provider status
+    providers = get_provider_status()
+    
+    return templates.TemplateResponse(
+        "status_dashboard.html",
+        {
+            "request": request, 
+            "providers": providers,
+            "debug_enabled": getattr(settings, 'debug', False),
+            "last_check": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        }
+    )
+
+@router.get("/env")
+@require_login
+async def env_debug(request: Request):
+    """
+    Debug endpoint to view environment variables and settings
+    Only shows values when DEBUG is True
+    """
+    # Default DEBUG to True for this route
+    debug_enabled = True
+    
+    # Get settings data
+    from app.utils.config_validator import get_settings_for_display
+    settings_data = get_settings_for_display(show_values=debug_enabled)
+    
+    return templates.TemplateResponse(
+        "env_debug.html",
+        {
+            "request": request, 
+            "settings": settings_data,
+            "debug_enabled": debug_enabled,
+            "app_version": getattr(settings, 'version', 'Unknown')
+        }
+    )

app/main.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 import os
+import logging
 
-from fastapi import FastAPI, HTTPException, UploadFile, File, status, Request
-from fastapi.responses import FileResponse
+from fastapi import FastAPI, HTTPException, Request, status
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from starlette.middleware.sessions import SessionMiddleware
@@ -13,16 +13,17 @@ from pathlib import Path
 
 from app.database import init_db
 from app.config import settings
-from app.tasks.process_document import process_document  # Updated import
-from app.tasks.upload_to_dropbox import upload_to_dropbox
-from app.tasks.upload_to_paperless import upload_to_paperless
-from app.tasks.upload_to_nextcloud import upload_to_nextcloud
-from app.tasks.send_to_all import send_to_all_destinations
+from app.utils.config_validator import check_all_configs
 
-from app.api import router as api_router
-from app.frontend import router as frontend_router
+# Import both the traditional and new routers - we'll keep both available for compatibility
+from app.frontend.router import router as frontend_router_original
+from app.api.router import router as api_router_original
 from app.auth import router as auth_router
 
+# Import also the organized router modules
+from app.frontend.router import router as frontend_router_organized
+from app.api.router import router as api_router_organized
+
 # Load configuration from .env for the session key
 config = Config(".env")
 SESSION_SECRET = config(
@@ -53,123 +54,27 @@ app.mount("/static", StaticFiles(directory=frontend_static_dir), name="static")
 def on_startup():
     init_db()  # Create tables if they don't exist
 
-@app.post("/process/")
-def process(file_path: str):
-    """
-    API Endpoint to start document processing.
-    This enqueues document processing which handles the full pipeline.
-    """
-    if not os.path.isabs(file_path):
-        file_path = os.path.join(settings.workdir, file_path)
-
-    if not os.path.exists(file_path):
-        raise HTTPException(
-            status_code=400, detail=f"File {file_path} not found."
-        )
-
-    task = process_document.delay(file_path)  # Updated function call
-    return {"task_id": task.id, "status": "queued"}
-
-@app.post("/send_to_dropbox/")
-def send_to_dropbox(file_path: str):
-    if not os.path.isabs(file_path):
-        file_path = os.path.join(settings.workdir, 'processed', file_path)
-    if not os.path.exists(file_path):
-        raise HTTPException(
-            status_code=400, detail=f"File {file_path} not found."
-        )
-    task = upload_to_dropbox.delay(file_path)
-    return {"task_id": task.id, "status": "queued"}
-
-@app.post("/send_to_paperless/")
-def send_to_paperless(file_path: str):
-    if not os.path.isabs(file_path):
-        file_path = os.path.join(settings.workdir, 'processed', file_path)
-    if not os.path.exists(file_path):
-        raise HTTPException(
-            status_code=400, detail=f"File {file_path} not found."
-        )
-    task = upload_to_paperless.delay(file_path)
-    return {"task_id": task.id, "status": "queued"}
-
-@app.post("/send_to_nextcloud/")
-def send_to_nextcloud(file_path: str):
-    if not os.path.isabs(file_path):
-        file_path = os.path.join(settings.workdir, 'processed', file_path)
-    if not os.path.exists(file_path):
-        raise HTTPException(
-            status_code=400, detail=f"File {file_path} not found."
-        )
-    task = upload_to_nextcloud.delay(file_path)
-    return {"task_id": task.id, "status": "queued"}
-
-@app.post("/send_to_all_destinations/")
-def send_to_all_destinations_endpoint(file_path: str):
-    """
-    Call the aggregator task that sends this file to dropbox, nextcloud, and paperless.
-    """
-    if not os.path.isabs(file_path):
-        file_path = os.path.join(settings.workdir, 'processed', file_path)
-
-    if not os.path.exists(file_path):
-        raise HTTPException(
-            status_code=400, detail=f"File {file_path} not found."
-        )
-
-    task = send_to_all_destinations.delay(file_path)
-    return {"task_id": task.id, "status": "queued", "file_path": file_path}
-
-@app.post("/processall")
-def process_all_pdfs_in_workdir():
-    """
-    Finds all .pdf files in <workdir> and enqueues them for processing.
-    """
-    target_dir = settings.workdir
-    if not os.path.exists(target_dir):
-        raise HTTPException(
-            status_code=400, detail=f"Directory {target_dir} does not exist."
-        )
-
-    pdf_files = []
-    for filename in os.listdir(target_dir):
-        if filename.lower().endswith(".pdf"):
-            pdf_files.append(filename)
-
-    if not pdf_files:
-        return {"message": "No PDF files found in that directory."}
-
-    task_ids = []
-    for pdf in pdf_files:
-        file_path = os.path.join(target_dir, pdf)
-        task = process_document.delay(file_path)  # Updated function call
-        task_ids.append(task.id)
-
-    return {
-        "message": f"Enqueued {len(pdf_files)} PDFs to upload_to_s3",
-        "pdf_files": pdf_files,
-        "task_ids": task_ids
-    }
-
-@app.post("/ui-upload")
-async def ui_upload(file: UploadFile = File(...)):
-    """Endpoint to accept a user-uploaded file and enqueue it for processing."""
-    workdir = "/workdir"
-    target_path = os.path.join(workdir, file.filename)
-    try:
-        with open(target_path, "wb") as f:
-            content = await file.read()
-            f.write(content)
-    except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=f"Failed to save file: {e}"
-        )
-
-    task = process_document.delay(target_path)  # Updated function call
-    return {"task_id": task.id, "status": "queued"}
+@app.on_event("startup")
+async def startup_event():
+    """Run startup tasks for the application"""
+    # Force settings dump to log for troubleshooting
+    from app.utils.config_validator import dump_all_settings
+    dump_all_settings()
+    
+    # Validate configuration
+    config_issues = check_all_configs()
+    
+    # Log overall status
+    has_issues = any(config_issues['email']) or any(len(issues) > 0 for provider, issues in config_issues['storage'].items())
+    if has_issues:
+        logging.warning("Application started with configuration issues - some features may be unavailable")
+    else:
+        logging.info("Application started with valid configuration")
+    
+    logging.info("Router organization note: Using route handlers from main app directory for now")
+    logging.info("In the future, we'll transition fully to the frontend/ and api/ submodules")
 
 # Custom 404 - we can still return the Jinja2 template, or the old static file:
-# For a dynamic 404 using the base layout, see "frontend/404.html" usage below:
 @app.exception_handler(404)
 async def custom_404_handler(request: Request, exc: HTTPException):
     # Serve the 404 template directly
@@ -194,7 +99,13 @@ async def custom_500_handler(request: Request, exc: Exception):
 def test_500():
     raise RuntimeError("Testing forced 500 error!")
 
-# Include the frontend and auth routers
-app.include_router(frontend_router)
+# Include the routers - for now we're using the original routers
+# Later we can switch to the organized router structure
+app.include_router(frontend_router_original)
 app.include_router(auth_router)
-app.include_router(api_router, prefix="/api")
+app.include_router(api_router_original, prefix="/api")
+
+# Note: These are the future, organized router modules that we'll eventually switch to
+# For now they are disabled to avoid duplicate route conflicts
+# app.include_router(frontend_router_organized) 
+# app.include_router(api_router_organized, prefix="/api")

app/tasks/send_to_all.py

@@ -1,21 +1,98 @@
-# app/tasks/send_to_all.py
+#!/usr/bin/env python3
 
-from app.celery_app import celery
+import os
+import logging
+from app.config import settings
+from app.tasks.retry_config import BaseTaskWithRetry
 from app.tasks.upload_to_dropbox import upload_to_dropbox
 from app.tasks.upload_to_nextcloud import upload_to_nextcloud
 from app.tasks.upload_to_paperless import upload_to_paperless
+from app.tasks.upload_to_google_drive import upload_to_google_drive
+from app.tasks.upload_to_webdav import upload_to_webdav
+from app.tasks.upload_to_ftp import upload_to_ftp
+from app.tasks.upload_to_sftp import upload_to_sftp
+from app.tasks.upload_to_email import upload_to_email
+from app.tasks.upload_to_onedrive import upload_to_onedrive
+from app.tasks.upload_to_s3 import upload_to_s3
+from app.celery_app import celery
 
-@celery.task
+logger = logging.getLogger(__name__)
+
+@celery.task(base=BaseTaskWithRetry)
 def send_to_all_destinations(file_path: str):
-    """
-    Fires off tasks to upload a single file to Dropbox, Nextcloud, and Paperless.
-    These tasks run in parallel (Celery returns immediately from each .delay()).
-    """
-    upload_to_dropbox.delay(file_path)
-    upload_to_nextcloud.delay(file_path)
-    upload_to_paperless.delay(file_path)
+    """Distribute a file to all configured storage destinations."""
+    
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+    
+    logger.info(f"Sending {file_path} to all configured destinations")
+    results = {}
+    
+    # Send to Dropbox if configured
+    if settings.dropbox_app_key and settings.dropbox_app_secret and settings.dropbox_refresh_token:
+        logger.info(f"Queueing {file_path} for Dropbox upload")
+        task = upload_to_dropbox.delay(file_path)
+        results["dropbox_task_id"] = task.id
+    
+    # Send to Nextcloud if configured
+    if settings.nextcloud_upload_url and settings.nextcloud_username and settings.nextcloud_password:
+        logger.info(f"Queueing {file_path} for Nextcloud upload")
+        task = upload_to_nextcloud.delay(file_path)
+        results["nextcloud_task_id"] = task.id
+    
+    # Send to Paperless if configured
+    if settings.paperless_ngx_api_token and settings.paperless_host:
+        logger.info(f"Queueing {file_path} for Paperless upload")
+        task = upload_to_paperless.delay(file_path)
+        results["paperless_task_id"] = task.id
+    
+    # Send to Google Drive if configured
+    if settings.google_drive_credentials_json:
+        logger.info(f"Queueing {file_path} for Google Drive upload")
+        task = upload_to_google_drive.delay(file_path)
+        results["google_drive_task_id"] = task.id
 
+    # Send to WebDAV if configured
+    if settings.webdav_url and settings.webdav_username and settings.webdav_password:
+        logger.info(f"Queueing {file_path} for WebDAV upload")
+        task = upload_to_webdav.delay(file_path)
+        results["webdav_task_id"] = task.id
+        
+    # Send to FTP if configured
+    if settings.ftp_host and settings.ftp_username and settings.ftp_password:
+        logger.info(f"Queueing {file_path} for FTP upload")
+        task = upload_to_ftp.delay(file_path)
+        results["ftp_task_id"] = task.id
+        
+    # Send to SFTP if configured
+    if settings.sftp_host and settings.sftp_username and (settings.sftp_password or settings.sftp_private_key):
+        logger.info(f"Queueing {file_path} for SFTP upload")
+        task = upload_to_sftp.delay(file_path)
+        results["sftp_task_id"] = task.id
+
+    # Send via email if configured
+    if settings.email_host and settings.email_username and settings.email_password and settings.email_default_recipient:
+        logger.info(f"Queueing {file_path} for email delivery")
+        task = upload_to_email.delay(file_path)
+        results["email_task_id"] = task.id
+        
+    # Send to OneDrive if configured
+    if settings.onedrive_client_id and settings.onedrive_client_secret and (
+        settings.onedrive_refresh_token or 
+        (settings.onedrive_tenant_id and settings.onedrive_tenant_id != "common")
+    ):
+        logger.info(f"Queueing {file_path} for OneDrive upload")
+        task = upload_to_onedrive.delay(file_path)
+        results["onedrive_task_id"] = task.id
+        
+    # Send to Amazon S3 if configured
+    if settings.s3_bucket_name and settings.aws_access_key_id and settings.aws_secret_access_key:
+        logger.info(f"Queueing {file_path} for S3 upload")
+        task = upload_to_s3.delay(file_path)
+        results["s3_task_id"] = task.id
+    
     return {
-        "status": "All upload tasks enqueued",
-        "file_path": file_path
+        "status": "Queued",
+        "file_path": file_path,
+        "tasks": results
     }

app/tasks/upload_to_dropbox.py

@@ -1,20 +1,38 @@
 #!/usr/bin/env python3
 
 import os
+import logging
 import requests
 import dropbox
+from dropbox.exceptions import ApiError, AuthError
 from app.config import settings
 from app.tasks.retry_config import BaseTaskWithRetry
 from app.celery_app import celery
+from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
+
+logger = logging.getLogger(__name__)
 
 def get_dropbox_access_token():
     """Refresh the Dropbox access token using the stored refresh token from ENV."""
+    
+    # Check if needed settings are available
+    if not hasattr(settings, 'dropbox_refresh_token') or not settings.dropbox_refresh_token:
+        logger.error("Cannot refresh Dropbox token: Missing refresh token")
+        return None
+        
+    if not hasattr(settings, 'dropbox_app_key') or not settings.dropbox_app_key:
+        logger.error("Cannot refresh Dropbox token: Missing app key")
+        return None
+        
+    if not hasattr(settings, 'dropbox_app_secret') or not settings.dropbox_app_secret:
+        logger.error("Cannot refresh Dropbox token: Missing app secret")
+        return None
 
     token_url = "https://api.dropbox.com/oauth2/token"
     headers = {"Content-Type": "application/x-www-form-urlencoded"}
     data = {
         "grant_type": "refresh_token",
-        "refresh_token": settings.dropbox_refresh_token,  # Now using ENV
+        "refresh_token": settings.dropbox_refresh_token,
         "client_id": settings.dropbox_app_key,
         "client_secret": settings.dropbox_app_secret,
     }
@@ -25,50 +43,110 @@ def get_dropbox_access_token():
         return response.json()["access_token"]
     else:
         error_msg = f"Failed to refresh Dropbox token: {response.status_code} - {response.text}"
-        print(f"[ERROR] {error_msg}")
+        logger.error(error_msg)
         raise Exception(error_msg)
 
 @celery.task(base=BaseTaskWithRetry)
 def upload_to_dropbox(file_path: str):
-    """Uploads a file to Dropbox using the API."""
-
+    """
+    Upload a file to Dropbox.
+    """
     if not os.path.exists(file_path):
-        raise FileNotFoundError(f"File not found: {file_path}")
-
-    # Extract filename and set target path
+        error_msg = f"File not found: {file_path}"
+        logger.error(error_msg)
+        raise FileNotFoundError(error_msg)
+    
+    # Check if Dropbox is properly configured
+    if not (hasattr(settings, 'dropbox_app_key') and settings.dropbox_app_key and
+            hasattr(settings, 'dropbox_app_secret') and settings.dropbox_app_secret and
+            hasattr(settings, 'dropbox_refresh_token') and settings.dropbox_refresh_token):
+        logger.info("Dropbox upload skipped: Missing configuration")
+        return {"status": "Skipped", "reason": "Dropbox settings not configured"}
+    
     filename = os.path.basename(file_path)
-    dropbox_path = f"{settings.dropbox_folder}/{filename}"
-
+    sanitized_filename = sanitize_filename(filename)
+    
     try:
-        # Get fresh access token
+        # Get access token from refresh token
         access_token = get_dropbox_access_token()
+        if not access_token:
+            return {"status": "Failed", "reason": "Could not obtain access token"}
+            
         dbx = dropbox.Dropbox(access_token)
-
-        file_size = os.path.getsize(file_path)
-        chunk_size = 4 * 1024 * 1024  # 4MB chunk size
-
-        with open(file_path, "rb") as file_data:
-            if file_size <= chunk_size:
-                dbx.files_upload(file_data.read(), dropbox_path)
-            else:
-                upload_session_start_result = dbx.files_upload_session_start(file_data.read(chunk_size))
-                cursor = dropbox.files.UploadSessionCursor(
-                    session_id=upload_session_start_result.session_id,
-                    offset=file_data.tell(),
-                )
-                commit = dropbox.files.CommitInfo(path=dropbox_path)
-
+        
+        # Calculate remote path based on local file structure
+        remote_base = settings.dropbox_folder or ""
+        remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
+        
+        # Function to check if file exists in Dropbox
+        def check_exists_in_dropbox(path):
+            try:
+                dbx.files_get_metadata(path)
+                return True
+            except ApiError as e:
+                if e.error.is_path() and e.error.get_path().is_not_found():
+                    return False
+                raise
+        
+        # Get a unique path in case of collision
+        remote_full_path = f"/{remote_path}"  # Dropbox paths should start with /
+        remote_full_path = remote_full_path.replace('//', '/')  # Clean double slashes
+        
+        # Check for potential file collision and get a unique name if needed
+        dropbox_path = get_unique_filename(remote_full_path, check_exists_in_dropbox)
+        
+        # Upload the file
+        logger.info(f"Uploading {filename} to Dropbox at {dropbox_path}")
+        with open(file_path, 'rb') as file_data:
+            # Use files_upload_session for large files to avoid timeouts
+            file_size = os.path.getsize(file_path)
+            if file_size > 10 * 1024 * 1024:  # 10 MB threshold for chunked upload
+                cursor = None
+                chunk_size = 4 * 1024 * 1024  # 4 MB chunks
+                file_data.seek(0)
+                
+                # Start upload session
+                session_start = dbx.files_upload_session_start(file_data.read(chunk_size))
+                cursor = dropbox.files.UploadSessionCursor(session_start.session_id, file_data.tell())
+                
+                # Upload chunks until we reach the end
                 while file_data.tell() < file_size:
                     if (file_size - file_data.tell()) <= chunk_size:
-                        dbx.files_upload_session_finish(file_data.read(chunk_size), cursor, commit)
+                        # Last chunk
+                        dbx.files_upload_session_finish(
+                            file_data.read(chunk_size),
+                            cursor,
+                            dropbox.files.CommitInfo(path=dropbox_path, mode=dropbox.files.WriteMode.overwrite)
+                        )
                     else:
+                        # More chunks to upload
                         dbx.files_upload_session_append_v2(file_data.read(chunk_size), cursor)
                         cursor.offset = file_data.tell()
-
-        print(f"[INFO] Successfully uploaded {filename} to Dropbox at {dropbox_path}.")
-        return {"status": "Completed", "file": file_path}
-
-    except Exception as e:
-        error_msg = f"[ERROR] Failed to upload {filename} to Dropbox: {str(e)}"
-        print(error_msg)
+            else:
+                # Small file, direct upload
+                file_data.seek(0)
+                dbx.files_upload(
+                    file_data.read(),
+                    dropbox_path,
+                    mode=dropbox.files.WriteMode.overwrite
+                )
+        
+        logger.info(f"Successfully uploaded {filename} to Dropbox at {dropbox_path}")
+        return {
+            "status": "Completed",
+            "file_path": file_path,
+            "dropbox_path": dropbox_path
+        }
+    
+    except AuthError:
+        error_msg = f"[ERROR] Authentication failed while uploading {filename} to Dropbox. Check token."
+        logger.error(error_msg)
+        raise Exception(error_msg)
+    except ApiError as e:
+        error_msg = f"[ERROR] Failed to upload {filename} to Dropbox: {e}"
+        logger.error(error_msg)
+        raise Exception(error_msg)
+    except Exception as e:
+        error_msg = f"[ERROR] Unexpected error uploading {filename} to Dropbox: {e}"
+        logger.error(error_msg)
         raise Exception(error_msg)

app/tasks/upload_to_email.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+
+import os
+import json
+import smtplib
+import socket
+import logging
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.mime.application import MIMEApplication
+from email.mime.image import MIMEImage
+from datetime import datetime
+from pathlib import Path
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+from app.config import settings
+from app.tasks.retry_config import BaseTaskWithRetry
+from app.celery_app import celery
+
+logger = logging.getLogger(__name__)
+
+def get_email_template(template_name="default.html"):
+    """
+    Load email template from one of these locations in order of precedence:
+    1. Custom template from workdir/templates/email/
+    2. Default template from app/templates/email/
+    """
+    # First try to load from workdir (user customizable location)
+    try:
+        workdir_template_path = os.path.join(settings.workdir, "templates", "email")
+        if os.path.exists(workdir_template_path):
+            env = Environment(
+                loader=FileSystemLoader(workdir_template_path),
+                autoescape=select_autoescape(['html', 'xml'])
+            )
+            env.globals['now'] = datetime.now  # Add the now function to the Jinja environment
+            template = env.get_template(template_name)
+            logger.info(f"Using custom email template from workdir: {template_name}")
+            return template
+    except Exception as e:
+        logger.warning(f"Failed to load custom email template: {str(e)}")
+    
+    # Fallback to built-in template
+    try:
+        # Get the app directory path (where this file is)
+        current_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        app_template_path = os.path.join(current_dir, "templates", "email")
+        env = Environment(
+            loader=FileSystemLoader(app_template_path),
+            autoescape=select_autoescape(['html', 'xml'])
+        )
+        env.globals['now'] = datetime.now  # Add the now function to the Jinja environment
+        template = env.get_template(template_name)
+        logger.info(f"Using built-in email template: {template_name}")
+        return template
+    except Exception as e:
+        logger.error(f"Failed to load built-in email template: {str(e)}")
+        raise ValueError(f"Could not find any valid email template: {str(e)}")
+
+def extract_metadata_from_file(file_path):
+    """
+    Try to extract metadata from a file using several methods:
+    1. Check for a .json metadata file with the same name
+    2. Extract metadata from PDF if it's embedded
+    
+    Returns a dictionary of metadata or None if not found
+    """
+    metadata = {}
+    
+    # Check for separate metadata JSON file
+    metadata_path = os.path.splitext(file_path)[0] + '.json'
+    if os.path.exists(metadata_path):
+        try:
+            with open(metadata_path, 'r', encoding='utf-8') as f:
+                metadata = json.load(f)
+                logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
+                return metadata
+        except Exception as e:
+            logger.warning(f"Failed to load metadata from JSON file: {str(e)}")
+    
+    # TODO: For PDF files, try to extract embedded metadata using PyPDF2
+    # This would require additional dependencies, so for now we'll just check for external JSON
+    
+    return metadata
+
+def attach_logo(msg):
+    """Attach the DocuNova logo to the email with proper Content-ID."""
+    try:
+        # Try to find logo in workdir first (for customization)
+        custom_logo_path = os.path.join(settings.workdir, "templates", "email", "logo.png")
+        if os.path.exists(custom_logo_path):
+            logo_path = custom_logo_path
+        else:
+            # Use built-in logo
+            app_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+            logo_path = os.path.join(app_dir, "static", "logo.png")
+            # Fallback to logo in frontend/static if app/static doesn't exist
+            if not os.path.exists(logo_path):
+                logo_path = os.path.join(app_dir, "..", "frontend", "static", "logo.png")
+        
+        if os.path.exists(logo_path):
+            with open(logo_path, 'rb') as img:
+                logo_data = img.read()
+                
+            # Determine image MIME type based on extension
+            mimetype = 'image/svg+xml' if logo_path.endswith('.svg') else 'image/png'
+            logo_attach = MIMEImage(logo_data, mimetype)
+            logo_attach.add_header('Content-ID', '<logo>')
+            logo_attach.add_header('Content-Disposition', 'inline', filename='logo.png')
+            msg.attach(logo_attach)
+            logger.info(f"Logo attached from {logo_path}")
+            return True
+        else:
+            logger.warning("Could not find logo file")
+            return False
+            
+    except Exception as e:
+        logger.warning(f"Error attaching logo: {str(e)}")
+        return False
+
+@celery.task(base=BaseTaskWithRetry)
+def upload_to_email(file_path: str, recipients=None, subject=None, message=None, template_name="default.html", include_metadata=True):
+    """
+    Sends a file via email to the specified recipients.
+    If recipients is None, uses the configured default email recipient.
+    """
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    # Extract filename
+    filename = os.path.basename(file_path)
+    
+    # Check if email settings are configured
+    if not settings.email_host:
+        error_msg = "Email host is not configured"
+        logger.error(error_msg)
+        return {"status": "Skipped", "reason": error_msg}
+        
+    # Log email configuration for debugging
+    logger.debug(f"Email config - Host: {settings.email_host}, Port: {settings.email_port}, " 
+                f"Username: {settings.email_username}, TLS: {settings.email_use_tls}")
+
+    # Use provided recipients or fall back to default
+    if not recipients:
+        if not settings.email_default_recipient:
+            error_msg = "No recipients specified and no default recipient configured"
+            logger.error(error_msg)
+            return {"status": "Skipped", "reason": error_msg}
+        recipients = [settings.email_default_recipient]
+    elif isinstance(recipients, str):
+        recipients = [recipients]  # Convert single email to list
+
+    # Use provided subject or create default
+    if not subject:
+        subject = f"DocuNova Document: {filename}"
+
+    # Extract document metadata if available
+    metadata = {}
+    if include_metadata:
+        metadata = extract_metadata_from_file(file_path)
+
+    try:
+        # Create the email
+        msg = MIMEMultipart('related')  # Changed to 'related' to properly handle inline images
+        msg['From'] = settings.email_sender or settings.email_username
+        msg['To'] = ", ".join(recipients)
+        msg['Subject'] = subject
+
+        # Create alternative part for HTML content
+        alt_part = MIMEMultipart('alternative')
+        msg.attach(alt_part)
+        
+        # Attach logo to the email
+        has_logo = attach_logo(msg)
+        
+        # Load and render template
+        template = get_email_template(template_name)
+        
+        # Context data for the template
+        context = {
+            "filename": filename,
+            "message": message or f"Attached is the document: {filename}",
+            "app_name": "DocuNova",
+            "app_url": f"https://{settings.external_hostname}" if settings.external_hostname else None,
+            "custom_message": message,
+            "metadata": metadata,
+            "has_metadata": bool(metadata),
+            "has_logo": has_logo,
+            "current_year": datetime.now().year
+        }
+        
+        # Render HTML body
+        html_content = template.render(**context)
+        alt_part.attach(MIMEText(html_content, 'html'))
+
+        # Attach the file
+        with open(file_path, "rb") as file:
+            attachment = MIMEApplication(file.read(), _subtype="pdf")
+            attachment.add_header('Content-Disposition', f'attachment; filename="{filename}"')
+            msg.attach(attachment)
+
+        try:
+            # First try to resolve the hostname
+            socket.gethostbyname(settings.email_host)
+            
+            # Connect to the SMTP server
+            with smtplib.SMTP(settings.email_host, settings.email_port, timeout=30) as server:
+                # Use TLS if specified
+                if settings.email_use_tls:
+                    server.starttls()
+                
+                # Login if credentials are provided
+                if settings.email_username and settings.email_password:
+                    server.login(settings.email_username, settings.email_password)
+                
+                # Send the email
+                server.send_message(msg)
+
+            logger.info(f"Successfully sent {filename} via email to {', '.join(recipients)}")
+            return {
+                "status": "Completed",
+                "file": file_path,
+                "recipients": recipients,
+                "subject": subject,
+                "metadata_included": bool(metadata),
+                "logo_included": has_logo
+            }
+        except socket.gaierror as e:
+            error_msg = f"Failed to resolve email host: {settings.email_host} - {str(e)}"
+            logger.error(error_msg)
+            return {"status": "Failed", "reason": error_msg, "error": str(e)}
+        except ConnectionRefusedError as e:
+            error_msg = f"Connection refused to SMTP server {settings.email_host}:{settings.email_port} - {str(e)}"
+            logger.error(error_msg)
+            return {"status": "Failed", "reason": error_msg, "error": str(e)}
+        except TimeoutError as e:
+            error_msg = f"Connection timeout to SMTP server {settings.email_host}:{settings.email_port} - {str(e)}"
+            logger.error(error_msg)
+            return {"status": "Failed", "reason": error_msg, "error": str(e)}
+
+    except Exception as e:
+        error_msg = f"Failed to send {filename} via email: {str(e)}"
+        logger.error(error_msg)
+        raise Exception(error_msg)

app/tasks/upload_to_ftp.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+import os
+import ftplib
+from app.config import settings
+from app.tasks.retry_config import BaseTaskWithRetry
+from app.celery_app import celery
+import logging
+
+logger = logging.getLogger(__name__)
+
+@celery.task(base=BaseTaskWithRetry)
+def upload_to_ftp(file_path: str):
+    """Uploads a file to an FTP server in the configured folder."""
+
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    # Extract filename
+    filename = os.path.basename(file_path)
+
+    # Check if FTP settings are configured
+    if not settings.ftp_host:
+        error_msg = "FTP host is not configured"
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+
+    try:
+        # Connect to FTP server
+        ftp = ftplib.FTP()
+        ftp.connect(
+            host=settings.ftp_host,
+            port=settings.ftp_port or 21
+        )
+        
+        # Login with credentials
+        ftp.login(
+            user=settings.ftp_username, 
+            passwd=settings.ftp_password
+        )
+        
+        # Change to target directory if specified
+        if settings.ftp_folder:
+            try:
+                # Try to navigate to the directory, create if it doesn't exist
+                ftp_folder = settings.ftp_folder
+                # Remove leading slash if present
+                if ftp_folder.startswith('/'):
+                    ftp_folder = ftp_folder[1:]
+                
+                # Try to change to the directory
+                try:
+                    ftp.cwd(ftp_folder)
+                except ftplib.error_perm:
+                    # Create directory structure if it doesn't exist
+                    folders = ftp_folder.split('/')
+                    current_dir = ''
+                    for folder in folders:
+                        if folder:
+                            current_dir += f"/{folder}"
+                            try:
+                                ftp.cwd(current_dir)
+                            except ftplib.error_perm:
+                                ftp.mkd(current_dir)
+                                ftp.cwd(current_dir)
+            except ftplib.Error as e:
+                error_msg = f"Failed to change/create directory on FTP server: {str(e)}"
+                logger.error(error_msg)
+                raise Exception(error_msg)
+        
+        # Upload the file
+        with open(file_path, 'rb') as file_data:
+            ftp.storbinary(f'STOR {filename}', file_data)
+        
+        # Close FTP connection
+        ftp.quit()
+        
+        logger.info(f"Successfully uploaded {filename} to FTP server at {settings.ftp_host}")
+        return {
+            "status": "Completed", 
+            "file": file_path, 
+            "ftp_host": settings.ftp_host,
+            "ftp_path": f"{settings.ftp_folder}/{filename}" if settings.ftp_folder else filename
+        }
+    
+    except Exception as e:
+        error_msg = f"Failed to upload {filename} to FTP server: {str(e)}"
+        logger.error(error_msg)
+        raise Exception(error_msg)

app/tasks/upload_to_google_drive.py

@@ -0,0 +1,103 @@
+"""
+app/tasks/upload_to_google_drive.py
+"""
+
+import os
+import json
+import logging
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaFileUpload
+from google.oauth2.service_account import Credentials
+
+from app.config import settings
+from app.tasks.retry_config import BaseTaskWithRetry
+from app.celery_app import celery
+
+logger = logging.getLogger(__name__)
+
+def get_google_drive_service():
+    """
+    Authenticate with Google Drive API using service account credentials
+    and return an authorized service object.
+    """
+    try:
+        # Load service account credentials from settings
+        if not settings.google_drive_credentials_json:
+            logger.error("Google Drive credentials not configured")
+            return None
+            
+        credentials_dict = json.loads(settings.google_drive_credentials_json)
+        credentials = Credentials.from_service_account_info(
+            credentials_dict, 
+            scopes=['https://www.googleapis.com/auth/drive']
+        )
+        
+        # Delegate to user if specified
+        if settings.google_drive_delegate_to:
+            credentials = credentials.with_subject(settings.google_drive_delegate_to)
+            
+        # Build and return the service
+        service = build('drive', 'v3', credentials=credentials)
+        return service
+    
+    except Exception as e:
+        logger.error(f"Failed to authenticate with Google Drive: {str(e)}")
+        return None
+
+
+@celery.task(base=BaseTaskWithRetry)
+def upload_to_google_drive(file_path: str):
+    """Uploads a file to Google Drive in the configured folder."""
+
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    # Extract filename from path
+    filename = os.path.basename(file_path)
+
+    try:
+        # Get Google Drive service
+        service = get_google_drive_service()
+        if not service:
+            raise Exception("Failed to initialize Google Drive service")
+
+        # Prepare the file metadata
+        file_metadata = {
+            'name': filename,
+        }
+        
+        # If folder ID is specified, set parent folder
+        if settings.google_drive_folder_id:
+            file_metadata['parents'] = [settings.google_drive_folder_id]
+        
+        # Upload file with metadata
+        media = MediaFileUpload(
+            file_path, 
+            mimetype='application/pdf',
+            resumable=True
+        )
+        
+        file = service.files().create(
+            body=file_metadata,
+            media_body=media,
+            fields='id,name,webViewLink'
+        ).execute()
+        
+        # Log success details
+        file_id = file.get('id')
+        web_view_link = file.get('webViewLink')
+        
+        logger.info(f"Successfully uploaded {filename} to Google Drive with ID: {file_id}")
+        logger.info(f"File accessible at: {web_view_link}")
+        
+        return {
+            "status": "Completed", 
+            "file_path": file_path,
+            "google_drive_file_id": file_id,
+            "google_drive_web_link": web_view_link
+        }
+
+    except Exception as e:
+        error_msg = f"Failed to upload {filename} to Google Drive: {str(e)}"
+        logger.error(error_msg)
+        raise Exception(error_msg)

app/tasks/upload_to_nextcloud.py

@@ -1,37 +1,128 @@
 #!/usr/bin/env python3
 
 import os
+import logging
 import requests
+from requests.auth import HTTPBasicAuth
 from app.config import settings
-from app.tasks.retry_config import BaseTaskWithRetry
 from app.celery_app import celery
+from app.tasks.retry_config import BaseTaskWithRetry
+from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
+
+logger = logging.getLogger(__name__)
 
 @celery.task(base=BaseTaskWithRetry)
 def upload_to_nextcloud(file_path: str):
-    """Uploads a file to Nextcloud in the configured folder."""
-
+    """
+    Upload a file to Nextcloud WebDAV.
+    """
     if not os.path.exists(file_path):
-        raise FileNotFoundError(f"File not found: {file_path}")
-
-    # Extract filename
+        error_msg = f"File not found: {file_path}"
+        logger.error(error_msg)
+        raise FileNotFoundError(error_msg)
+    
+    # For Nextcloud, we need to check for 'nextcloud_upload_url' instead of 'nextcloud_url'
+    # This is what's shown in your env view
+    if not (getattr(settings, 'nextcloud_upload_url', None) and 
+            getattr(settings, 'nextcloud_username', None) and 
+            getattr(settings, 'nextcloud_password', None)):
+        logger.info("Nextcloud upload skipped: Missing configuration")
+        return {"status": "Skipped", "reason": "Nextcloud settings not configured"}
+    
     filename = os.path.basename(file_path)
-
-    # Construct the full upload URL
-    nextcloud_url = f"{settings.nextcloud_upload_url}/{settings.nextcloud_folder}/{filename}"
-
-    # Read file content
-    with open(file_path, "rb") as file_data:
-        response = requests.put(
-            nextcloud_url,
-            auth=(settings.nextcloud_username, settings.nextcloud_password),
-            data=file_data
-        )
-
-    # Check if upload was successful
-    if response.status_code in (200, 201):
-        print(f"[INFO] Successfully uploaded {filename} to Nextcloud at {nextcloud_url}.")
-        return {"status": "Completed", "file": file_path}
-    else:
-        error_msg = f"[ERROR] Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
-        print(error_msg)
+    sanitized_filename = sanitize_filename(filename)
+    
+    try:
+        # Prepare WebDAV URL - use nextcloud_upload_url instead of nextcloud_url
+        webdav_url = settings.nextcloud_upload_url
+        if not webdav_url.endswith('/'):
+            webdav_url += '/'
+            
+        # Calculate remote path based on local file structure
+        remote_base = getattr(settings, 'nextcloud_folder', '') or ""
+        remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
+        full_url = f"{webdav_url}/{remote_path}"
+        
+        # Remove any double slashes (except in http://)
+        full_url = full_url.replace('://', '$PLACEHOLDER$')
+        while '//' in full_url:
+            full_url = full_url.replace('//', '/')
+        full_url = full_url.replace('$PLACEHOLDER$', '://')
+        
+        # Function to check if file exists in Nextcloud
+        def check_exists_in_nextcloud(path):
+            check_url = f"{webdav_url}{os.path.dirname(path)}"
+            try:
+                response = requests.request(
+                    'PROPFIND',
+                    check_url,
+                    auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
+                    headers={'Depth': '1'},
+                    timeout=10
+                )
+                
+                return path in response.text
+            except Exception:
+                # If we can't check, assume it doesn't exist
+                return False
+        
+        # Check for potential file collision and get a unique name if needed
+        remote_path = get_unique_filename(remote_path, check_exists_in_nextcloud)
+        full_url = f"{webdav_url}/{remote_path}"
+        
+        # Fix double slashes again
+        full_url = full_url.replace('://', '$PLACEHOLDER$')
+        while '//' in full_url:
+            full_url = full_url.replace('//', '/')
+        full_url = full_url.replace('$PLACEHOLDER$', '://')
+        
+        # Create necessary parent folders
+        parent_dirs = os.path.dirname(remote_path)
+        if parent_dirs:
+            current_path = ""
+            for folder in parent_dirs.split('/'):
+                if not folder:
+                    continue
+                current_path += f"{folder}/"
+                mkdir_url = f"{webdav_url}/{current_path}"
+                # Fix double slashes
+                mkdir_url = mkdir_url.replace('://', '$PLACEHOLDER$')
+                while '//' in mkdir_url:
+                    mkdir_url = mkdir_url.replace('//', '/')
+                mkdir_url = mkdir_url.replace('$PLACEHOLDER$', '://')
+                
+                requests.request(
+                    'MKCOL',
+                    mkdir_url,
+                    auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
+                    timeout=10
+                )
+        
+        # Upload the file
+        logger.info(f"Uploading {filename} to Nextcloud at {full_url}")
+        with open(file_path, 'rb') as file_data:
+            response = requests.put(
+                full_url,
+                data=file_data,
+                auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
+                headers={'Content-Type': 'application/octet-stream'},
+                timeout=60  # Longer timeout for larger files
+            )
+        
+        if response.status_code in (201, 204):  # Created or No Content
+            logger.info(f"Successfully uploaded {filename} to Nextcloud at {remote_path}")
+            return {
+                "status": "Completed",
+                "file_path": file_path,
+                "nextcloud_path": remote_path,
+                "response_code": response.status_code
+            }
+        else:
+            error_msg = f"[ERROR] Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
+            logger.error(error_msg)
+            raise Exception(error_msg)
+    
+    except Exception as e:
+        error_msg = f"[ERROR] Failed to upload {filename} to Nextcloud: {str(e)}"
+        logger.error(error_msg)
         raise Exception(error_msg)

app/tasks/upload_to_onedrive.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+
+import os
+import time
+import logging
+import requests
+import msal
+from app.config import settings
+from app.tasks.retry_config import BaseTaskWithRetry
+from app.celery_app import celery
+
+logger = logging.getLogger(__name__)
+
+def get_onedrive_token():
+    """
+    Get an access token for Microsoft Graph API using the appropriate flow.
+    For personal accounts, uses refresh token flow.
+    For organizational accounts, uses client credentials flow if refresh token isn't provided.
+    """
+    # Check for required settings
+    if not settings.onedrive_client_id or not settings.onedrive_client_secret:
+        raise ValueError("OneDrive client ID and client secret must be configured")
+    
+    # Use refresh token flow (works for both personal and org accounts)
+    if settings.onedrive_refresh_token:
+        # Use MSAL to get token from refresh token
+        app = msal.PublicClientApplication(settings.onedrive_client_id)
+        
+        # Request new token using refresh token
+        token_response = app.acquire_token_by_refresh_token(
+            refresh_token=settings.onedrive_refresh_token,
+            scopes=["https://graph.microsoft.com/Files.ReadWrite"]
+        )
+        
+        if "access_token" not in token_response:
+            error = token_response.get("error", "")
+            error_desc = token_response.get("error_description", "Unknown error")
+            raise ValueError(f"Failed to get access token: {error} - {error_desc}")
+            
+        return token_response["access_token"]
+    
+    # No refresh token - try client credentials (only works for org accounts)
+    elif settings.onedrive_tenant_id and settings.onedrive_tenant_id != "common":
+        authority = f"https://login.microsoftonline.com/{settings.onedrive_tenant_id}"
+        app = msal.ConfidentialClientApplication(
+            client_id=settings.onedrive_client_id,
+            client_credential=settings.onedrive_client_secret,
+            authority=authority
+        )
+        
+        # Acquire token for application
+        token_response = app.acquire_token_for_client(
+            scopes=["https://graph.microsoft.com/.default"]
+        )
+        
+        if "access_token" not in token_response:
+            error = token_response.get("error", "")
+            error_desc = token_response.get("error_description", "Unknown error")
+            raise ValueError(f"Failed to get access token: {error} - {error_desc}")
+            
+        return token_response["access_token"]
+    
+    else:
+        raise ValueError("For personal Microsoft accounts, ONEDRIVE_REFRESH_TOKEN must be configured")
+
+def create_upload_session(filename, folder_path, access_token):
+    """Creates an upload session for large files in Microsoft Graph API."""
+    # Construct the API endpoint
+    base_url = "https://graph.microsoft.com/v1.0/me/drive"
+    
+    # Format the folder path correctly
+    if folder_path:
+        # Remove leading/trailing slashes
+        folder_path = folder_path.strip('/')
+        # Replace spaces with %20
+        folder_path = folder_path.replace(' ', '%20')
+        item_path = f"/root:/{folder_path}/{filename}:/createUploadSession"
+    else:
+        item_path = f"/root:/{filename}:/createUploadSession"
+    
+    url = f"{base_url}{item_path}"
+    
+    headers = {
+        "Authorization": f"Bearer {access_token}",
+        "Content-Type": "application/json"
+    }
+    
+    response = requests.post(url, headers=headers)
+    
+    if response.status_code == 200:
+        return response.json().get("uploadUrl")
+    else:
+        error_msg = f"Failed to create upload session: {response.status_code} - {response.text}"
+        logger.error(error_msg)
+        raise Exception(error_msg)
+
+def upload_large_file(file_path, upload_url):
+    """
+    Upload a large file to OneDrive using the upload session URL.
+    Uses chunked upload for reliability.
+    """
+    # Get file size
+    file_size = os.path.getsize(file_path)
+    
+    # Define chunk size (10 MB)
+    chunk_size = 10 * 1024 * 1024
+    
+    # Open and read file in chunks
+    with open(file_path, 'rb') as f:
+        # Process file in chunks
+        chunk_number = 0
+        while True:
+            chunk = f.read(chunk_size)
+            if not chunk:
+                break
+                
+            # Get the position in the file
+            chunk_start = chunk_number * chunk_size
+            chunk_end = chunk_start + len(chunk) - 1
+            
+            # Prepare content range header
+            content_range = f"bytes {chunk_start}-{chunk_end}/{file_size}"
+            
+            # Upload chunk
+            headers = {
+                "Content-Length": str(len(chunk)),
+                "Content-Range": content_range
+            }
+            
+            # Try to upload chunk with retries
+            max_retries = 3
+            retry_delay = 2  # seconds
+            
+            for attempt in range(max_retries):
+                try:
+                    response = requests.put(
+                        upload_url,
+                        headers=headers,
+                        data=chunk
+                    )
+                    
+                    # Check if successful
+                    if response.status_code in (201, 202):
+                        # 201 = Created (final chunk), 202 = Accepted (more chunks coming)
+                        break
+                    else:
+                        logger.warning(f"Chunk upload failed (attempt {attempt+1}): {response.status_code}")
+                        if attempt < max_retries - 1:
+                            time.sleep(retry_delay * (attempt + 1))
+                except Exception as e:
+                    logger.warning(f"Chunk upload error (attempt {attempt+1}): {str(e)}")
+                    if attempt < max_retries - 1:
+                        time.sleep(retry_delay * (attempt + 1))
+            
+            if response.status_code not in (201, 202):
+                raise Exception(f"Failed to upload chunk after {max_retries} attempts: {response.status_code} - {response.text}")
+            
+            # Move to next chunk
+            chunk_number += 1
+            
+    # If we get here, all chunks were uploaded successfully
+    # The last response should contain the file metadata
+    return response.json()
+
+@celery.task(base=BaseTaskWithRetry)
+def upload_to_onedrive(file_path: str):
+    """Uploads a file to OneDrive in the configured folder."""
+    
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    # Extract filename
+    filename = os.path.basename(file_path)
+
+    # Check if OneDrive settings are configured
+    if not settings.onedrive_client_id:
+        error_msg = "OneDrive client ID is not configured"
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+
+    try:
+        # Get access token
+        access_token = get_onedrive_token()
+        
+        # Create upload session
+        upload_url = create_upload_session(filename, settings.onedrive_folder_path, access_token)
+        
+        # Upload the file
+        result = upload_large_file(file_path, upload_url)
+        
+        # Log success
+        web_url = result.get("webUrl", "Not available")
+        logger.info(f"Successfully uploaded {filename} to OneDrive at path {settings.onedrive_folder_path}")
+        logger.info(f"File accessible at: {web_url}")
+        
+        return {
+            "status": "Completed",
+            "file_path": file_path,
+            "onedrive_path": f"{settings.onedrive_folder_path}/{filename}",
+            "web_url": web_url
+        }
+        
+    except Exception as e:
+        error_msg = f"Failed to upload {filename} to OneDrive: {str(e)}"
+        logger.error(error_msg)
+        raise Exception(error_msg)

app/tasks/upload_to_paperless.py

@@ -82,31 +82,31 @@ def poll_task_for_document_id(task_id: str) -> int:
     )
 
 @celery.task(base=BaseTaskWithRetry)
-def upload_to_paperless(file_path: str) -> Dict[str, Any]:
-    """
-    Uploads a PDF to Paperless with minimal metadata (filename and date only).
-    
-    1. Extracts the filename and date from the file.
-    2. POSTs the PDF to Paperless => returns a quoted UUID string (task_id).
-    3. Polls /api/tasks/?task_id=<uuid> until SUCCESS or FAILURE => doc_id.
+def upload_to_paperless(file_path: str):
+    """Uploads a file to Paperless-ngx."""
 
-    Returns a dict with status, the paperless_task_id, paperless_document_id, and file_path.
-    """
     if not os.path.exists(file_path):
         raise FileNotFoundError(f"File not found: {file_path}")
 
-    base_name = os.path.basename(file_path)
+    # Extract filename
+    filename = os.path.basename(file_path)
+
+    # Check if Paperless settings are configured
+    if not settings.paperless_host or not settings.paperless_ngx_api_token:
+        error_msg = "Paperless-ngx credentials are not fully configured"
+        logger.error(error_msg)
+        raise ValueError(error_msg)
 
     # Upload the PDF
     post_url = _paperless_api_url("/api/documents/post_document/")
     with open(file_path, "rb") as f:
         files = {
-            "document": (base_name, f, "application/pdf"),
+            "document": (filename, f, "application/pdf"),
         }
-        data = {"title": base_name}  # Title = Filename (no additional metadata)
+        data = {"title": filename}  # Title = Filename (no additional metadata)
 
         try:
-            logger.debug("Posting document to Paperless: file=%s", base_name)
+            logger.debug("Posting document to Paperless: file=%s", filename)
             resp = requests.post(post_url, headers=_get_headers(), files=files, data=data)
             resp.raise_for_status()
         except requests.exceptions.RequestException as exc:

app/tasks/upload_to_s3.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+import os
+import logging
+import boto3
+from botocore.exceptions import ClientError
+from app.config import settings
+from app.tasks.retry_config import BaseTaskWithRetry
+from app.celery_app import celery
+
+logger = logging.getLogger(__name__)
+
+@celery.task(base=BaseTaskWithRetry)
+def upload_to_s3(file_path: str):
+    """Uploads a file to Amazon S3 in the configured bucket and folder."""
+
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    # Extract filename
+    filename = os.path.basename(file_path)
+
+    # Check if S3 settings are configured
+    if not settings.s3_bucket_name:
+        error_msg = "S3 bucket name is not configured"
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+
+    if not settings.aws_access_key_id or not settings.aws_secret_access_key:
+        error_msg = "AWS credentials are not configured"
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+
+    try:
+        # Create S3 client
+        s3_client = boto3.client(
+            's3',
+            region_name=settings.aws_region,
+            aws_access_key_id=settings.aws_access_key_id,
+            aws_secret_access_key=settings.aws_secret_access_key
+        )
+        
+        # Construct the S3 key (path within the bucket)
+        if settings.s3_folder_prefix:
+            # Ensure folder prefix ends with a slash
+            folder_prefix = settings.s3_folder_prefix
+            if not folder_prefix.endswith('/'):
+                folder_prefix += '/'
+            s3_key = f"{folder_prefix}{filename}"
+        else:
+            s3_key = filename
+        
+        # Prepare extra arguments
+        extra_args = {
+            'StorageClass': settings.s3_storage_class
+        }
+        
+        # Add ACL if configured
+        if settings.s3_acl:
+            extra_args['ACL'] = settings.s3_acl
+        
+        # Upload file
+        s3_client.upload_file(
+            file_path, 
+            settings.s3_bucket_name, 
+            s3_key,
+            ExtraArgs=extra_args
+        )
+        
+        # Generate URL to the file (useful for public files)
+        # For private files, this is just a reference and won't be accessible directly
+        s3_url = f"https://{settings.s3_bucket_name}.s3.{settings.aws_region}.amazonaws.com/{s3_key}"
+        
+        logger.info(f"Successfully uploaded {filename} to S3 bucket {settings.s3_bucket_name} at path {s3_key}")
+        return {
+            "status": "Completed",
+            "file": file_path,
+            "s3_bucket": settings.s3_bucket_name,
+            "s3_key": s3_key,
+            "s3_url": s3_url
+        }
+    
+    except ClientError as e:
+        error_msg = f"Failed to upload {filename} to S3: {str(e)}"
+        logger.error(error_msg)
+        raise Exception(error_msg)
+    
+    except Exception as e:
+        error_msg = f"Error uploading {filename} to S3: {str(e)}"
+        logger.error(error_msg)
+        raise Exception(error_msg)

app/tasks/upload_to_sftp.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+
+import os
+import logging
+import paramiko
+from pathlib import Path
+from app.config import settings
+from app.celery_app import celery
+from app.tasks.retry_config import BaseTaskWithRetry
+from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
+
+logger = logging.getLogger(__name__)
+
+@celery.task(base=BaseTaskWithRetry)
+def upload_to_sftp(file_path: str):
+    """
+    Upload a file to an SFTP server.
+    """
+    if not os.path.exists(file_path):
+        error_msg = f"File not found: {file_path}"
+        logger.error(error_msg)
+        raise FileNotFoundError(error_msg)
+    
+    if not (settings.sftp_host and settings.sftp_port and settings.sftp_username):
+        logger.info("SFTP upload skipped: Missing configuration")
+        return {"status": "Skipped", "reason": "SFTP settings not configured"}
+    
+    filename = os.path.basename(file_path)
+    sanitized_filename = sanitize_filename(filename)
+    
+    # SSH client for SFTP connection
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    
+    try:
+        # Setup connection parameters
+        connect_kwargs = {
+            "hostname": settings.sftp_host,
+            "port": settings.sftp_port,
+            "username": settings.sftp_username,
+        }
+        
+        # Check for authentication methods - use key if available, otherwise try password
+        sftp_key_path = getattr(settings, 'sftp_private_key', None)
+        sftp_key_passphrase = getattr(settings, 'sftp_private_key_passphrase', None)
+        
+        if sftp_key_path and os.path.exists(sftp_key_path):
+            logger.info(f"Using SSH key authentication with key: {sftp_key_path}")
+            connect_kwargs["key_filename"] = sftp_key_path
+            if sftp_key_passphrase:
+                connect_kwargs["passphrase"] = sftp_key_passphrase
+        elif settings.sftp_password:
+            logger.info("Using password authentication for SFTP")
+            connect_kwargs["password"] = settings.sftp_password
+        else:
+            error_msg = "No authentication method available for SFTP (no key or password)"
+            logger.error(error_msg)
+            raise Exception(error_msg)
+        
+        # Connect to the server
+        logger.info(f"Connecting to SFTP server at {settings.sftp_host}:{settings.sftp_port}")
+        ssh.connect(**connect_kwargs)
+        
+        # Open SFTP session
+        sftp = ssh.open_sftp()
+        
+        # Calculate remote path based on local file structure
+        remote_base = settings.sftp_folder or ""
+        remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
+        
+        # Function to check if file exists in SFTP server
+        def check_exists_in_sftp(path):
+            try:
+                sftp.stat(path)
+                return True
+            except FileNotFoundError:
+                return False
+        
+        # Check for potential file collision and get a unique name if needed
+        remote_path = get_unique_filename(remote_path, check_exists_in_sftp)
+        
+        # Create parent directories if needed
+        remote_dir = os.path.dirname(remote_path)
+        if remote_dir:
+            try:
+                # Try to create the full directory path
+                current_dir = ""
+                for dir_part in remote_dir.split("/"):
+                    if not dir_part:
+                        continue
+                    current_dir += f"/{dir_part}"
+                    try:
+                        sftp.stat(current_dir)
+                    except FileNotFoundError:
+                        logger.info(f"Creating directory on SFTP server: {current_dir}")
+                        sftp.mkdir(current_dir)
+            except Exception as e:
+                logger.warning(f"Failed to create directory structure {remote_dir}: {str(e)}")
+        
+        # Upload the file
+        logger.info(f"Uploading {filename} to SFTP at {remote_path}")
+        sftp.put(file_path, remote_path)
+        logger.info(f"Successfully uploaded {filename} to SFTP at {remote_path}")
+        
+        # Close connections
+        sftp.close()
+        ssh.close()
+        
+        return {
+            "status": "Completed",
+            "file_path": file_path,
+            "sftp_path": remote_path
+        }
+    
+    except Exception as e:
+        # Make sure connections are closed
+        try:
+            if 'sftp' in locals():
+                sftp.close()
+            ssh.close()
+        except:
+            pass
+            
+        error_msg = f"Failed to upload {filename} to SFTP server: {str(e)}"
+        logger.error(error_msg)
+        raise Exception(error_msg)

app/tasks/upload_to_webdav.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+import os
+import requests
+from urllib.parse import urljoin
+from app.config import settings
+from app.tasks.retry_config import BaseTaskWithRetry
+from app.celery_app import celery
+import logging
+
+logger = logging.getLogger(__name__)
+
+@celery.task(base=BaseTaskWithRetry)
+def upload_to_webdav(file_path: str):
+    """Uploads a file to a WebDAV server in the configured folder."""
+
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    # Extract filename
+    filename = os.path.basename(file_path)
+
+    # Check if WebDAV settings are configured
+    if not settings.webdav_url:
+        error_msg = "WebDAV URL is not configured"
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+
+    # Construct the full upload URL
+    webdav_folder = settings.webdav_folder or ""
+    # Ensure folder doesn't have leading slash if we're joining it to the base URL
+    if webdav_folder and webdav_folder.startswith("/"):
+        webdav_folder = webdav_folder[1:]
+    
+    # Join the base URL and folder path
+    target_url = urljoin(settings.webdav_url, webdav_folder)
+    # Ensure URL ends with a slash for proper joining with filename
+    if not target_url.endswith("/"):
+        target_url += "/"
+    
+    # Construct final URL with filename
+    webdav_url = urljoin(target_url, filename)
+
+    # Read file content
+    try:
+        with open(file_path, "rb") as file_data:
+            response = requests.put(
+                webdav_url,
+                auth=(settings.webdav_username, settings.webdav_password),
+                data=file_data,
+                verify=settings.webdav_verify_ssl if hasattr(settings, "webdav_verify_ssl") else True
+            )
+
+        # Check if upload was successful
+        if response.status_code in (200, 201, 204):
+            logger.info(f"Successfully uploaded {filename} to WebDAV at {webdav_url}.")
+            return {"status": "Completed", "file": file_path, "url": webdav_url}
+        else:
+            error_msg = f"Failed to upload {filename} to WebDAV: {response.status_code} - {response.text}"
+            logger.error(error_msg)
+            raise Exception(error_msg)
+    
+    except Exception as e:
+        error_msg = f"Error uploading {filename} to WebDAV: {str(e)}"
+        logger.error(error_msg)
+        raise Exception(error_msg)

app/tasks/upload_with_rclone.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import json
+import tempfile
+import logging
+from pathlib import Path
+from app.config import settings
+from app.tasks.retry_config import BaseTaskWithRetry
+from app.celery_app import celery
+
+logger = logging.getLogger(__name__)
+
+@celery.task(base=BaseTaskWithRetry)
+def upload_with_rclone(file_path: str, destination: str):
+    """
+    Uploads a file using rclone to the specified destination.
+    
+    Args:
+        file_path: Path to the file to upload
+        destination: Rclone destination in format "remote:path/to/folder"
+                     e.g. "gdrive:Uploads" or "dropbox:Documents/Uploads"
+    """
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+    
+    # Extract filename
+    filename = os.path.basename(file_path)
+    
+    # Check if rclone is installed and config exists
+    rclone_config_path = os.path.join(settings.workdir, "rclone.conf")
+    if not os.path.exists(rclone_config_path):
+        error_msg = f"Rclone configuration not found at {rclone_config_path}"
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+    
+    try:
+        # Split destination into remote and path
+        if ":" not in destination:
+            raise ValueError(f"Invalid destination format: {destination}. Expected format: remote:path")
+        
+        remote, remote_path = destination.split(":", 1)
+        
+        # Ensure the remote path exists (create folders if needed)
+        mkdir_cmd = [
+            "rclone", 
+            "mkdir", 
+            "--config", rclone_config_path,
+            destination
+        ]
+        
+        subprocess.run(mkdir_cmd, check=True, capture_output=True)
+        
+        # Construct the upload command
+        upload_cmd = [
+            "rclone", 
+            "copy",
+            "--config", rclone_config_path,
+            file_path,
+            destination,
+            "--progress"
+        ]
+        
+        # Execute the upload command
+        result = subprocess.run(upload_cmd, check=True, capture_output=True, text=True)
+        
+        # Check if upload was successful
+        if result.returncode == 0:
+            # Try to get a public link if possible
+            try:
+                link_cmd = [
+                    "rclone",
+                    "link",
+                    "--config", rclone_config_path,
+                    f"{destination}/{filename}"
+                ]
+                link_result = subprocess.run(link_cmd, capture_output=True, text=True)
+                public_url = link_result.stdout.strip() if link_result.returncode == 0 else None
+            except Exception:
+                public_url = None
+            
+            logger.info(f"Successfully uploaded {filename} to {destination}")
+            return {
+                "status": "Completed",
+                "file": file_path,
+                "destination": destination,
+                "public_url": public_url
+            }
+        else:
+            error_msg = f"Failed to upload {filename} to {destination}: {result.stderr}"
+            logger.error(error_msg)
+            raise Exception(error_msg)
+            
+    except subprocess.CalledProcessError as e:
+        error_msg = f"Rclone error: {e.stderr.decode('utf-8') if hasattr(e.stderr, 'decode') else e.stderr}"
+        logger.error(error_msg)
+        raise Exception(error_msg)
+    
+    except Exception as e:
+        error_msg = f"Error uploading {filename} to {destination}: {str(e)}"
+        logger.error(error_msg)
+        raise Exception(error_msg)
+
+
+@celery.task(base=BaseTaskWithRetry)
+def send_to_all_rclone_destinations(file_path: str):
+    """
+    Uploads a file to all configured rclone destinations.
+    Destinations are loaded from the rclone configuration file.
+    """
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    # Extract filename
+    filename = os.path.basename(file_path)
+    
+    # Path to rclone config
+    rclone_config_path = os.path.join(settings.workdir, "rclone.conf")
+    if not os.path.exists(rclone_config_path):
+        error_msg = f"Rclone configuration not found at {rclone_config_path}"
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+    
+    # Get list of configured destinations from rclone
+    try:
+        remotes_cmd = ["rclone", "listremotes", "--config", rclone_config_path]
+        result = subprocess.run(remotes_cmd, check=True, capture_output=True, text=True)
+        
+        if result.returncode == 0:
+            # Process the list of remotes
+            remotes = [r.strip() for r in result.stdout.splitlines() if r.strip()]
+            
+            # Target directories for each remote (from settings)
+            remote_paths = {}
+            for remote in remotes:
+                remote_name = remote.rstrip(':')
+                path_setting_name = f"rclone_{remote_name}_path"
+                if hasattr(settings, path_setting_name) and getattr(settings, path_setting_name):
+                    remote_paths[remote] = getattr(settings, path_setting_name)
+                else:
+                    # Default to root of remote if not specified
+                    remote_paths[remote] = ""
+            
+            # Queue upload tasks for each configured destination
+            results = {}
+            for remote, path in remote_paths.items():
+                full_destination = f"{remote}{path}"
+                if path and not path.endswith('/'):
+                    full_destination += '/'
+                
+                logger.info(f"Queueing {file_path} for upload to {full_destination}")
+                task = upload_with_rclone.delay(file_path, full_destination)
+                results[f"rclone_{remote.rstrip(':')}_task_id"] = task.id
+            
+            return {
+                "status": "Queued",
+                "file_path": file_path,
+                "tasks": results
+            }
+        else:
+            error_msg = f"Failed to list rclone remotes: {result.stderr}"
+            logger.error(error_msg)
+            raise Exception(error_msg)
+            
+    except Exception as e:
+        error_msg = f"Error setting up rclone uploads for {filename}: {str(e)}"
+        logger.error(error_msg)
+        raise Exception(error_msg)

app/templates/email/.gitkeep

@@ -0,0 +1,2 @@
+# This directory contains email templates
+# The .gitkeep file ensures the directory is included in git

app/templates/email/default.html

@@ -0,0 +1,161 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Document from DocuNova</title>
+    <style>
+        body {
+            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+            line-height: 1.6;
+            color: #333;
+            max-width: 600px;
+            margin: 0 auto;
+            padding: 20px;
+        }
+        .header {
+            text-align: center;
+            margin-bottom: 30px;
+        }
+        .logo {
+            max-width: 200px;
+            margin-bottom: 15px;
+        }
+        .container {
+            background-color: #f9f9f9;
+            border-radius: 8px;
+            padding: 25px;
+            border: 1px solid #e0e0e0;
+        }
+        .footer {
+            margin-top: 30px;
+            font-size: 12px;
+            color: #666;
+            text-align: center;
+            border-top: 1px solid #eee;
+            padding-top: 20px;
+        }
+        .button {
+            display: inline-block;
+            background-color: #3182ce;
+            color: white !important;
+            padding: 10px 20px;
+            text-decoration: none;
+            border-radius: 4px;
+            margin-top: 15px;
+            text-align: center;
+        }
+        .message {
+            margin-bottom: 20px;
+        }
+        .attachment-info {
+            background-color: #ebf8ff;
+            border-left: 4px solid #3182ce;
+            padding: 10px 15px;
+            margin-bottom: 20px;
+        }
+        .metadata-section {
+            background-color: #f0fff4;
+            border-left: 4px solid #38a169;
+            padding: 10px 15px;
+            margin: 20px 0;
+        }
+        .metadata-title {
+            font-weight: bold;
+            margin-bottom: 10px;
+            color: #2f855a;
+        }
+        .metadata-table {
+            width: 100%;
+            border-collapse: collapse;
+        }
+        .metadata-table th, .metadata-table td {
+            text-align: left;
+            padding: 8px;
+            border-bottom: 1px solid #ddd;
+        }
+        .metadata-table th {
+            width: 40%;
+            color: #4a5568;
+        }
+        .metadata-value {
+            word-break: break-word;
+        }
+        .document-type {
+            display: inline-block;
+            background-color: #3182ce;
+            color: white;
+            padding: 4px 10px;
+            border-radius: 12px;
+            font-size: 0.85em;
+            margin-top: 5px;
+        }
+    </style>
+</head>
+<body>
+    <div class="header">
+        {% if has_logo %}
+        <img src="cid:logo" alt="DocuNova Logo" class="logo">
+        {% endif %}
+        <h1>Document Delivery</h1>
+    </div>
+    
+    <div class="container">
+        <p class="message">
+            {% if custom_message %}
+                {{ custom_message }}
+            {% else %}
+                A document has been shared with you from DocuNova.
+            {% endif %}
+        </p>
+        
+        <div class="attachment-info">
+            <strong>Attached Document:</strong> {{ filename }}
+        </div>
+        
+        {% if has_metadata %}
+        <div class="metadata-section">
+            <div class="metadata-title">Document Metadata</div>
+            
+            {% if metadata.document_type %}
+            <span class="document-type">{{ metadata.document_type }}</span>
+            {% endif %}
+            
+            <table class="metadata-table">
+                {% for key, value in metadata.items() %}
+                    {% if key != "document_type" and value %}
+                    <tr>
+                        <th>{{ key | replace("_", " ") | title }}</th>
+                        <td class="metadata-value">
+                            {% if value is mapping %}
+                                {% for subkey, subvalue in value.items() %}
+                                    <strong>{{ subkey | replace("_", " ") | title }}:</strong> {{ subvalue }}<br>
+                                {% endfor %}
+                            {% elif value is iterable and value is not string %}
+                                {{ value | join(", ") }}
+                            {% else %}
+                                {{ value }}
+                            {% endif %}
+                        </td>
+                    </tr>
+                    {% endif %}
+                {% endfor %}
+            </table>
+        </div>
+        {% endif %}
+        
+        <p>You can find the attached document in this email. Please let us know if you have any questions.</p>
+        
+        {% if app_url %}
+        <p>
+            <a href="{{ app_url }}" class="button">Visit DocuNova</a>
+        </p>
+        {% endif %}
+    </div>
+    
+    <div class="footer">
+        <p>This is an automated message from DocuNova Document Processing System.</p>
+        <p>© {{ now().year }} DocuNova</p>
+    </div>
+</body>
+</html>

app/utils.py

@@ -1,35 +1,7 @@
 # app/utils.py
-import hashlib
-from app.database import SessionLocal
-from app.models import ProcessingLog
+# This file is deprecated. Functions have been moved to the utils package.
+# To avoid breaking existing imports, we'll import and re-export the functions
+from app.utils.file_operations import hash_file
+from app.utils.logging import log_task_progress
 
-def hash_file(filepath, chunk_size=65536):
-    """
-    Returns the SHA-256 hash of the file at 'filepath'.
-    Reads the file in chunks to handle large files efficiently.
-    """
-    sha256 = hashlib.sha256()
-    with open(filepath, "rb") as f:
-        while True:
-            data = f.read(chunk_size)
-            if not data:
-                break
-            sha256.update(data)
-    return sha256.hexdigest()
-
-
-
-def log_task_progress(task_id, step_name, status, message=None, file_id=None):
-    """
-    Logs the progress of a Celery task to the database.
-    """
-    with SessionLocal() as db:
-        log_entry = ProcessingLog(
-            task_id=task_id,
-            step_name=step_name,
-            status=status,
-            message=message,
-            file_id=file_id,
-        )
-        db.add(log_entry)
-        db.commit()
+# These functions are now available directly from the app.utils package

app/utils/__init__.py

@@ -0,0 +1,10 @@
+"""
+Utility functions and helpers for the document processor application.
+"""
+
+# Import functions to make them available through the package
+from app.utils.file_operations import hash_file
+from app.utils.logging import log_task_progress
+
+# Export all the functions that should be available when importing from app.utils
+__all__ = ['hash_file', 'log_task_progress']

app/utils/config_validator.py

@@ -0,0 +1,324 @@
+#!/usr/bin/env python3
+
+import os
+import socket
+import logging
+import inspect
+from app.config import settings
+
+logger = logging.getLogger(__name__)
+
+def validate_email_config():
+    """Validates email configuration settings"""
+    issues = []
+    
+    # Check for required email settings
+    if not getattr(settings, 'email_host', None):
+        issues.append("EMAIL_HOST is not configured")
+    if not getattr(settings, 'email_port', None):
+        issues.append("EMAIL_PORT is not configured")
+    
+    # Test SMTP server connectivity if host is configured
+    if getattr(settings, 'email_host', None) and getattr(settings, 'email_port', None):
+        try:
+            # Attempt to resolve the hostname
+            socket.gethostbyname(settings.email_host)
+        except socket.gaierror:
+            issues.append(f"Cannot resolve email host: {settings.email_host}")
+
+    # Check for authentication settings
+    if not getattr(settings, 'email_username', None):
+        issues.append("EMAIL_USERNAME is not configured")
+    if not getattr(settings, 'email_password', None):
+        issues.append("EMAIL_PASSWORD is not configured")
+    
+    return issues
+
+def validate_storage_configs():
+    """Validates configuration for all storage providers"""
+    issues = {}
+    
+    # Validate Dropbox config
+    dropbox_issues = []
+    if not (getattr(settings, 'dropbox_app_key', None) and 
+            getattr(settings, 'dropbox_app_secret', None) and 
+            getattr(settings, 'dropbox_refresh_token', None)):
+        dropbox_issues.append("Dropbox credentials are not fully configured")
+    issues['dropbox'] = dropbox_issues
+    
+    # Validate Nextcloud config
+    nextcloud_issues = []
+    if not (getattr(settings, 'nextcloud_upload_url', None) and 
+            getattr(settings, 'nextcloud_username', None) and 
+            getattr(settings, 'nextcloud_password', None)):
+        nextcloud_issues.append("Nextcloud credentials are not fully configured")
+    issues['nextcloud'] = nextcloud_issues
+    
+    # Validate SFTP config
+    sftp_issues = []
+    if not getattr(settings, 'sftp_host', None):
+        sftp_issues.append("SFTP_HOST is not configured")
+    
+    sftp_key_path = getattr(settings, 'sftp_private_key', None)
+    if sftp_key_path and not os.path.exists(sftp_key_path):
+        sftp_issues.append(f"SFTP_KEY_PATH file not found: {sftp_key_path}")
+    
+    if not sftp_key_path and not getattr(settings, 'sftp_password', None):
+        sftp_issues.append("Neither SFTP_KEY_PATH nor SFTP_PASSWORD is configured")
+    
+    issues['sftp'] = sftp_issues
+    
+    # Validate Email sending
+    email_issues = []
+    if not getattr(settings, 'email_host', None):
+        email_issues.append("EMAIL_HOST is not configured")
+    if not getattr(settings, 'email_default_recipient', None):
+        email_issues.append("EMAIL_DEFAULT_RECIPIENT is not configured")
+    issues['email'] = email_issues
+    
+    # Validate S3
+    s3_issues = []
+    if not getattr(settings, 's3_bucket_name', None):
+        s3_issues.append("S3_BUCKET_NAME is not configured")
+    if not (getattr(settings, 'aws_access_key_id', None) and 
+            getattr(settings, 'aws_secret_access_key', None)):
+        s3_issues.append("AWS credentials are not configured")
+    issues['s3'] = s3_issues
+    
+    # Validate FTP
+    ftp_issues = []
+    if not getattr(settings, 'ftp_host', None):
+        ftp_issues.append("FTP_HOST is not configured")
+    if not getattr(settings, 'ftp_username', None):
+        ftp_issues.append("FTP_USERNAME is not configured")
+    if not getattr(settings, 'ftp_password', None):
+        ftp_issues.append("FTP_PASSWORD is not configured")
+    issues['ftp'] = ftp_issues
+    
+    # Validate WebDAV
+    webdav_issues = []
+    if not getattr(settings, 'webdav_url', None):
+        webdav_issues.append("WEBDAV_URL is not configured")
+    if not getattr(settings, 'webdav_username', None):
+        webdav_issues.append("WEBDAV_USERNAME is not configured")
+    if not getattr(settings, 'webdav_password', None):
+        webdav_issues.append("WEBDAV_PASSWORD is not configured")
+    issues['webdav'] = webdav_issues
+    
+    # Validate Google Drive
+    gdrive_issues = []
+    if not getattr(settings, 'google_drive_credentials_json', None):
+        gdrive_issues.append("GOOGLE_DRIVE_CREDENTIALS_JSON is not configured")
+    if not getattr(settings, 'google_drive_folder_id', None):
+        gdrive_issues.append("GOOGLE_DRIVE_FOLDER_ID is not configured")
+    issues['google_drive'] = gdrive_issues
+    
+    # Validate Paperless
+    paperless_issues = []
+    if not getattr(settings, 'paperless_host', None):
+        paperless_issues.append("PAPERLESS_HOST is not configured")
+    if not getattr(settings, 'paperless_ngx_api_token', None):
+        paperless_issues.append("PAPERLESS_NGX_API_TOKEN is not configured")
+    issues['paperless'] = paperless_issues
+    
+    # Validate OneDrive
+    onedrive_issues = []
+    if not (getattr(settings, 'onedrive_client_id', None) and 
+            getattr(settings, 'onedrive_client_secret', None) and
+            getattr(settings, 'onedrive_refresh_token', None)):
+        onedrive_issues.append("OneDrive credentials are not fully configured")
+    issues['onedrive'] = onedrive_issues
+    
+    return issues
+
+def get_provider_status():
+    """Get the status of each provider for the dashboard"""
+    providers = {
+        "Email": {
+            "configured": bool(getattr(settings, 'email_host', None) and 
+                               getattr(settings, 'email_username', None) and
+                               getattr(settings, 'email_password', None)),
+            "icon": "mail",
+            "url": getattr(settings, 'email_host', None) or "",
+            "description": f"Send to {getattr(settings, 'email_default_recipient', 'Not configured')}"
+        },
+        "Dropbox": {
+            "configured": bool(getattr(settings, 'dropbox_app_key', None) and 
+                               getattr(settings, 'dropbox_app_secret', None) and
+                               getattr(settings, 'dropbox_refresh_token', None)),
+            "icon": "dropbox",
+            "url": "https://dropbox.com",
+            "description": f"Upload to folder: {getattr(settings, 'dropbox_folder', 'Root')}"
+        },
+        "Nextcloud": {
+            "configured": bool(getattr(settings, 'nextcloud_upload_url', None) and
+                               getattr(settings, 'nextcloud_username', None)),
+            "icon": "cloud",
+            "url": getattr(settings, 'nextcloud_upload_url', "").split('/remote.php')[0] if getattr(settings, 'nextcloud_upload_url', None) else "",
+            "description": f"Upload to folder: {getattr(settings, 'nextcloud_folder', 'Root')}"
+        },
+        "SFTP": {
+            "configured": bool(getattr(settings, 'sftp_host', None) and
+                               getattr(settings, 'sftp_username', None) and
+                               (getattr(settings, 'sftp_password', None) or getattr(settings, 'sftp_private_key', None))),
+            "icon": "server",
+            "url": f"sftp://{getattr(settings, 'sftp_host', '')}:{getattr(settings, 'sftp_port', 22)}",
+            "description": f"Upload to {getattr(settings, 'sftp_host', 'Not configured')}:{getattr(settings, 'sftp_folder', '/')}"
+        },
+        "Paperless": {
+            "configured": bool(getattr(settings, 'paperless_host', None) and
+                               getattr(settings, 'paperless_ngx_api_token', None)),
+            "icon": "file-text",
+            "url": getattr(settings, 'paperless_host', ""),
+            "description": "Document management system"
+        },
+        "S3": {
+            "configured": bool(getattr(settings, 's3_bucket_name', None) and
+                              getattr(settings, 'aws_access_key_id', None)),
+            "icon": "database",
+            "url": f"https://s3.console.aws.amazon.com/s3/buckets/{getattr(settings, 's3_bucket_name', '')}",
+            "description": f"Bucket: {getattr(settings, 's3_bucket_name', 'Not configured')}"
+        },
+        "FTP": {
+            "configured": bool(getattr(settings, 'ftp_host', None) and
+                              getattr(settings, 'ftp_username', None)),
+            "icon": "hard-drive",
+            "url": f"ftp://{getattr(settings, 'ftp_host', '')}:{getattr(settings, 'ftp_port', 21)}",
+            "description": f"Upload to {getattr(settings, 'ftp_host', 'Not configured')}:{getattr(settings, 'ftp_folder', '/')}"
+        },
+        "WebDAV": {
+            "configured": bool(getattr(settings, 'webdav_url', None) and
+                              getattr(settings, 'webdav_username', None)),
+            "icon": "globe",
+            "url": getattr(settings, 'webdav_url', ""),
+            "description": f"Upload to {getattr(settings, 'webdav_folder', '/')}"
+        },
+        "Google Drive": {
+            "configured": bool(getattr(settings, 'google_drive_credentials_json', None)),
+            "icon": "google",
+            "url": "https://drive.google.com",
+            "description": f"Folder ID: {getattr(settings, 'google_drive_folder_id', 'Not configured')}"
+        },
+        "OneDrive": {
+            "configured": bool(getattr(settings, 'onedrive_client_id', None) and
+                               getattr(settings, 'onedrive_refresh_token', None)),
+            "icon": "microsoft",
+            "url": "https://onedrive.live.com",
+            "description": f"Upload to folder: {getattr(settings, 'onedrive_folder_path', 'Not configured')}"
+        }
+    }
+    
+    return providers
+
+def dump_all_settings():
+    """Dump all settings to the log for debugging"""
+    logger.info("================ SETTINGS DUMP ================")
+    
+    # Get all attributes from settings object
+    attributes = inspect.getmembers(settings, lambda a: not inspect.isroutine(a))
+    settings_dict = {a[0]: a[1] for a in attributes 
+                    if not a[0].startswith('_') and not callable(a[1])}
+    
+    # Sort keys for better readability
+    for key in sorted(settings_dict.keys()):
+        value = settings_dict[key]
+        # Hide sensitive values
+        if any(sensitive in key.lower() for sensitive in ['password', 'secret', 'token', 'key']):
+            if value:
+                value = "******** [HIDDEN FOR SECURITY]"
+        logger.info(f"  {key} = {value}")
+    
+    # Also log all environment variables
+    logger.info("----------- ENVIRONMENT VARIABLES -----------")
+    env_vars_to_log = {}
+    for key in sorted(os.environ.keys()):
+        value = os.environ[key]
+        # Hide sensitive values
+        if any(sensitive in key.lower() for sensitive in ['password', 'secret', 'token', 'key']):
+            if value:
+                value = "******** [HIDDEN FOR SECURITY]"
+        env_vars_to_log[key] = value
+    
+    for key in sorted(env_vars_to_log.keys()):
+        logger.info(f"  {key} = {env_vars_to_log[key]}")
+    
+    logger.info("=============================================")
+
+def get_settings_for_display(show_values=False):
+    """Get all settings organized by category for display in UI"""
+    # Get all attributes from settings object
+    attributes = inspect.getmembers(settings, lambda a: not inspect.isroutine(a))
+    settings_dict = {a[0]: a[1] for a in attributes 
+                    if not a[0].startswith('_') and not callable(a[1])}
+    
+    # Categorize settings
+    categories = {
+        "Core": [],
+        "Email": [],
+        "IMAP": [],
+        "Storage": [],
+        "Authentication": [],
+        "Integration": [],
+        "Other": []
+    }
+    
+    # Sort keys for better readability
+    for key in sorted(settings_dict.keys()):
+        value = settings_dict[key]
+        # Mask sensitive values if show_values is False
+        display_value = value
+        if not show_values or any(sensitive in key.lower() for sensitive in ['password', 'secret', 'token', 'key']):
+            if value:
+                display_value = "******** [HIDDEN]"
+            else:
+                display_value = None
+        
+        # Categorize by key prefix
+        setting_item = {"name": key, "value": display_value, "is_configured": value is not None and value != ""}
+        
+        if key.startswith(('email_', 'smtp_')):
+            categories["Email"].append(setting_item)
+        elif key.startswith('imap'):
+            categories["IMAP"].append(setting_item)
+        elif key.startswith(('s3_', 'aws_', 'dropbox_', 'nextcloud_', 'sftp_', 'ftp_', 'google_drive_')):
+            categories["Storage"].append(setting_item)
+        elif key.startswith(('auth_', 'jwt_', 'oauth_')):
+            categories["Authentication"].append(setting_item)
+        elif key.startswith(('paperless_', 'tesseract_', 'azure_')):
+            categories["Integration"].append(setting_item)
+        elif key in ('workdir', 'external_hostname', 'debug', 'version', 'env', 'log_level'):
+            categories["Core"].append(setting_item)
+        else:
+            categories["Other"].append(setting_item)
+    
+    # Remove empty categories
+    return {k: v for k, v in categories.items() if v}
+
+def check_all_configs():
+    """Run all configuration validations and log results"""
+    logger.info("Validating application configuration...")
+    
+    # Check if debug is enabled and dump all settings if it is
+    if hasattr(settings, 'debug') and settings.debug:
+        dump_all_settings()
+    
+    # Check email config
+    email_issues = validate_email_config()
+    if email_issues:
+        logger.warning(f"Email configuration issues: {', '.join(email_issues)}")
+    else:
+        logger.info("Email configuration OK")
+    
+    # Check storage configs
+    storage_issues = validate_storage_configs()
+    for provider, issues in storage_issues.items():
+        if issues:
+            logger.warning(f"{provider.capitalize()} configuration issues: {', '.join(issues)}")
+        else:
+            logger.info(f"{provider.capitalize()} configuration OK")
+    
+    # Return all identified issues
+    return {
+        'email': email_issues,
+        'storage': storage_issues
+    }

app/utils/file_operations.py

@@ -0,0 +1,15 @@
+import hashlib
+
+def hash_file(filepath, chunk_size=65536):
+    """
+    Returns the SHA-256 hash of the file at 'filepath'.
+    Reads the file in chunks to handle large files efficiently.
+    """
+    sha256 = hashlib.sha256()
+    with open(filepath, "rb") as f:
+        while True:
+            data = f.read(chunk_size)
+            if not data:
+                break
+            sha256.update(data)
+    return sha256.hexdigest()

app/utils/filename_utils.py

@@ -0,0 +1,131 @@
+import os
+import re
+import uuid
+import logging
+from datetime import datetime
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+def get_unique_filename(original_path, check_exists_func=None):
+    """
+    Generates a unique filename by appending a timestamp or counter when a collision occurs.
+    
+    Args:
+        original_path (str): The original file path
+        check_exists_func (callable): Function that checks if file exists in target system.
+                                     Takes a path string and returns True if exists, False otherwise.
+                                     If None, will use local filesystem check.
+    
+    Returns:
+        str: A unique filename that doesn't collide with existing files
+    """
+    if check_exists_func is None:
+        check_exists_func = os.path.exists
+    
+    path = Path(original_path)
+    directory = str(path.parent)
+    filename = path.name
+    name, ext = os.path.splitext(filename)
+    
+    # If file doesn't exist, return the original
+    if not check_exists_func(original_path):
+        return original_path
+    
+    # Try timestamp-based suffix first (more user-friendly)
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    new_filename = f"{name}_{timestamp}{ext}"
+    new_path = os.path.join(directory, new_filename)
+    
+    if not check_exists_func(new_path):
+        logger.info(f"Renamed '{filename}' to '{new_filename}' to avoid collision")
+        return new_path
+    
+    # If timestamp-based name also exists, try random UUID
+    uuid_str = str(uuid.uuid4())[:8]  # Use first 8 chars of UUID for brevity
+    new_filename = f"{name}_{uuid_str}{ext}"
+    new_path = os.path.join(directory, new_filename)
+    
+    if not check_exists_func(new_path):
+        logger.info(f"Renamed '{filename}' to '{new_filename}' using UUID to avoid collision")
+        return new_path
+    
+    # If that still exists (very unlikely), use incremental numbering
+    counter = 1
+    while counter < 1000:  # Limit to avoid infinite loop
+        new_filename = f"{name}_{counter}{ext}"
+        new_path = os.path.join(directory, new_filename)
+        if not check_exists_func(new_path):
+            logger.info(f"Renamed '{filename}' to '{new_filename}' using counter to avoid collision")
+            return new_path
+        counter += 1
+    
+    # If we got here, something is weird - just use a full UUID
+    new_filename = f"{name}_{str(uuid.uuid4())}{ext}"
+    new_path = os.path.join(directory, new_filename)
+    logger.warning(f"Had to use full UUID to rename '{filename}' to '{new_filename}'")
+    
+    return new_path
+
+def sanitize_filename(filename):
+    """
+    Sanitize a filename to ensure it's valid across different file systems.
+    
+    Args:
+        filename (str): The filename to sanitize
+        
+    Returns:
+        str: A sanitized filename
+    """
+    # Replace characters that are problematic in various filesystems
+    # Keep only alphanumeric, dash, underscore, period, and space
+    sanitized = re.sub(r'[^\w\-\. ]', '_', filename)
+    
+    # Replace multiple spaces/underscores with single ones
+    sanitized = re.sub(r'__+', '_', sanitized)
+    sanitized = re.sub(r'  +', ' ', sanitized)
+    
+    # Trim leading/trailing spaces and periods which cause issues in Windows
+    sanitized = sanitized.strip('. ')
+    
+    # Ensure the filename isn't empty after sanitization
+    if not sanitized or sanitized == '.':
+        sanitized = f"document_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+    
+    return sanitized
+
+def extract_remote_path(local_path, base_dir, remote_base=None):
+    """
+    Extracts the appropriate remote path based on a local path structure.
+    
+    Args:
+        local_path (str): The local file path
+        base_dir (str): The local base directory to remove from path
+        remote_base (str, optional): Remote base directory to prepend
+        
+    Returns:
+        str: The calculated remote path
+    """
+    # Convert both paths to use forward slashes for consistency
+    local_path = local_path.replace('\\', '/')
+    base_dir = base_dir.replace('\\', '/')
+    
+    # Make sure base_dir ends with a slash
+    if not base_dir.endswith('/'):
+        base_dir += '/'
+    
+    # Remove the base directory from the local path
+    if local_path.startswith(base_dir):
+        relative_path = local_path[len(base_dir):]
+    else:
+        # If local_path is not within base_dir, just use the filename
+        relative_path = os.path.basename(local_path)
+    
+    # Prepend the remote base if provided
+    if remote_base:
+        # Ensure remote_base ends with slash
+        if not remote_base.endswith('/'):
+            remote_base += '/'
+        return remote_base + relative_path
+    
+    return relative_path

app/utils/logging.py

@@ -0,0 +1,17 @@
+from app.database import SessionLocal
+from app.models import ProcessingLog
+
+def log_task_progress(task_id, step_name, status, message=None, file_id=None):
+    """
+    Logs the progress of a Celery task to the database.
+    """
+    with SessionLocal() as db:
+        log_entry = ProcessingLog(
+            task_id=task_id,
+            step_name=step_name,
+            status=status,
+            message=message,
+            file_id=file_id,
+        )
+        db.add(log_entry)
+        db.commit()