Merge pull request #426 from christianlouis/copilot/support-directory-upload

feat(upload): directory drag-and-drop, adaptive 429 throttling, full Gotenberg file type support
2026-02-27 00:56:17 +01:00
parent 8dedd40ef3 4a251fd3ca
commit 585b30a93e
15 changed files with 900 additions and 278 deletions
@@ -18,6 +18,7 @@ from app.database import get_db
 from app.models import FileRecord, ProcessingLog
 from app.tasks.convert_to_pdf import convert_to_pdf
 from app.tasks.process_document import process_document
+from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, IMAGE_MIME_TYPES
 from app.utils.file_queries import apply_status_filter
 from app.utils.file_status import get_files_processing_status
 from app.utils.filename_utils import sanitize_filename
@@ -1034,33 +1035,6 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
    logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")
    file_size = written_size

-    # Same set of allowed file types as in the IMAP task
-    ALLOWED_MIME_TYPES = {
-        "application/pdf",
-        "application/msword",
-        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-        "application/vnd.ms-excel",
-        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-        "application/vnd.ms-powerpoint",
-        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-        "text/plain",
-        "text/csv",
-        "application/rtf",
-        "text/rtf",
-    }
-
-    # Image MIME types that need conversion
-    IMAGE_MIME_TYPES = {
-        "image/jpeg",
-        "image/jpg",
-        "image/png",
-        "image/gif",
-        "image/bmp",
-        "image/tiff",
-        "image/webp",
-        "image/svg+xml",
-    }
-
    # Determine if the file is a PDF or needs conversion
    mime_type, _ = mimetypes.guess_type(target_path)
    file_ext = os.path.splitext(target_path)[1].lower()
@@ -1114,32 +1088,24 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
        # If it's a PDF, process directly
        task = process_document.delay(target_path, original_filename=safe_filename)
        logger.info(f"Enqueued PDF for processing: {target_path}")
-    elif mime_type in IMAGE_MIME_TYPES or any(
-        file_ext.endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"]
-    ):
+    elif mime_type in IMAGE_MIME_TYPES or file_ext in {
+        ".jpg",
+        ".jpeg",
+        ".png",
+        ".gif",
+        ".bmp",
+        ".tiff",
+        ".tif",
+        ".webp",
+        ".svg",
+    }:
        # If it's an image, convert to PDF first
        task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
        logger.info(f"Enqueued image for PDF conversion: {target_path}")
-    elif mime_type in ALLOWED_MIME_TYPES or any(
-        file_ext.endswith(ext)
-        for ext in [
-            ".doc",
-            ".docx",
-            ".xls",
-            ".xlsx",
-            ".ppt",
-            ".pptx",
-            ".odt",
-            ".ods",
-            ".odp",
-            ".rtf",
-            ".txt",
-            ".csv",
-        ]
-    ):
-        # If it's an office document, convert to PDF first
+    elif mime_type in ALLOWED_MIME_TYPES or file_ext in ALLOWED_EXTENSIONS:
+        # Office document, HTML, Markdown, or other Gotenberg-supported format
        task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
-        logger.info(f"Enqueued office document for PDF conversion: {target_path}")
+        logger.info(f"Enqueued document for PDF conversion: {target_path}")
    else:
        # For any other file type, attempt conversion but log a warning
        logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
@@ -17,6 +17,7 @@ from pydantic import BaseModel, HttpUrl, field_validator
 from app.auth import require_login
 from app.config import settings
 from app.tasks.process_document import process_document
+from app.utils.allowed_types import ALLOWED_MIME_TYPES
 from app.utils.filename_utils import sanitize_filename

 # Set up logging
@@ -108,7 +109,7 @@ def validate_url_safety(url: str) -> None:

 def validate_file_type(content_type: str, filename: str) -> bool:
    """
-    Validate that the file type is supported.
+    Validate that the file type is supported (i.e. processable by Gotenberg).

    Args:
        content_type: MIME type from response headers
@@ -117,44 +118,18 @@ def validate_file_type(content_type: str, filename: str) -> bool:
    Returns:
        True if file type is allowed
    """
-    # Same allowed types as regular upload
-    ALLOWED_MIME_TYPES = {
-        "application/pdf",
-        "application/msword",
-        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-        "application/vnd.ms-excel",
-        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-        "application/vnd.ms-powerpoint",
-        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-        "text/plain",
-        "text/csv",
-        "application/rtf",
-        "text/rtf",
-    }
-
-    IMAGE_MIME_TYPES = {
-        "image/jpeg",
-        "image/jpg",
-        "image/png",
-        "image/gif",
-        "image/bmp",
-        "image/tiff",
-        "image/webp",
-        "image/svg+xml",
-    }
-
    # Check content type from header
    if content_type:
        # Handle content-type with charset (e.g., "application/pdf; charset=utf-8")
        base_content_type = content_type.split(";", maxsplit=1)[0].strip().lower()
-        if base_content_type in ALLOWED_MIME_TYPES or base_content_type in IMAGE_MIME_TYPES:
+        if base_content_type in ALLOWED_MIME_TYPES:
            return True

    # Also check by extension as fallback
    _, ext = os.path.splitext(filename)
    if ext:
        guessed_type, _ = mimetypes.guess_type(filename)
-        if guessed_type and (guessed_type in ALLOWED_MIME_TYPES or guessed_type in IMAGE_MIME_TYPES):
+        if guessed_type and guessed_type in ALLOWED_MIME_TYPES:
            return True

    return False
@@ -232,6 +232,22 @@ class Settings(BaseSettings):
        description="Delay in seconds between each task submission when throttling in /processall",
    )

+    # Client-side upload throttling settings (applied when uploading files via the web UI)
+    upload_concurrency: int = Field(
+        default=3,
+        description=(
+            "Maximum number of files uploaded simultaneously from the browser. "
+            "Limits parallel uploads to prevent API overload when dragging directories. Default: 3."
+        ),
+    )
+    upload_queue_delay_ms: int = Field(
+        default=500,
+        description=(
+            "Delay in milliseconds between starting each upload slot when queue is active. "
+            "Staggers upload starts to smooth out server load. Default: 500 ms."
+        ),
+    )
+
    # Notification settings
    notification_urls: Union[List[str], str] = Field(
        default_factory=list,
@@ -13,6 +13,7 @@ from celery import shared_task
 from app.config import settings
 from app.tasks.convert_to_pdf import convert_to_pdf  # new conversion task
 from app.tasks.process_document import process_document  # Updated import
+from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES

 logger = logging.getLogger(__name__)

@@ -268,20 +269,6 @@ def fetch_attachments_and_enqueue(email_message):

    Returns True if at least one allowed attachment was processed.
    """
-    ALLOWED_MIME_TYPES = {
-        "application/pdf",
-        "application/msword",
-        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-        "application/vnd.ms-excel",
-        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-        "application/vnd.ms-powerpoint",
-        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-        "text/plain",
-        "text/csv",
-        "application/rtf",
-        "text/rtf",
-    }
-
    has_attachment = False
    for part in email_message.walk():
        if part.get_content_maintype() == "multipart":
@@ -295,8 +282,9 @@ def fetch_attachments_and_enqueue(email_message):
        is_pdf_by_extension = filename.lower().endswith(".pdf")

        mime_type = part.get_content_type()
-        # Accept file if it has an allowed MIME type OR it's a PDF by extension
-        if mime_type not in ALLOWED_MIME_TYPES and not is_pdf_by_extension:
+        file_ext = os.path.splitext(filename)[1].lower()
+        # Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
+        if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension:
            logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
            continue

@@ -0,0 +1,133 @@
+"""
+Canonical file-type lists for DocuElevate uploads.
+
+All file types listed here are supported by Gotenberg (the PDF-conversion service
+used by DocuElevate) via its LibreOffice, Chromium, or Markdown routes.  This
+module is the single source of truth consumed by:
+
+  - app/api/files.py        (ui-upload endpoint)
+  - app/api/url_upload.py   (URL-upload endpoint)
+  - app/tasks/imap_tasks.py (IMAP email-attachment ingestion)
+  - frontend/static/js/upload.js  (client-side validation mirror)
+
+Keep in sync with the OFFICE_EXTENSIONS / IMAGE_EXTENSIONS sets defined in
+app/tasks/convert_to_pdf.py.
+"""
+
+# ---------------------------------------------------------------------------
+# Document / office MIME types (converted via Gotenberg LibreOffice route)
+# ---------------------------------------------------------------------------
+DOCUMENT_MIME_TYPES: set[str] = {
+    # PDF
+    "application/pdf",
+    # Word
+    "application/msword",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
+    "application/vnd.ms-word.document.macroEnabled.12",
+    "application/vnd.ms-word.template.macroEnabled.12",
+    # Excel
+    "application/vnd.ms-excel",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
+    "application/vnd.ms-excel.sheet.macroEnabled.12",
+    "application/vnd.ms-excel.sheet.binary.macroEnabled.12",
+    # PowerPoint
+    "application/vnd.ms-powerpoint",
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    "application/vnd.openxmlformats-officedocument.presentationml.template",
+    "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
+    "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
+    # OpenDocument (LibreOffice native)
+    "application/vnd.oasis.opendocument.text",
+    "application/vnd.oasis.opendocument.spreadsheet",
+    "application/vnd.oasis.opendocument.presentation",
+    "application/vnd.oasis.opendocument.graphics",
+    "application/vnd.oasis.opendocument.formula",
+    # Plain text / data
+    "text/plain",
+    "text/csv",
+    "application/rtf",
+    "text/rtf",
+    # HTML (converted via Gotenberg Chromium route)
+    "text/html",
+    # Markdown (converted via Gotenberg Chromium/Markdown route)
+    "text/markdown",
+    "text/x-markdown",
+}
+
+# ---------------------------------------------------------------------------
+# Image MIME types (converted via Gotenberg LibreOffice route)
+# ---------------------------------------------------------------------------
+IMAGE_MIME_TYPES: set[str] = {
+    "image/jpeg",
+    "image/jpg",
+    "image/png",
+    "image/gif",
+    "image/bmp",
+    "image/tiff",
+    "image/webp",
+    "image/svg+xml",
+}
+
+# ---------------------------------------------------------------------------
+# Combined set – every MIME type accepted by the upload endpoints
+# ---------------------------------------------------------------------------
+ALLOWED_MIME_TYPES: set[str] = DOCUMENT_MIME_TYPES | IMAGE_MIME_TYPES
+
+# ---------------------------------------------------------------------------
+# File extensions (lower-case, with leading dot) accepted by Gotenberg
+# ---------------------------------------------------------------------------
+ALLOWED_EXTENSIONS: set[str] = {
+    # PDF
+    ".pdf",
+    # Word
+    ".doc",
+    ".docx",
+    ".docm",
+    ".dot",
+    ".dotx",
+    ".dotm",
+    # Excel
+    ".xls",
+    ".xlsx",
+    ".xlsm",
+    ".xlsb",
+    ".xlt",
+    ".xltx",
+    ".xlw",
+    # PowerPoint
+    ".ppt",
+    ".pptx",
+    ".pptm",
+    ".pps",
+    ".ppsx",
+    ".pot",
+    ".potx",
+    # OpenDocument
+    ".odt",
+    ".ods",
+    ".odp",
+    ".odg",
+    ".odf",
+    # Text / data
+    ".rtf",
+    ".txt",
+    ".csv",
+    # Images
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".gif",
+    ".bmp",
+    ".tiff",
+    ".tif",
+    ".webp",
+    ".svg",
+    # Web
+    ".html",
+    ".htm",
+    # Markdown
+    ".md",
+    ".markdown",
+}
@@ -1057,6 +1057,28 @@ SETTING_METADATA = {
        "required": False,
        "restart_required": False,
    },
+    "upload_concurrency": {
+        "category": "Processing",
+        "description": (
+            "Maximum number of files uploaded simultaneously from the browser. "
+            "Limits parallel uploads to prevent API overload when dragging directories. Default: 3."
+        ),
+        "type": "integer",
+        "sensitive": False,
+        "required": False,
+        "restart_required": False,
+    },
+    "upload_queue_delay_ms": {
+        "category": "Processing",
+        "description": (
+            "Delay in milliseconds between starting each upload slot when queue is active. "
+            "Staggers upload starts to smooth out server load. Default: 500 ms."
+        ),
+        "type": "integer",
+        "sensitive": False,
+        "required": False,
+        "restart_required": False,
+    },
    "enable_text_quality_check": {
        "category": "Processing",
        "description": (
@@ -33,6 +33,8 @@ def files_page(
    """
    Return the 'files.html' template with server-side pagination, sorting, and filtering
    """
+    from app.config import settings
+
    try:
        # Import the model here to avoid circular imports
        from sqlalchemy import asc, desc
@@ -111,6 +113,8 @@ def files_page(
                "mime_type": mime_type or "",
                "status": status or "",
                "mime_types": mime_types,
+                "upload_concurrency": settings.upload_concurrency,
+                "upload_queue_delay_ms": settings.upload_queue_delay_ms,
            },
        )
    except Exception as e:
@@ -124,6 +128,8 @@ def files_page(
                "files": [],
                "pagination": {"page": 1, "per_page": per_page, "total_items": 0, "total_pages": 0},
                "error": str(e),
+                "upload_concurrency": settings.upload_concurrency,
+                "upload_queue_delay_ms": settings.upload_queue_delay_ms,
            },
        )

@@ -99,7 +99,16 @@ async def serve_imprint(request: Request):
@require_login
 async def serve_upload(request: Request):
    """Serve the upload page."""
-    return templates.TemplateResponse("upload.html", {"request": request})
+    from app.config import settings
+
+    return templates.TemplateResponse(
+        "upload.html",
+        {
+            "request": request,
+            "upload_concurrency": settings.upload_concurrency,
+            "upload_queue_delay_ms": settings.upload_queue_delay_ms,
+        },
+    )


@router.get("/favicon.ico", include_in_schema=False)