Merge pull request #426 from christianlouis/copilot/support-directory-upload

feat(upload): directory drag-and-drop, adaptive 429 throttling, full Gotenberg file type support

.env.demo

@@ -16,6 +16,12 @@ ALLOW_FILE_DELETE=true  # Allow deletion of file records
 PROCESSALL_THROTTLE_THRESHOLD=20  # Number of files above which throttling is applied (default: 20)
 PROCESSALL_THROTTLE_DELAY=3  # Delay in seconds between each task submission when throttling (default: 3)
 
+# **Client-Side Upload Throttling**
+# Controls pacing when the browser uploads files (especially large directory drops).
+# The browser auto-detects rate-limit (HTTP 429) responses and backs off accordingly.
+UPLOAD_CONCURRENCY=3         # Max simultaneous uploads from the browser (default: 3)
+UPLOAD_QUEUE_DELAY_MS=500    # Delay (ms) between starting each upload slot (default: 500)
+
 # **File Upload Size Limits** (Security - see SECURITY_AUDIT.md)
 # Maximum file upload size in bytes. Default: 1GB (1073741824 bytes)
 # Prevents resource exhaustion attacks. Adjust based on your server capacity.

app/api/files.py

@@ -18,6 +18,7 @@ from app.database import get_db
 from app.models import FileRecord, ProcessingLog
 from app.tasks.convert_to_pdf import convert_to_pdf
 from app.tasks.process_document import process_document
+from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, IMAGE_MIME_TYPES
 from app.utils.file_queries import apply_status_filter
 from app.utils.file_status import get_files_processing_status
 from app.utils.filename_utils import sanitize_filename
@@ -1034,33 +1035,6 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
     logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")
     file_size = written_size
 
-    # Same set of allowed file types as in the IMAP task
-    ALLOWED_MIME_TYPES = {
-        "application/pdf",
-        "application/msword",
-        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-        "application/vnd.ms-excel",
-        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-        "application/vnd.ms-powerpoint",
-        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-        "text/plain",
-        "text/csv",
-        "application/rtf",
-        "text/rtf",
-    }
-
-    # Image MIME types that need conversion
-    IMAGE_MIME_TYPES = {
-        "image/jpeg",
-        "image/jpg",
-        "image/png",
-        "image/gif",
-        "image/bmp",
-        "image/tiff",
-        "image/webp",
-        "image/svg+xml",
-    }
-
     # Determine if the file is a PDF or needs conversion
     mime_type, _ = mimetypes.guess_type(target_path)
     file_ext = os.path.splitext(target_path)[1].lower()
@@ -1114,32 +1088,24 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
         # If it's a PDF, process directly
         task = process_document.delay(target_path, original_filename=safe_filename)
         logger.info(f"Enqueued PDF for processing: {target_path}")
-    elif mime_type in IMAGE_MIME_TYPES or any(
-        file_ext.endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"]
-    ):
+    elif mime_type in IMAGE_MIME_TYPES or file_ext in {
+        ".jpg",
+        ".jpeg",
+        ".png",
+        ".gif",
+        ".bmp",
+        ".tiff",
+        ".tif",
+        ".webp",
+        ".svg",
+    }:
         # If it's an image, convert to PDF first
         task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
         logger.info(f"Enqueued image for PDF conversion: {target_path}")
-    elif mime_type in ALLOWED_MIME_TYPES or any(
-        file_ext.endswith(ext)
-        for ext in [
-            ".doc",
-            ".docx",
-            ".xls",
-            ".xlsx",
-            ".ppt",
-            ".pptx",
-            ".odt",
-            ".ods",
-            ".odp",
-            ".rtf",
-            ".txt",
-            ".csv",
-        ]
-    ):
-        # If it's an office document, convert to PDF first
+    elif mime_type in ALLOWED_MIME_TYPES or file_ext in ALLOWED_EXTENSIONS:
+        # Office document, HTML, Markdown, or other Gotenberg-supported format
         task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
-        logger.info(f"Enqueued office document for PDF conversion: {target_path}")
+        logger.info(f"Enqueued document for PDF conversion: {target_path}")
     else:
         # For any other file type, attempt conversion but log a warning
         logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")

app/api/url_upload.py

@@ -17,6 +17,7 @@ from pydantic import BaseModel, HttpUrl, field_validator
 from app.auth import require_login
 from app.config import settings
 from app.tasks.process_document import process_document
+from app.utils.allowed_types import ALLOWED_MIME_TYPES
 from app.utils.filename_utils import sanitize_filename
 
 # Set up logging
@@ -108,7 +109,7 @@ def validate_url_safety(url: str) -> None:
 
 def validate_file_type(content_type: str, filename: str) -> bool:
     """
-    Validate that the file type is supported.
+    Validate that the file type is supported (i.e. processable by Gotenberg).
 
     Args:
         content_type: MIME type from response headers
@@ -117,44 +118,18 @@ def validate_file_type(content_type: str, filename: str) -> bool:
     Returns:
         True if file type is allowed
     """
-    # Same allowed types as regular upload
-    ALLOWED_MIME_TYPES = {
-        "application/pdf",
-        "application/msword",
-        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-        "application/vnd.ms-excel",
-        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-        "application/vnd.ms-powerpoint",
-        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-        "text/plain",
-        "text/csv",
-        "application/rtf",
-        "text/rtf",
-    }
-
-    IMAGE_MIME_TYPES = {
-        "image/jpeg",
-        "image/jpg",
-        "image/png",
-        "image/gif",
-        "image/bmp",
-        "image/tiff",
-        "image/webp",
-        "image/svg+xml",
-    }
-
     # Check content type from header
     if content_type:
         # Handle content-type with charset (e.g., "application/pdf; charset=utf-8")
         base_content_type = content_type.split(";", maxsplit=1)[0].strip().lower()
-        if base_content_type in ALLOWED_MIME_TYPES or base_content_type in IMAGE_MIME_TYPES:
+        if base_content_type in ALLOWED_MIME_TYPES:
             return True
 
     # Also check by extension as fallback
     _, ext = os.path.splitext(filename)
     if ext:
         guessed_type, _ = mimetypes.guess_type(filename)
-        if guessed_type and (guessed_type in ALLOWED_MIME_TYPES or guessed_type in IMAGE_MIME_TYPES):
+        if guessed_type and guessed_type in ALLOWED_MIME_TYPES:
             return True
 
     return False

app/config.py

@@ -232,6 +232,22 @@ class Settings(BaseSettings):
         description="Delay in seconds between each task submission when throttling in /processall",
     )
 
+    # Client-side upload throttling settings (applied when uploading files via the web UI)
+    upload_concurrency: int = Field(
+        default=3,
+        description=(
+            "Maximum number of files uploaded simultaneously from the browser. "
+            "Limits parallel uploads to prevent API overload when dragging directories. Default: 3."
+        ),
+    )
+    upload_queue_delay_ms: int = Field(
+        default=500,
+        description=(
+            "Delay in milliseconds between starting each upload slot when queue is active. "
+            "Staggers upload starts to smooth out server load. Default: 500 ms."
+        ),
+    )
+
     # Notification settings
     notification_urls: Union[List[str], str] = Field(
         default_factory=list,

app/tasks/imap_tasks.py

@@ -13,6 +13,7 @@ from celery import shared_task
 from app.config import settings
 from app.tasks.convert_to_pdf import convert_to_pdf  # new conversion task
 from app.tasks.process_document import process_document  # Updated import
+from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES
 
 logger = logging.getLogger(__name__)
 
@@ -268,20 +269,6 @@ def fetch_attachments_and_enqueue(email_message):
 
     Returns True if at least one allowed attachment was processed.
     """
-    ALLOWED_MIME_TYPES = {
-        "application/pdf",
-        "application/msword",
-        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-        "application/vnd.ms-excel",
-        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-        "application/vnd.ms-powerpoint",
-        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-        "text/plain",
-        "text/csv",
-        "application/rtf",
-        "text/rtf",
-    }
-
     has_attachment = False
     for part in email_message.walk():
         if part.get_content_maintype() == "multipart":
@@ -295,8 +282,9 @@ def fetch_attachments_and_enqueue(email_message):
         is_pdf_by_extension = filename.lower().endswith(".pdf")
 
         mime_type = part.get_content_type()
-        # Accept file if it has an allowed MIME type OR it's a PDF by extension
-        if mime_type not in ALLOWED_MIME_TYPES and not is_pdf_by_extension:
+        file_ext = os.path.splitext(filename)[1].lower()
+        # Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
+        if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension:
             logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
             continue
 

app/utils/allowed_types.py

@@ -0,0 +1,133 @@
+"""
+Canonical file-type lists for DocuElevate uploads.
+
+All file types listed here are supported by Gotenberg (the PDF-conversion service
+used by DocuElevate) via its LibreOffice, Chromium, or Markdown routes.  This
+module is the single source of truth consumed by:
+
+  - app/api/files.py        (ui-upload endpoint)
+  - app/api/url_upload.py   (URL-upload endpoint)
+  - app/tasks/imap_tasks.py (IMAP email-attachment ingestion)
+  - frontend/static/js/upload.js  (client-side validation mirror)
+
+Keep in sync with the OFFICE_EXTENSIONS / IMAGE_EXTENSIONS sets defined in
+app/tasks/convert_to_pdf.py.
+"""
+
+# ---------------------------------------------------------------------------
+# Document / office MIME types (converted via Gotenberg LibreOffice route)
+# ---------------------------------------------------------------------------
+DOCUMENT_MIME_TYPES: set[str] = {
+    # PDF
+    "application/pdf",
+    # Word
+    "application/msword",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
+    "application/vnd.ms-word.document.macroEnabled.12",
+    "application/vnd.ms-word.template.macroEnabled.12",
+    # Excel
+    "application/vnd.ms-excel",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
+    "application/vnd.ms-excel.sheet.macroEnabled.12",
+    "application/vnd.ms-excel.sheet.binary.macroEnabled.12",
+    # PowerPoint
+    "application/vnd.ms-powerpoint",
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    "application/vnd.openxmlformats-officedocument.presentationml.template",
+    "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
+    "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
+    # OpenDocument (LibreOffice native)
+    "application/vnd.oasis.opendocument.text",
+    "application/vnd.oasis.opendocument.spreadsheet",
+    "application/vnd.oasis.opendocument.presentation",
+    "application/vnd.oasis.opendocument.graphics",
+    "application/vnd.oasis.opendocument.formula",
+    # Plain text / data
+    "text/plain",
+    "text/csv",
+    "application/rtf",
+    "text/rtf",
+    # HTML (converted via Gotenberg Chromium route)
+    "text/html",
+    # Markdown (converted via Gotenberg Chromium/Markdown route)
+    "text/markdown",
+    "text/x-markdown",
+}
+
+# ---------------------------------------------------------------------------
+# Image MIME types (converted via Gotenberg LibreOffice route)
+# ---------------------------------------------------------------------------
+IMAGE_MIME_TYPES: set[str] = {
+    "image/jpeg",
+    "image/jpg",
+    "image/png",
+    "image/gif",
+    "image/bmp",
+    "image/tiff",
+    "image/webp",
+    "image/svg+xml",
+}
+
+# ---------------------------------------------------------------------------
+# Combined set – every MIME type accepted by the upload endpoints
+# ---------------------------------------------------------------------------
+ALLOWED_MIME_TYPES: set[str] = DOCUMENT_MIME_TYPES | IMAGE_MIME_TYPES
+
+# ---------------------------------------------------------------------------
+# File extensions (lower-case, with leading dot) accepted by Gotenberg
+# ---------------------------------------------------------------------------
+ALLOWED_EXTENSIONS: set[str] = {
+    # PDF
+    ".pdf",
+    # Word
+    ".doc",
+    ".docx",
+    ".docm",
+    ".dot",
+    ".dotx",
+    ".dotm",
+    # Excel
+    ".xls",
+    ".xlsx",
+    ".xlsm",
+    ".xlsb",
+    ".xlt",
+    ".xltx",
+    ".xlw",
+    # PowerPoint
+    ".ppt",
+    ".pptx",
+    ".pptm",
+    ".pps",
+    ".ppsx",
+    ".pot",
+    ".potx",
+    # OpenDocument
+    ".odt",
+    ".ods",
+    ".odp",
+    ".odg",
+    ".odf",
+    # Text / data
+    ".rtf",
+    ".txt",
+    ".csv",
+    # Images
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".gif",
+    ".bmp",
+    ".tiff",
+    ".tif",
+    ".webp",
+    ".svg",
+    # Web
+    ".html",
+    ".htm",
+    # Markdown
+    ".md",
+    ".markdown",
+}

app/utils/settings_service.py

@@ -1057,6 +1057,28 @@ SETTING_METADATA = {
         "required": False,
         "restart_required": False,
     },
+    "upload_concurrency": {
+        "category": "Processing",
+        "description": (
+            "Maximum number of files uploaded simultaneously from the browser. "
+            "Limits parallel uploads to prevent API overload when dragging directories. Default: 3."
+        ),
+        "type": "integer",
+        "sensitive": False,
+        "required": False,
+        "restart_required": False,
+    },
+    "upload_queue_delay_ms": {
+        "category": "Processing",
+        "description": (
+            "Delay in milliseconds between starting each upload slot when queue is active. "
+            "Staggers upload starts to smooth out server load. Default: 500 ms."
+        ),
+        "type": "integer",
+        "sensitive": False,
+        "required": False,
+        "restart_required": False,
+    },
     "enable_text_quality_check": {
         "category": "Processing",
         "description": (

app/views/files.py

@@ -33,6 +33,8 @@ def files_page(
     """
     Return the 'files.html' template with server-side pagination, sorting, and filtering
     """
+    from app.config import settings
+
     try:
         # Import the model here to avoid circular imports
         from sqlalchemy import asc, desc
@@ -111,6 +113,8 @@ def files_page(
                 "mime_type": mime_type or "",
                 "status": status or "",
                 "mime_types": mime_types,
+                "upload_concurrency": settings.upload_concurrency,
+                "upload_queue_delay_ms": settings.upload_queue_delay_ms,
             },
         )
     except Exception as e:
@@ -124,6 +128,8 @@ def files_page(
                 "files": [],
                 "pagination": {"page": 1, "per_page": per_page, "total_items": 0, "total_pages": 0},
                 "error": str(e),
+                "upload_concurrency": settings.upload_concurrency,
+                "upload_queue_delay_ms": settings.upload_queue_delay_ms,
             },
         )
 

app/views/general.py

@@ -99,7 +99,16 @@ async def serve_imprint(request: Request):
 @require_login
 async def serve_upload(request: Request):
     """Serve the upload page."""
-    return templates.TemplateResponse("upload.html", {"request": request})
+    from app.config import settings
+
+    return templates.TemplateResponse(
+        "upload.html",
+        {
+            "request": request,
+            "upload_concurrency": settings.upload_concurrency,
+            "upload_queue_delay_ms": settings.upload_queue_delay_ms,
+        },
+    )
 
 
 @router.get("/favicon.ico", include_in_schema=False)

docs/ConfigurationGuide.md

@@ -31,6 +31,19 @@ Control how the `/processall` endpoint handles large batches of files to prevent
 - Total queue time: (25-1) × 3 = 72 seconds
 - Prevents API rate limit issues and ensures smooth processing
 
+### Client-Side Upload Throttling
+
+Control how the web UI queues and paces file uploads to avoid overwhelming the backend, especially when dragging large directories (potentially thousands of files) onto the upload area.
+
+| **Variable**               | **Description**                                                                                                               | **Default** |
+|----------------------------|-------------------------------------------------------------------------------------------------------------------------------|-------------|
+| `UPLOAD_CONCURRENCY`       | Maximum number of files uploaded simultaneously from the browser.                                                            | `3`         |
+| `UPLOAD_QUEUE_DELAY_MS`    | Delay in milliseconds between starting each upload slot. Staggers upload starts to smooth out server load.                   | `500`       |
+
+**Adaptive back-off**: The browser automatically slows down if the server responds with HTTP 429 (Too Many Requests). It reads the `Retry-After` header, pauses the queue for the indicated time, doubles the inter-slot delay (exponential back-off, capped at 30 s), and reduces concurrency to 1. After 5 consecutive successes it gradually recovers toward the configured values.
+
+**Example**: With `UPLOAD_CONCURRENCY=3` and `UPLOAD_QUEUE_DELAY_MS=500`, a directory of 5,000 files is uploaded ≈ 3 at a time with 500 ms pacing – the backend processes files at its own rate while the queue drains in the background without triggering API rate limits.
+
 ### File Upload Size Limits
 
 **Security Feature**: Control file upload sizes to prevent resource exhaustion attacks. See [SECURITY_AUDIT.md](../SECURITY_AUDIT.md#5-file-upload-size-limits) for security details.

frontend/static/js/upload.js

@@ -1,221 +1,492 @@
 // frontend/static/js/upload.js
 // Reusable drag-and-drop upload functionality for DocuElevate
 
-// Configuration
-const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MB
+// ── Configuration ─────────────────────────────────────────────────────────────
+const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500 MB
 
-// Allowed file types
+/** Fallback upload throttling values when window.uploadConfig is not set. */
+const DEFAULT_UPLOAD_CONCURRENCY = 3;
+const DEFAULT_UPLOAD_QUEUE_DELAY_MS = 500;
+
+/** Maximum number of 429 retries before a file is permanently marked failed. */
+const MAX_RATE_LIMIT_RETRIES = 5;
+
+// ── Accepted MIME types (mirrors app/utils/allowed_types.py) ─────────────────
+// All types processable by Gotenberg (LibreOffice, Chromium, or Markdown routes).
 const ACCEPTED_TYPES = {
-  // PDF files
+  // PDF
   'application/pdf': true,
-
-  // Image formats
-  'image/jpeg': true, 'image/jpg': true, 'image/png': true,
-  'image/gif': true, 'image/bmp': true, 'image/tiff': true,
-  'image/webp': true, 'image/svg+xml': true,
-
-  // Office document formats - Word
+  // Word
   'application/msword': true,
   'application/vnd.openxmlformats-officedocument.wordprocessingml.document': true,
   'application/vnd.openxmlformats-officedocument.wordprocessingml.template': true,
   'application/vnd.ms-word.document.macroEnabled.12': true,
-
+  'application/vnd.ms-word.template.macroEnabled.12': true,
   // Excel
   'application/vnd.ms-excel': true,
   'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': true,
   'application/vnd.openxmlformats-officedocument.spreadsheetml.template': true,
   'application/vnd.ms-excel.sheet.macroEnabled.12': true,
-
+  'application/vnd.ms-excel.sheet.binary.macroEnabled.12': true,
   // PowerPoint
   'application/vnd.ms-powerpoint': true,
   'application/vnd.openxmlformats-officedocument.presentationml.presentation': true,
   'application/vnd.openxmlformats-officedocument.presentationml.template': true,
   'application/vnd.openxmlformats-officedocument.presentationml.slideshow': true,
-
-  // Other common formats
+  'application/vnd.ms-powerpoint.presentation.macroEnabled.12': true,
+  // OpenDocument (LibreOffice native)
+  'application/vnd.oasis.opendocument.text': true,
+  'application/vnd.oasis.opendocument.spreadsheet': true,
+  'application/vnd.oasis.opendocument.presentation': true,
+  'application/vnd.oasis.opendocument.graphics': true,
+  'application/vnd.oasis.opendocument.formula': true,
+  // Images
+  'image/jpeg': true, 'image/jpg': true, 'image/png': true,
+  'image/gif': true, 'image/bmp': true, 'image/tiff': true,
+  'image/webp': true, 'image/svg+xml': true,
+  // Text / data
   'text/plain': true,
   'text/csv': true,
   'application/rtf': true,
   'text/rtf': true,
+  // HTML (Gotenberg Chromium route)
   'text/html': true,
-  'application/xml': true,
-  'text/xml': true
+  // Markdown (Gotenberg Chromium/Markdown route)
+  'text/markdown': true,
+  'text/x-markdown': true,
 };
 
-// File extensions that are always allowed (even if mime type is not recognized)
-const ACCEPTED_EXTENSIONS = [
-  '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
-  '.odt', '.ods', '.odp', '.rtf', '.txt', '.csv',
-  '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg', '.md'
-];
+// File extensions always accepted even when the browser reports no / wrong MIME type.
+const ACCEPTED_EXTENSIONS = new Set([
+  // PDF
+  '.pdf',
+  // Word
+  '.doc', '.docx', '.docm', '.dot', '.dotx', '.dotm',
+  // Excel
+  '.xls', '.xlsx', '.xlsm', '.xlsb', '.xlt', '.xltx', '.xlw',
+  // PowerPoint
+  '.ppt', '.pptx', '.pptm', '.pps', '.ppsx', '.pot', '.potx',
+  // OpenDocument
+  '.odt', '.ods', '.odp', '.odg', '.odf',
+  // Text / data
+  '.rtf', '.txt', '.csv',
+  // Images
+  '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif', '.webp', '.svg',
+  // Web
+  '.html', '.htm',
+  // Markdown
+  '.md', '.markdown',
+]);
+
+// ── Adaptive throttle state ───────────────────────────────────────────────────
+// Module-level so the backoff state persists across multiple drop/select events
+// on the same page (rate limits are per-user on the server).
+const _adaptiveState = {
+  /** Current effective inter-slot delay (ms). null = use window.uploadConfig value. */
+  delayMs: null,
+  /** Current effective concurrency. null = use window.uploadConfig value. */
+  concurrency: null,
+  /** Consecutive successful uploads without a 429. Resets on each 429 or recovery step. */
+  consecutiveOk: 0,
+  /** Date.now() timestamp after which the queue may resume (set on 429 backoff). */
+  pauseUntil: 0,
+};
+
+function _cfgDelay() {
+  return (window.uploadConfig && window.uploadConfig.queueDelayMs != null)
+    ? window.uploadConfig.queueDelayMs
+    : DEFAULT_UPLOAD_QUEUE_DELAY_MS;
+}
+
+function _cfgConcurrency() {
+  return (window.uploadConfig && window.uploadConfig.concurrency != null)
+    ? window.uploadConfig.concurrency
+    : DEFAULT_UPLOAD_CONCURRENCY;
+}
+
+/** Effective delay between upload slot starts (increased during backoff). */
+function _effectiveDelay() {
+  return _adaptiveState.delayMs !== null ? _adaptiveState.delayMs : _cfgDelay();
+}
+
+/** Fallback backoff multiplier when no Retry-After header is present. */
+const FALLBACK_BACKOFF_MULTIPLIER = 4;
+/** Minimum fallback pause duration (ms) when no Retry-After header is present. */
+const MIN_FALLBACK_BACKOFF_MS = 5000;
+/** Maximum inter-slot delay after repeated exponential backoff (ms). */
+const MAX_BACKOFF_DELAY_MS = 30000;
+
+/** Effective concurrency (reduced to 1 during backoff). */
+function _effectiveConcurrency() {
+  return _adaptiveState.concurrency !== null ? _adaptiveState.concurrency : _cfgConcurrency();
+}
 
 /**
- * Process a list of files for upload
- * @param {FileList} files - Files to process
- * @param {HTMLElement} progressContainer - Container element for progress display
- * @param {HTMLElement} statusMessage - Element for status message display
+ * Called when an HTTP 429 Too Many Requests response is received.
+ * Pauses the queue and applies exponential backoff.
+ * @param {number} retryAfterSeconds - Value of the Retry-After header (0 = absent).
+ */
+function _onRateLimited(retryAfterSeconds) {
+  // Determine how long to pause – prefer the server's Retry-After; fall back to
+  // FALLBACK_BACKOFF_MULTIPLIER × the current delay (minimum MIN_FALLBACK_BACKOFF_MS).
+  const waitMs = retryAfterSeconds > 0
+    ? retryAfterSeconds * 1000
+    : Math.max(_effectiveDelay() * FALLBACK_BACKOFF_MULTIPLIER, MIN_FALLBACK_BACKOFF_MS);
+
+  _adaptiveState.pauseUntil = Date.now() + waitMs;
+  // Exponential backoff on the inter-slot delay, capped at MAX_BACKOFF_DELAY_MS.
+  _adaptiveState.delayMs = Math.min(_effectiveDelay() * 2, MAX_BACKOFF_DELAY_MS);
+  // Serialize uploads while we recover.
+  _adaptiveState.concurrency = 1;
+  _adaptiveState.consecutiveOk = 0;
+
+  console.warn(
+    `[DocuElevate] Rate limited. Pausing ${waitMs} ms. ` +
+    `New delay: ${_adaptiveState.delayMs} ms, concurrency: 1.`
+  );
+}
+
+/**
+ * Called after each successful (non-429) upload.
+ * After 5 consecutive successes, gently recovers toward the configured values.
+ */
+function _onUploadSuccess() {
+  _adaptiveState.consecutiveOk++;
+  if (_adaptiveState.consecutiveOk < 5) return;
+
+  // One recovery step every 5 successes.
+  _adaptiveState.consecutiveOk = 0;
+  const cfgDelay = _cfgDelay();
+  const cfgConc = _cfgConcurrency();
+
+  if (_adaptiveState.delayMs !== null && _adaptiveState.delayMs > cfgDelay) {
+    _adaptiveState.delayMs = Math.max(Math.round(_adaptiveState.delayMs * 0.75), cfgDelay);
+    if (_adaptiveState.delayMs <= cfgDelay) _adaptiveState.delayMs = null; // fully recovered
+  }
+  if (_adaptiveState.concurrency !== null && _adaptiveState.concurrency < cfgConc) {
+    _adaptiveState.concurrency = Math.min(_adaptiveState.concurrency + 1, cfgConc);
+    if (_adaptiveState.concurrency >= cfgConc) _adaptiveState.concurrency = null; // fully recovered
+  }
+}
+
+// ── Directory traversal helpers ───────────────────────────────────────────────
+
+/**
+ * Read all entries from a DirectoryReader, handling the browser's 100-entry
+ * per-batch limit by calling readEntries() repeatedly.
+ * @param {FileSystemDirectoryReader} reader
+ * @returns {Promise<FileSystemEntry[]>}
+ */
+function readAllDirectoryEntries(reader) {
+  return new Promise((resolve, reject) => {
+    const entries = [];
+    function readBatch() {
+      reader.readEntries((batch) => {
+        if (batch.length === 0) { resolve(entries); return; }
+        entries.push(...batch);
+        readBatch();
+      }, reject);
+    }
+    readBatch();
+  });
+}
+
+/**
+ * Recursively collect all File objects from a FileSystemEntry tree.
+ * @param {FileSystemEntry} entry
+ * @param {File[]} files - accumulator
+ * @returns {Promise<void>}
+ */
+async function traverseFileEntry(entry, files) {
+  if (entry.isFile) {
+    await new Promise((resolve) => {
+      entry.file((file) => { files.push(file); resolve(); }, resolve);
+    });
+  } else if (entry.isDirectory) {
+    const subEntries = await readAllDirectoryEntries(entry.createReader());
+    for (const sub of subEntries) {
+      await traverseFileEntry(sub, files);
+    }
+  }
+}
+
+/**
+ * Extract all File objects from a DataTransfer, recursively expanding any
+ * dropped directories.  Falls back gracefully to dataTransfer.files when the
+ * FileSystem Entry API is unavailable (Safari < 11.1, some mobile browsers).
+ * @param {DataTransfer} dataTransfer
+ * @returns {Promise<File[]>}
+ */
+async function getFilesFromDataTransfer(dataTransfer) {
+  if (dataTransfer.items && dataTransfer.items.length > 0) {
+    const files = [];
+    for (let i = 0; i < dataTransfer.items.length; i++) {
+      const item = dataTransfer.items[i];
+      const entry = item.webkitGetAsEntry ? item.webkitGetAsEntry() : null;
+      if (entry) {
+        await traverseFileEntry(entry, files);
+      } else if (item.kind === 'file') {
+        const file = item.getAsFile();
+        if (file) files.push(file);
+      }
+    }
+    return files;
+  }
+  return Array.from(dataTransfer.files || []);
+}
+
+// ── Core queue runner ─────────────────────────────────────────────────────────
+
+/**
+ * Validate and queue files for upload with adaptive throttling.
+ *
+ * Files are pre-rendered as progress rows so the user immediately sees the
+ * full list.  The queue runner respects the current effective concurrency and
+ * delay, slowing down automatically when the server signals rate limiting (429).
+ *
+ * @param {File[]|FileList} files
+ * @param {HTMLElement} progressContainer
+ * @param {HTMLElement} statusMessage
  */
 function processFiles(files, progressContainer, statusMessage) {
-  if (files.length === 0) return;
+  const fileArray = Array.from(files);
+  if (!fileArray.length) return;
 
   if (statusMessage) {
-    statusMessage.textContent = `Processing ${files.length} file(s)...`;
+    statusMessage.textContent = `Queued ${fileArray.length} file(s) for upload…`;
+  }
+  if (progressContainer) progressContainer.innerHTML = '';
+
+  // Pre-create one progress row per file.
+  const queueItems = fileArray.map((file) => {
+    const row = document.createElement('div');
+    row.className = 'flex flex-col mb-2';
+    row.innerHTML = `
+      <div class="flex justify-between">
+        <span class="text-sm truncate" title="${file.name}">${file.name}</span>
+        <span class="text-xs text-gray-500">${formatFileSize(file.size)}</span>
+      </div>
+      <div class="w-full bg-gray-200 h-2 rounded-full mt-1">
+        <div class="file-progress-bar bg-gray-300 h-2 rounded-full" style="width:0%"></div>
+      </div>
+      <div class="file-status text-xs text-gray-400 mt-1">Queued</div>
+    `;
+    if (progressContainer) progressContainer.appendChild(row);
+    return {
+      file,
+      progressBar: row.querySelector('.file-progress-bar'),
+      statusEl: row.querySelector('.file-status'),
+      retryCount: 0,
+    };
+  });
+
+  // Mutable queue – rate-limited items are pushed back to the front.
+  const queue = [...queueItems];
+  let active = 0;
+
+  function scheduleNext() {
+    // Respect global backoff pause.
+    const pauseRemaining = _adaptiveState.pauseUntil - Date.now();
+    if (pauseRemaining > 0) {
+      setTimeout(scheduleNext, pauseRemaining + 50);
+      return;
+    }
+
+    while (active < _effectiveConcurrency() && queue.length > 0) {
+      const item = queue.shift();
+      active++;
+
+      // Validate before hitting the network.
+      if (!_isAcceptedFile(item.file)) {
+        item.progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
+        item.statusEl.textContent = 'Unsupported file type';
+        item.statusEl.className = 'text-xs text-red-500 mt-1';
+        active--;
+        updateOverallStatus(statusMessage);
+        // No HTTP request – skip straight to next without adding delay.
+        scheduleNext();
+        continue;
+      }
+
+      if (item.file.size > MAX_FILE_SIZE) {
+        item.progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
+        item.statusEl.textContent = 'Exceeds 500 MB limit';
+        item.statusEl.className = 'text-xs text-red-500 mt-1';
+        active--;
+        updateOverallStatus(statusMessage);
+        scheduleNext();
+        continue;
+      }
+
+      _uploadSingleFile(item.file, item.progressBar, item.statusEl, statusMessage)
+        .then((result) => {
+          active--;
+          if (result.rateLimited) {
+            _onRateLimited(result.retryAfterSeconds);
+            item.retryCount++;
+            if (item.retryCount < MAX_RATE_LIMIT_RETRIES) {
+              // Re-insert at the front of the queue to retry after the pause.
+              queue.unshift(item);
+            } else {
+              item.progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
+              item.statusEl.textContent = 'Failed: rate limit retries exhausted';
+              item.statusEl.className = 'text-xs text-red-500 mt-1';
+              updateOverallStatus(statusMessage);
+            }
+            // Resume after the backoff window.
+            const wait = Math.max(_adaptiveState.pauseUntil - Date.now() + 50, 0);
+            setTimeout(scheduleNext, wait);
+          } else {
+            // Success or permanent error – wait the configured delay before next slot.
+            setTimeout(scheduleNext, _effectiveDelay());
+          }
+        });
+    }
   }
 
-  // Clear previous upload progress
-  if (progressContainer) {
-    progressContainer.innerHTML = "";
-  }
-
-  // Process each file
-  for (let i = 0; i < files.length; i++) {
-    const file = files[i];
-    validateAndUpload(file, progressContainer, statusMessage);
-  }
+  scheduleNext();
 }
 
 /**
- * Validate and upload a single file
- * @param {File} file - File to validate and upload
- * @param {HTMLElement} progressContainer - Container element for progress display
- * @param {HTMLElement} statusMessage - Element for status message display
+ * Check whether a file passes MIME type and extension validation.
+ * @param {File} file
+ * @returns {boolean}
  */
-function validateAndUpload(file, progressContainer, statusMessage) {
-  // Create progress element for this file
-  const fileProgress = document.createElement("div");
-  fileProgress.className = "flex flex-col mb-2";
-  fileProgress.innerHTML = `
-    <div class="flex justify-between">
-      <span class="text-sm truncate" title="${file.name}">${file.name}</span>
-      <span class="text-xs text-gray-500">${formatFileSize(file.size)}</span>
-    </div>
-    <div class="w-full bg-gray-200 h-2 rounded-full mt-1">
-      <div class="file-progress-bar bg-blue-500 h-2 rounded-full" style="width: 0%"></div>
-    </div>
-    <div class="file-status text-xs text-gray-600 mt-1">Validating...</div>
-  `;
-
-  if (progressContainer) {
-    progressContainer.appendChild(fileProgress);
-  }
-
-  const progressBar = fileProgress.querySelector(".file-progress-bar");
-  const statusEl = fileProgress.querySelector(".file-status");
-
-  // Validate file type by checking both MIME type and extension
-  const isValidMimeType = ACCEPTED_TYPES[file.type] || false;
-  const fileExtension = '.' + file.name.split('.').pop().toLowerCase();
-  const isValidExtension = ACCEPTED_EXTENSIONS.includes(fileExtension);
-
-  if (!isValidMimeType && !isValidExtension) {
-    statusEl.textContent = `Error: ${file.name} - Unsupported file type`;
-    statusEl.className = "text-xs text-red-500 mt-1";
-    return;
-  }
-
-  // Validate file size
-  if (file.size > MAX_FILE_SIZE) {
-    statusEl.textContent = `Error: ${file.name} - File size exceeds 500MB limit`;
-    statusEl.className = "text-xs text-red-500 mt-1";
-    return;
-  }
-
-  // Upload the file
-  uploadFile(file, progressBar, statusEl, statusMessage);
+function _isAcceptedFile(file) {
+  if (ACCEPTED_TYPES[file.type]) return true;
+  const ext = '.' + file.name.split('.').pop().toLowerCase();
+  return ACCEPTED_EXTENSIONS.has(ext);
 }
 
 /**
- * Upload a file to the server
- * @param {File} file - File to upload
- * @param {HTMLElement} progressBar - Progress bar element
- * @param {HTMLElement} statusEl - Status element
- * @param {HTMLElement} statusMessage - Overall status message element
+ * Upload a single file via XHR, returning a structured result.
+ * Detects HTTP 429 responses and reads the Retry-After / X-RateLimit-Reset
+ * headers so the caller can apply precise backoff.
+ *
+ * @param {File} file
+ * @param {HTMLElement} progressBar
+ * @param {HTMLElement} statusEl
+ * @param {HTMLElement} statusMessage
+ * @returns {Promise<{rateLimited: boolean, retryAfterSeconds: number}>}
  */
-async function uploadFile(file, progressBar, statusEl, statusMessage) {
-  statusEl.textContent = `Uploading...`;
-  try {
-    let formData = new FormData();
-    formData.append("file", file);
+function _uploadSingleFile(file, progressBar, statusEl, statusMessage) {
+  statusEl.textContent = 'Uploading…';
+  statusEl.className = 'text-xs text-gray-600 mt-1';
+  progressBar.style.width = '0%';
+  progressBar.className = 'file-progress-bar bg-blue-500 h-2 rounded-full';
+
+  return new Promise((resolve) => {
+    const formData = new FormData();
+    formData.append('file', file);
 
     const xhr = new XMLHttpRequest();
-    xhr.open("POST", "/api/ui-upload", true);
+    xhr.open('POST', '/api/ui-upload', true);
 
-    // Attach CSRF token so the server-side CSRF middleware accepts the request.
     const csrfToken = typeof getCsrfToken === 'function' ? getCsrfToken() : '';
-    if (csrfToken) {
-      xhr.setRequestHeader("X-CSRF-Token", csrfToken);
-    }
+    if (csrfToken) xhr.setRequestHeader('X-CSRF-Token', csrfToken);
 
     xhr.upload.onprogress = (e) => {
       if (e.lengthComputable) {
-        const percentComplete = (e.loaded / e.total) * 100;
-        progressBar.style.width = percentComplete + "%";
-        statusEl.textContent = `Uploading: ${Math.round(percentComplete)}%`;
+        const pct = Math.round((e.loaded / e.total) * 100);
+        progressBar.style.width = pct + '%';
+        statusEl.textContent = `Uploading: ${pct}%`;
       }
     };
 
-    xhr.onload = function() {
+    xhr.onload = () => {
       if (xhr.status === 200) {
         const result = JSON.parse(xhr.responseText);
-        progressBar.style.width = "100%";
-        progressBar.className = "file-progress-bar bg-green-500 h-2 rounded-full";
+        progressBar.style.width = '100%';
+        progressBar.className = 'file-progress-bar bg-green-500 h-2 rounded-full';
         statusEl.textContent = `Success: Task ID: ${result.task_id}`;
-        statusEl.className = "text-xs text-green-600 mt-1";
+        statusEl.className = 'text-xs text-green-600 mt-1';
+        _onUploadSuccess();
         updateOverallStatus(statusMessage);
+        resolve({ rateLimited: false, retryAfterSeconds: 0 });
+
+      } else if (xhr.status === 429) {
+        // Parse Retry-After (seconds integer).
+        let retryAfter = parseInt(xhr.getResponseHeader('Retry-After') || '0', 10);
+        if (!retryAfter) {
+          // Fall back to X-RateLimit-Reset (Unix timestamp).
+          const reset = parseInt(xhr.getResponseHeader('X-RateLimit-Reset') || '0', 10);
+          if (reset) retryAfter = Math.max(reset - Math.floor(Date.now() / 1000), 1);
+        }
+        progressBar.className = 'file-progress-bar bg-yellow-400 h-2 rounded-full';
+        statusEl.textContent = 'Rate limited – queued to retry…';
+        statusEl.className = 'text-xs text-yellow-600 mt-1';
+        resolve({ rateLimited: true, retryAfterSeconds: retryAfter });
+
       } else {
-        throw new Error(`Upload failed with status ${xhr.status}`);
+        progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
+        statusEl.textContent = `Error: HTTP ${xhr.status}`;
+        statusEl.className = 'text-xs text-red-500 mt-1';
+        updateOverallStatus(statusMessage);
+        resolve({ rateLimited: false, retryAfterSeconds: 0 });
       }
     };
 
-    xhr.onerror = function() {
-      throw new Error("Network error occurred");
+    xhr.onerror = () => {
+      progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
+      statusEl.textContent = 'Error: Network error';
+      statusEl.className = 'text-xs text-red-500 mt-1';
+      updateOverallStatus(statusMessage);
+      resolve({ rateLimited: false, retryAfterSeconds: 0 });
     };
 
     xhr.send(formData);
-
-  } catch (err) {
-    statusEl.textContent = `Error: ${err.message}`;
-    statusEl.className = "text-xs text-red-500 mt-1";
-    progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
-    updateOverallStatus(statusMessage);
-  }
+  });
 }
 
+// ── Legacy single-file entry point (kept for backward compat) ─────────────────
+
 /**
- * Update the overall status message based on file statuses
- * @param {HTMLElement} statusMessage - Status message element
+ * Validate and upload a single file (legacy path – wraps the queue runner).
+ * @param {File} file
+ * @param {HTMLElement} progressContainer
+ * @param {HTMLElement} statusMessage
+ */
+function validateAndUpload(file, progressContainer, statusMessage) {
+  processFiles([file], progressContainer, statusMessage);
+}
+
+// ── Status helpers ────────────────────────────────────────────────────────────
+
+/**
+ * Re-calculate and display the overall upload status.
+ * Fires 'allUploadsComplete' when every item has a terminal status.
+ * @param {HTMLElement} statusMessage
  */
 function updateOverallStatus(statusMessage) {
   if (!statusMessage) return;
 
-  // Count success/failure
   const fileStatuses = document.querySelectorAll('.file-status');
-  let completed = 0;
-  let total = fileStatuses.length;
+  let done = 0;
+  const total = fileStatuses.length;
 
-  fileStatuses.forEach(status => {
-    if (status.textContent.includes('Success') || status.textContent.includes('Error')) {
-      completed++;
-    }
+  fileStatuses.forEach((s) => {
+    const t = s.textContent;
+    if (
+      t.startsWith('Success') ||
+      t.startsWith('Error') ||
+      t.startsWith('Unsupported') ||
+      t.startsWith('Exceeds') ||
+      t.startsWith('Failed:')
+    ) done++;
   });
 
-  if (completed === total) {
-    statusMessage.textContent = `All uploads completed (${completed}/${total})`;
-
-    // Trigger a custom event when all uploads are complete
-    const allUploadsComplete = new CustomEvent('allUploadsComplete', {
-      detail: { total: total, completed: completed }
-    });
-    window.dispatchEvent(allUploadsComplete);
+  if (done === total && total > 0) {
+    statusMessage.textContent = `All uploads completed (${done}/${total})`;
+    window.dispatchEvent(new CustomEvent('allUploadsComplete', { detail: { total, completed: done } }));
   } else {
-    statusMessage.textContent = `Uploading files (${completed}/${total})`;
+    statusMessage.textContent = `Uploading files (${done}/${total})`;
   }
 }
 
 /**
- * Format file size for display
- * @param {number} bytes - File size in bytes
- * @returns {string} Formatted file size
+ * Format a byte count for human-readable display.
+ * @param {number} bytes
+ * @returns {string}
  */
 function formatFileSize(bytes) {
   if (bytes === 0) return '0 Bytes';
@@ -225,52 +496,43 @@ function formatFileSize(bytes) {
   return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
 }
 
+// ── Drag-and-drop initialiser ─────────────────────────────────────────────────
+
 /**
- * Initialize drag-and-drop on an element
- * @param {HTMLElement} element - Element to enable drag-and-drop on
- * @param {HTMLElement} progressContainer - Container for progress display
- * @param {HTMLElement} statusMessage - Element for status messages
- * @param {Object} options - Additional options
+ * Wire up drag-and-drop on an element with full directory-traversal support.
+ * @param {HTMLElement} element
+ * @param {HTMLElement} progressContainer
+ * @param {HTMLElement} statusMessage
+ * @param {Object} [options]
+ * @param {string} [options.dragOverClass] - CSS class added during drag-over
  */
 function initDragAndDrop(element, progressContainer, statusMessage, options = {}) {
   if (!element) {
-    console.error("Element not found for drag-and-drop initialization");
+    console.error('[DocuElevate] Element not found for drag-and-drop initialization');
     return;
   }
 
-  // Add event listeners for drag-and-drop
-  element.addEventListener("dragover", (e) => {
+  element.addEventListener('dragover', (e) => {
     e.preventDefault();
     e.stopPropagation();
-    e.dataTransfer.dropEffect = "copy";
-
-    // Add visual feedback
-    if (options.dragOverClass) {
-      element.classList.add(options.dragOverClass);
-    }
+    e.dataTransfer.dropEffect = 'copy';
+    if (options.dragOverClass) element.classList.add(options.dragOverClass);
   });
 
-  element.addEventListener("dragleave", (e) => {
+  element.addEventListener('dragleave', (e) => {
     e.preventDefault();
     e.stopPropagation();
-
-    // Remove visual feedback
-    if (options.dragOverClass) {
-      element.classList.remove(options.dragOverClass);
-    }
+    if (options.dragOverClass) element.classList.remove(options.dragOverClass);
   });
 
-  element.addEventListener("drop", (e) => {
+  element.addEventListener('drop', async (e) => {
     e.preventDefault();
     e.stopPropagation();
+    if (options.dragOverClass) element.classList.remove(options.dragOverClass);
 
-    // Remove visual feedback
-    if (options.dragOverClass) {
-      element.classList.remove(options.dragOverClass);
-    }
-
-    if (e.dataTransfer.files.length) {
-      processFiles(e.dataTransfer.files, progressContainer, statusMessage);
-    }
+    const files = await getFilesFromDataTransfer(e.dataTransfer);
+    if (files.length > 0) processFiles(files, progressContainer, statusMessage);
   });
 }
+
+

frontend/templates/files.html

@@ -3,6 +3,12 @@
 
 {% block head_extra %}
   <script src="/static/js/common.js"></script>
+  <script>
+    window.uploadConfig = {
+      concurrency: {{ upload_concurrency }},
+      queueDelayMs: {{ upload_queue_delay_ms }}
+    };
+  </script>
   <script src="/static/js/upload.js"></script>
   <style>
     /* Drop overlay styles - covers entire page for drag-and-drop anywhere */
@@ -342,8 +348,8 @@
 <div id="dropOverlay" class="drop-overlay">
   <div class="drop-message">
     <i class="fas fa-cloud-upload-alt"></i>
-    <p>Drop files anywhere to upload</p>
-    <div class="drop-hint">Supports PDF, Office docs, images, and more</div>
+    <p>Drop files or folders anywhere to upload</p>
+    <div class="drop-hint">Supports PDF, Office docs, images, HTML, Markdown, and more – folders are processed recursively</div>
   </div>
 </div>
 
@@ -902,18 +908,19 @@
       e.dataTransfer.dropEffect = 'copy';
     });
 
-    window.addEventListener('drop', (e) => {
+    window.addEventListener('drop', async (e) => {
       e.preventDefault();
       dragCounter = 0;
       dropOverlay.classList.remove('active');
 
-      if (e.dataTransfer.files.length > 0) {
+      const files = await getFilesFromDataTransfer(e.dataTransfer);
+      if (files.length > 0) {
         // Show upload modal
         uploadModal.classList.add('active');
         uploadProgressContainer.innerHTML = '';
 
-        // Process the dropped files
-        processFiles(e.dataTransfer.files, uploadProgressContainer, uploadStatusMessage);
+        // Process the dropped files (queue-based, with adaptive throttling)
+        processFiles(files, uploadProgressContainer, uploadStatusMessage);
       }
     });
 

frontend/templates/upload.html

@@ -12,12 +12,11 @@
       <div
         id="dropZone"
         class="border-4 border-dashed border-gray-300 rounded-lg p-6 sm:p-8 bg-white text-center w-full"
-        ondrop="handleDrop(event)"
         ondragover="handleDragOver(event)"
         ondragleave="handleDragLeave(event)"
       >
         <p class="text-gray-500 mb-4 hidden sm:block">
-          Drag &amp; drop files here, or click to select files.
+          Drag &amp; drop files or folders here, or click to select files.
         </p>
         <p class="text-gray-500 mb-4 sm:hidden">
           Tap to select files or use the camera button below.
@@ -42,7 +41,7 @@
         </button>
         <div class="text-sm text-gray-500 mt-2">
           <p>Allowed types: PDF, Office documents (Word, Excel, PowerPoint, etc.), Images</p>
-          <p>Maximum size: 500MB per file</p>
+          <p>Maximum size: 500MB per file &bull; Directories are uploaded recursively</p>
         </div>
       </div>
 
@@ -122,6 +121,13 @@
 {% endblock %}
 
 {% block scripts %}
+<script>
+  // Expose server-side upload throttling configuration to upload.js
+  window.uploadConfig = {
+    concurrency: {{ upload_concurrency }},
+    queueDelayMs: {{ upload_queue_delay_ms }}
+  };
+</script>
 <script src="/static/js/upload.js"></script>
 <script>
   // Drag-and-drop / file input logic
@@ -143,10 +149,12 @@
     dropZone.classList.remove("bg-gray-100");
   }
 
-  function handleDrop(e) {
+  dropZone.addEventListener("drop", async (e) => {
     e.preventDefault();
     dropZone.classList.remove("bg-gray-100");
-    if (e.dataTransfer.files.length) {
+
+    const files = await getFilesFromDataTransfer(e.dataTransfer);
+    if (files.length) {
       // Clear previous upload progress
       uploadProgress.innerHTML = "";
 
@@ -155,10 +163,10 @@
       progressContainer.className = "space-y-2";
       uploadProgress.appendChild(progressContainer);
 
-      // Use the shared processFiles function
-      processFiles(e.dataTransfer.files, progressContainer, statusMessage);
+      // Use the shared processFiles function (queue-based)
+      processFiles(files, progressContainer, statusMessage);
     }
-  }
+  });
 
   function handleFileSelect(e) {
     if (e.target.files.length) {
@@ -171,7 +179,7 @@
       uploadProgress.appendChild(progressContainer);
 
       // Use the shared processFiles function
-      processFiles(e.target.files, progressContainer, statusMessage);
+      processFiles(Array.from(e.target.files), progressContainer, statusMessage);
     }
   }
 

tests/test_allowed_types.py

@@ -0,0 +1,211 @@
+"""Tests for app/utils/allowed_types.py – canonical Gotenberg file-type lists."""
+
+import pytest
+
+from app.utils.allowed_types import (
+    ALLOWED_EXTENSIONS,
+    ALLOWED_MIME_TYPES,
+    DOCUMENT_MIME_TYPES,
+    IMAGE_MIME_TYPES,
+)
+
+
+@pytest.mark.unit
+class TestAllowedTypesStructure:
+    """Structural sanity checks on the exported sets."""
+
+    def test_allowed_mime_types_is_union(self):
+        """ALLOWED_MIME_TYPES must equal DOCUMENT_MIME_TYPES ∪ IMAGE_MIME_TYPES."""
+        assert ALLOWED_MIME_TYPES == DOCUMENT_MIME_TYPES | IMAGE_MIME_TYPES
+
+    def test_sets_are_disjoint(self):
+        """DOCUMENT_MIME_TYPES and IMAGE_MIME_TYPES should not overlap."""
+        assert DOCUMENT_MIME_TYPES.isdisjoint(IMAGE_MIME_TYPES)
+
+    def test_extensions_have_leading_dot(self):
+        """All entries in ALLOWED_EXTENSIONS must start with '.'."""
+        for ext in ALLOWED_EXTENSIONS:
+            assert ext.startswith("."), f"Extension without leading dot: {ext!r}"
+
+    def test_extensions_are_lowercase(self):
+        """All entries in ALLOWED_EXTENSIONS must be lower-case."""
+        for ext in ALLOWED_EXTENSIONS:
+            assert ext == ext.lower(), f"Non-lowercase extension: {ext!r}"
+
+    def test_no_empty_entries(self):
+        """No set should contain empty strings."""
+        for s in (DOCUMENT_MIME_TYPES, IMAGE_MIME_TYPES, ALLOWED_EXTENSIONS):
+            assert "" not in s
+
+
+@pytest.mark.unit
+class TestGotenbergCoverageDocuments:
+    """Verify that every extension Gotenberg handles is present."""
+
+    # These must match OFFICE_EXTENSIONS in app/tasks/convert_to_pdf.py
+    _office_extensions = {
+        ".doc",
+        ".docx",
+        ".docm",
+        ".dot",
+        ".dotx",
+        ".dotm",
+        ".xls",
+        ".xlsx",
+        ".xlsm",
+        ".xlsb",
+        ".xlt",
+        ".xltx",
+        ".xlw",
+        ".ppt",
+        ".pptx",
+        ".pptm",
+        ".pps",
+        ".ppsx",
+        ".pot",
+        ".potx",
+        ".odt",
+        ".ods",
+        ".odp",
+        ".odg",
+        ".odf",
+        ".rtf",
+        ".txt",
+        ".csv",
+    }
+    _image_extensions = {
+        ".jpg",
+        ".jpeg",
+        ".png",
+        ".gif",
+        ".bmp",
+        ".tiff",
+        ".tif",
+        ".webp",
+        ".svg",
+    }
+    _html_extensions = {".html", ".htm"}
+    _markdown_extensions = {".md", ".markdown"}
+
+    def test_office_extensions_all_present(self):
+        missing = self._office_extensions - ALLOWED_EXTENSIONS
+        assert not missing, f"Office extensions missing from ALLOWED_EXTENSIONS: {missing}"
+
+    def test_image_extensions_all_present(self):
+        missing = self._image_extensions - ALLOWED_EXTENSIONS
+        assert not missing, f"Image extensions missing from ALLOWED_EXTENSIONS: {missing}"
+
+    def test_html_extensions_present(self):
+        missing = self._html_extensions - ALLOWED_EXTENSIONS
+        assert not missing, f"HTML extensions missing from ALLOWED_EXTENSIONS: {missing}"
+
+    def test_markdown_extensions_present(self):
+        missing = self._markdown_extensions - ALLOWED_EXTENSIONS
+        assert not missing, f"Markdown extensions missing from ALLOWED_EXTENSIONS: {missing}"
+
+    def test_pdf_extension_present(self):
+        assert ".pdf" in ALLOWED_EXTENSIONS
+
+
+@pytest.mark.unit
+class TestGotenbergCoverageMimeTypes:
+    """Verify that key MIME types are present."""
+
+    @pytest.mark.parametrize(
+        "mime_type",
+        [
+            "application/pdf",
+            "application/msword",
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            "application/vnd.ms-excel",
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+            "application/vnd.ms-powerpoint",
+            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+            # OpenDocument
+            "application/vnd.oasis.opendocument.text",
+            "application/vnd.oasis.opendocument.spreadsheet",
+            "application/vnd.oasis.opendocument.presentation",
+            # Images
+            "image/jpeg",
+            "image/png",
+            "image/tiff",
+            "image/webp",
+            "image/svg+xml",
+            # Text
+            "text/plain",
+            "text/csv",
+            "text/html",
+            "text/markdown",
+            "text/x-markdown",
+            # RTF
+            "application/rtf",
+            "text/rtf",
+        ],
+    )
+    def test_mime_type_in_allowed(self, mime_type):
+        assert mime_type in ALLOWED_MIME_TYPES, f"{mime_type} not in ALLOWED_MIME_TYPES"
+
+    @pytest.mark.parametrize(
+        "mime_type",
+        [
+            # New extended Office variants
+            "application/vnd.ms-word.document.macroEnabled.12",
+            "application/vnd.ms-word.template.macroEnabled.12",
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
+            "application/vnd.ms-excel.sheet.macroEnabled.12",
+            "application/vnd.ms-excel.sheet.binary.macroEnabled.12",
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
+            "application/vnd.openxmlformats-officedocument.presentationml.template",
+            "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
+            "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
+            "application/vnd.oasis.opendocument.graphics",
+            "application/vnd.oasis.opendocument.formula",
+        ],
+    )
+    def test_extended_mime_type_in_allowed(self, mime_type):
+        """Extended / macro-enabled Office and ODF types must be present."""
+        assert mime_type in ALLOWED_MIME_TYPES, f"{mime_type} not in ALLOWED_MIME_TYPES"
+
+
+@pytest.mark.unit
+class TestUploadViewsPassConfig:
+    """Verify that the upload views inject uploadConfig into their templates."""
+
+    def test_upload_page_contains_upload_config(self, client):
+        """The /upload page must render window.uploadConfig with numeric values."""
+        response = client.get("/upload")
+        assert response.status_code == 200
+        content = response.text
+        assert "window.uploadConfig" in content
+        # Verify that literal integers are rendered (not the placeholder names)
+        assert "concurrency: 3" in content or "concurrency:" in content
+        assert "queueDelayMs: 500" in content or "queueDelayMs:" in content
+
+    def test_files_page_contains_upload_config(self, client):
+        """The /files page must render window.uploadConfig with numeric values."""
+        response = client.get("/files")
+        assert response.status_code == 200
+        content = response.text
+        assert "window.uploadConfig" in content
+        assert "concurrency: 3" in content or "concurrency:" in content
+        assert "queueDelayMs: 500" in content or "queueDelayMs:" in content
+
+    def test_upload_config_has_numeric_concurrency(self, client):
+        """Concurrency in uploadConfig must be a positive integer."""
+        import re
+
+        response = client.get("/upload")
+        assert response.status_code == 200
+        match = re.search(r"concurrency:\s*(\d+)", response.text)
+        assert match is not None, "concurrency integer not found in window.uploadConfig"
+        assert int(match.group(1)) > 0
+
+    def test_upload_config_has_numeric_delay(self, client):
+        """queueDelayMs in uploadConfig must be a non-negative integer."""
+        import re
+
+        response = client.get("/upload")
+        assert response.status_code == 200
+        match = re.search(r"queueDelayMs:\s*(\d+)", response.text)
+        assert match is not None, "queueDelayMs integer not found in window.uploadConfig"
+        assert int(match.group(1)) >= 0

tests/test_files_view.py

@@ -90,4 +90,4 @@ class TestFilesView:
 
         # Check for drag-and-drop event handlers
         assert "dragenter" in content or "drag" in content, "Drag event handlers should be present"
-        assert "Drop files anywhere to upload" in content, "Drop message should be present"
+        assert "Drop files or folders anywhere to upload" in content, "Drop message should be present"