feat(upload): add directory traversal, queue throttling, and upload config settings

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/config.py

@@ -232,6 +232,22 @@ class Settings(BaseSettings):
         description="Delay in seconds between each task submission when throttling in /processall",
     )
 
+    # Client-side upload throttling settings (applied when uploading files via the web UI)
+    upload_concurrency: int = Field(
+        default=3,
+        description=(
+            "Maximum number of files uploaded simultaneously from the browser. "
+            "Limits parallel uploads to prevent API overload when dragging directories. Default: 3."
+        ),
+    )
+    upload_queue_delay_ms: int = Field(
+        default=500,
+        description=(
+            "Delay in milliseconds between starting each upload slot when queue is active. "
+            "Staggers upload starts to smooth out server load. Default: 500 ms."
+        ),
+    )
+
     # Notification settings
     notification_urls: Union[List[str], str] = Field(
         default_factory=list,

app/utils/settings_service.py

@@ -1057,6 +1057,28 @@ SETTING_METADATA = {
         "required": False,
         "restart_required": False,
     },
+    "upload_concurrency": {
+        "category": "Processing",
+        "description": (
+            "Maximum number of files uploaded simultaneously from the browser. "
+            "Limits parallel uploads to prevent API overload when dragging directories. Default: 3."
+        ),
+        "type": "integer",
+        "sensitive": False,
+        "required": False,
+        "restart_required": False,
+    },
+    "upload_queue_delay_ms": {
+        "category": "Processing",
+        "description": (
+            "Delay in milliseconds between starting each upload slot when queue is active. "
+            "Staggers upload starts to smooth out server load. Default: 500 ms."
+        ),
+        "type": "integer",
+        "sensitive": False,
+        "required": False,
+        "restart_required": False,
+    },
     "enable_text_quality_check": {
         "category": "Processing",
         "description": (

app/views/files.py

@@ -17,6 +17,26 @@ router = APIRouter()
 _FILE_NOT_FOUND = "File not found"
 
 
+def _get_upload_concurrency() -> int:
+    """Return the configured upload concurrency (falls back to default on error)."""
+    try:
+        from app.config import settings
+
+        return settings.upload_concurrency
+    except Exception:
+        return 3
+
+
+def _get_upload_queue_delay_ms() -> int:
+    """Return the configured upload queue delay in ms (falls back to default on error)."""
+    try:
+        from app.config import settings
+
+        return settings.upload_queue_delay_ms
+    except Exception:
+        return 500
+
+
 @router.get("/files")
 @require_login
 def files_page(
@@ -111,6 +131,8 @@ def files_page(
                 "mime_type": mime_type or "",
                 "status": status or "",
                 "mime_types": mime_types,
+                "upload_concurrency": _get_upload_concurrency(),
+                "upload_queue_delay_ms": _get_upload_queue_delay_ms(),
             },
         )
     except Exception as e:
@@ -124,6 +146,8 @@ def files_page(
                 "files": [],
                 "pagination": {"page": 1, "per_page": per_page, "total_items": 0, "total_pages": 0},
                 "error": str(e),
+                "upload_concurrency": _get_upload_concurrency(),
+                "upload_queue_delay_ms": _get_upload_queue_delay_ms(),
             },
         )
 

app/views/general.py

@@ -99,7 +99,16 @@ async def serve_imprint(request: Request):
 @require_login
 async def serve_upload(request: Request):
     """Serve the upload page."""
-    return templates.TemplateResponse("upload.html", {"request": request})
+    from app.config import settings
+
+    return templates.TemplateResponse(
+        "upload.html",
+        {
+            "request": request,
+            "upload_concurrency": settings.upload_concurrency,
+            "upload_queue_delay_ms": settings.upload_queue_delay_ms,
+        },
+    )
 
 
 @router.get("/favicon.ico", include_in_schema=False)

frontend/static/js/upload.js

@@ -4,6 +4,10 @@
 // Configuration
 const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MB
 
+// Upload throttling defaults (overridden by window.uploadConfig when available)
+const DEFAULT_UPLOAD_CONCURRENCY = 3;
+const DEFAULT_UPLOAD_QUEUE_DELAY_MS = 500;
+
 // Allowed file types
 const ACCEPTED_TYPES = {
   // PDF files
@@ -49,17 +53,101 @@ const ACCEPTED_EXTENSIONS = [
   '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg', '.md'
 ];
 
+// ---------------------------------------------------------------------------
+// Directory traversal helpers (FileSystem Access API)
+// ---------------------------------------------------------------------------
+
 /**
- * Process a list of files for upload
- * @param {FileList} files - Files to process
+ * Read all file entries from a DirectoryReader, handling the 100-entry limit
+ * by calling readEntries() repeatedly until it returns an empty batch.
+ * @param {FileSystemDirectoryReader} reader
+ * @returns {Promise<FileSystemEntry[]>}
+ */
+function readAllDirectoryEntries(reader) {
+  return new Promise((resolve, reject) => {
+    const entries = [];
+    function readBatch() {
+      reader.readEntries((batch) => {
+        if (batch.length === 0) {
+          resolve(entries);
+        } else {
+          entries.push(...batch);
+          readBatch();
+        }
+      }, reject);
+    }
+    readBatch();
+  });
+}
+
+/**
+ * Recursively collect all File objects from a FileSystemEntry tree.
+ * @param {FileSystemEntry} entry
+ * @param {File[]} files  - accumulator array
+ * @returns {Promise<void>}
+ */
+async function traverseFileEntry(entry, files) {
+  if (entry.isFile) {
+    await new Promise((resolve) => {
+      entry.file((file) => { files.push(file); resolve(); }, resolve);
+    });
+  } else if (entry.isDirectory) {
+    const reader = entry.createReader();
+    const subEntries = await readAllDirectoryEntries(reader);
+    for (const sub of subEntries) {
+      await traverseFileEntry(sub, files);
+    }
+  }
+}
+
+/**
+ * Extract all File objects from a DataTransfer, recursively expanding any
+ * dropped directories.  Falls back gracefully to dataTransfer.files when
+ * the FileSystem Entry API is unavailable.
+ * @param {DataTransfer} dataTransfer
+ * @returns {Promise<File[]>}
+ */
+async function getFilesFromDataTransfer(dataTransfer) {
+  // Use the FileSystem Entry API when available (all modern browsers)
+  if (dataTransfer.items && dataTransfer.items.length > 0) {
+    const files = [];
+    const itemList = dataTransfer.items;
+    for (let i = 0; i < itemList.length; i++) {
+      const item = itemList[i];
+      const entry = item.webkitGetAsEntry ? item.webkitGetAsEntry() : null;
+      if (entry) {
+        await traverseFileEntry(entry, files);
+      } else if (item.kind === 'file') {
+        const file = item.getAsFile();
+        if (file) files.push(file);
+      }
+    }
+    return files;
+  }
+  // Fallback: plain FileList (no directory support)
+  return Array.from(dataTransfer.files || []);
+}
+
+// ---------------------------------------------------------------------------
+// Queue-based upload runner
+// ---------------------------------------------------------------------------
+
+/**
+ * Upload a list of files using a concurrency-limited queue with a configurable
+ * delay between slot starts to prevent server overload.
+ *
+ * @param {File[]} files - Files to upload
  * @param {HTMLElement} progressContainer - Container element for progress display
  * @param {HTMLElement} statusMessage - Element for status message display
  */
 function processFiles(files, progressContainer, statusMessage) {
   if (files.length === 0) return;
 
+  const concurrency = (window.uploadConfig && window.uploadConfig.concurrency) || DEFAULT_UPLOAD_CONCURRENCY;
+  const delayMs = (window.uploadConfig && window.uploadConfig.queueDelayMs) || DEFAULT_UPLOAD_QUEUE_DELAY_MS;
+
   if (statusMessage) {
-    statusMessage.textContent = `Processing ${files.length} file(s)...`;
+    statusMessage.textContent = `Queued ${files.length} file(s) for upload…`;
   }
 
   // Clear previous upload progress
@@ -67,15 +155,79 @@ function processFiles(files, progressContainer, statusMessage) {
     progressContainer.innerHTML = "";
   }
 
-  // Process each file
-  for (let i = 0; i < files.length; i++) {
-    const file = files[i];
-    validateAndUpload(file, progressContainer, statusMessage);
+  // Pre-create all progress elements so the user sees the full list immediately
+  const progressElements = files.map((file) => {
+    const fileProgress = document.createElement("div");
+    fileProgress.className = "flex flex-col mb-2";
+    fileProgress.innerHTML = `
+      <div class="flex justify-between">
+        <span class="text-sm truncate" title="${file.name}">${file.name}</span>
+        <span class="text-xs text-gray-500">${formatFileSize(file.size)}</span>
+      </div>
+      <div class="w-full bg-gray-200 h-2 rounded-full mt-1">
+        <div class="file-progress-bar bg-gray-300 h-2 rounded-full" style="width: 0%"></div>
+      </div>
+      <div class="file-status text-xs text-gray-400 mt-1">Queued</div>
+    `;
+    if (progressContainer) progressContainer.appendChild(fileProgress);
+    return fileProgress;
+  });
+
+  // Queue runner
+  let index = 0;
+  let active = 0;
+
+  function startNext() {
+    while (active < concurrency && index < files.length) {
+      const i = index++;
+      active++;
+      const progressBar = progressElements[i].querySelector(".file-progress-bar");
+      const statusEl = progressElements[i].querySelector(".file-status");
+      validateAndUploadQueued(files[i], progressBar, statusEl, statusMessage).finally(() => {
+        active--;
+        setTimeout(startNext, delayMs);
+      });
+    }
   }
+
+  startNext();
 }
 
 /**
- * Validate and upload a single file
+ * Validate and upload a single file (used by the queue runner).
+ * @param {File} file
+ * @param {HTMLElement} progressBar
+ * @param {HTMLElement} statusEl
+ * @param {HTMLElement} statusMessage
+ * @returns {Promise<void>}
+ */
+function validateAndUploadQueued(file, progressBar, statusEl, statusMessage) {
+  // Validate file type by checking both MIME type and extension
+  const isValidMimeType = ACCEPTED_TYPES[file.type] || false;
+  const fileExtension = '.' + file.name.split('.').pop().toLowerCase();
+  const isValidExtension = ACCEPTED_EXTENSIONS.includes(fileExtension);
+
+  if (!isValidMimeType && !isValidExtension) {
+    statusEl.textContent = `Unsupported file type`;
+    statusEl.className = "text-xs text-red-500 mt-1";
+    progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
+    updateOverallStatus(statusMessage);
+    return Promise.resolve();
+  }
+
+  if (file.size > MAX_FILE_SIZE) {
+    statusEl.textContent = `Exceeds 500 MB limit`;
+    statusEl.className = "text-xs text-red-500 mt-1";
+    progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
+    updateOverallStatus(statusMessage);
+    return Promise.resolve();
+  }
+
+  return uploadFile(file, progressBar, statusEl, statusMessage);
+}
+
+/**
+ * Validate and upload a single file (legacy entry point kept for compatibility).
  * @param {File} file - File to validate and upload
  * @param {HTMLElement} progressContainer - Container element for progress display
  * @param {HTMLElement} statusMessage - Element for status message display
@@ -102,26 +254,7 @@ function validateAndUpload(file, progressContainer, statusMessage) {
   const progressBar = fileProgress.querySelector(".file-progress-bar");
   const statusEl = fileProgress.querySelector(".file-status");
 
-  // Validate file type by checking both MIME type and extension
-  const isValidMimeType = ACCEPTED_TYPES[file.type] || false;
-  const fileExtension = '.' + file.name.split('.').pop().toLowerCase();
-  const isValidExtension = ACCEPTED_EXTENSIONS.includes(fileExtension);
-
-  if (!isValidMimeType && !isValidExtension) {
-    statusEl.textContent = `Error: ${file.name} - Unsupported file type`;
-    statusEl.className = "text-xs text-red-500 mt-1";
-    return;
-  }
-
-  // Validate file size
-  if (file.size > MAX_FILE_SIZE) {
-    statusEl.textContent = `Error: ${file.name} - File size exceeds 500MB limit`;
-    statusEl.className = "text-xs text-red-500 mt-1";
-    return;
-  }
-
-  // Upload the file
-  uploadFile(file, progressBar, statusEl, statusMessage);
+  validateAndUploadQueued(file, progressBar, statusEl, statusMessage);
 }
 
 /**
@@ -130,55 +263,68 @@ function validateAndUpload(file, progressContainer, statusMessage) {
  * @param {HTMLElement} progressBar - Progress bar element
  * @param {HTMLElement} statusEl - Status element
  * @param {HTMLElement} statusMessage - Overall status message element
+ * @returns {Promise<void>}
  */
-async function uploadFile(file, progressBar, statusEl, statusMessage) {
-  statusEl.textContent = `Uploading...`;
-  try {
-    let formData = new FormData();
-    formData.append("file", file);
+function uploadFile(file, progressBar, statusEl, statusMessage) {
+  statusEl.textContent = `Uploading…`;
+  statusEl.className = "text-xs text-gray-600 mt-1";
+  progressBar.className = "file-progress-bar bg-blue-500 h-2 rounded-full";
 
-    const xhr = new XMLHttpRequest();
-    xhr.open("POST", "/api/ui-upload", true);
+  return new Promise((resolve) => {
+    try {
+      const formData = new FormData();
+      formData.append("file", file);
 
-    // Attach CSRF token so the server-side CSRF middleware accepts the request.
-    const csrfToken = typeof getCsrfToken === 'function' ? getCsrfToken() : '';
-    if (csrfToken) {
-      xhr.setRequestHeader("X-CSRF-Token", csrfToken);
-    }
+      const xhr = new XMLHttpRequest();
+      xhr.open("POST", "/api/ui-upload", true);
 
-    xhr.upload.onprogress = (e) => {
-      if (e.lengthComputable) {
-        const percentComplete = (e.loaded / e.total) * 100;
-        progressBar.style.width = percentComplete + "%";
-        statusEl.textContent = `Uploading: ${Math.round(percentComplete)}%`;
+      // Attach CSRF token so the server-side CSRF middleware accepts the request.
+      const csrfToken = typeof getCsrfToken === 'function' ? getCsrfToken() : '';
+      if (csrfToken) {
+        xhr.setRequestHeader("X-CSRF-Token", csrfToken);
       }
-    };
 
-    xhr.onload = function() {
-      if (xhr.status === 200) {
-        const result = JSON.parse(xhr.responseText);
-        progressBar.style.width = "100%";
-        progressBar.className = "file-progress-bar bg-green-500 h-2 rounded-full";
-        statusEl.textContent = `Success: Task ID: ${result.task_id}`;
-        statusEl.className = "text-xs text-green-600 mt-1";
+      xhr.upload.onprogress = (e) => {
+        if (e.lengthComputable) {
+          const percentComplete = (e.loaded / e.total) * 100;
+          progressBar.style.width = percentComplete + "%";
+          statusEl.textContent = `Uploading: ${Math.round(percentComplete)}%`;
+        }
+      };
+
+      xhr.onload = function () {
+        if (xhr.status === 200) {
+          const result = JSON.parse(xhr.responseText);
+          progressBar.style.width = "100%";
+          progressBar.className = "file-progress-bar bg-green-500 h-2 rounded-full";
+          statusEl.textContent = `Success: Task ID: ${result.task_id}`;
+          statusEl.className = "text-xs text-green-600 mt-1";
+        } else {
+          progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
+          statusEl.textContent = `Error: Upload failed (HTTP ${xhr.status})`;
+          statusEl.className = "text-xs text-red-500 mt-1";
+        }
         updateOverallStatus(statusMessage);
-      } else {
-        throw new Error(`Upload failed with status ${xhr.status}`);
-      }
-    };
+        resolve();
+      };
 
-    xhr.onerror = function() {
-      throw new Error("Network error occurred");
-    };
+      xhr.onerror = function () {
+        progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
+        statusEl.textContent = `Error: Network error`;
+        statusEl.className = "text-xs text-red-500 mt-1";
+        updateOverallStatus(statusMessage);
+        resolve();
+      };
 
-    xhr.send(formData);
-
-  } catch (err) {
-    statusEl.textContent = `Error: ${err.message}`;
-    statusEl.className = "text-xs text-red-500 mt-1";
-    progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
-    updateOverallStatus(statusMessage);
-  }
+      xhr.send(formData);
+    } catch (err) {
+      statusEl.textContent = `Error: ${err.message}`;
+      statusEl.className = "text-xs text-red-500 mt-1";
+      progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
+      updateOverallStatus(statusMessage);
+      resolve();
+    }
+  });
 }
 
 /**
@@ -191,10 +337,10 @@ function updateOverallStatus(statusMessage) {
   // Count success/failure
   const fileStatuses = document.querySelectorAll('.file-status');
   let completed = 0;
-  let total = fileStatuses.length;
+  const total = fileStatuses.length;
 
   fileStatuses.forEach(status => {
-    if (status.textContent.includes('Success') || status.textContent.includes('Error')) {
+    if (status.textContent.includes('Success') || status.textContent.includes('Error') || status.textContent.includes('Unsupported') || status.textContent.includes('Exceeds')) {
       completed++;
     }
   });
@@ -226,7 +372,7 @@ function formatFileSize(bytes) {
 }
 
 /**
- * Initialize drag-and-drop on an element
+ * Initialize drag-and-drop on an element with directory support.
  * @param {HTMLElement} element - Element to enable drag-and-drop on
  * @param {HTMLElement} progressContainer - Container for progress display
  * @param {HTMLElement} statusMessage - Element for status messages
@@ -260,7 +406,7 @@ function initDragAndDrop(element, progressContainer, statusMessage, options = {}
     }
   });
 
-  element.addEventListener("drop", (e) => {
+  element.addEventListener("drop", async (e) => {
     e.preventDefault();
     e.stopPropagation();
 
@@ -269,8 +415,10 @@ function initDragAndDrop(element, progressContainer, statusMessage, options = {}
       element.classList.remove(options.dragOverClass);
     }
 
-    if (e.dataTransfer.files.length) {
-      processFiles(e.dataTransfer.files, progressContainer, statusMessage);
+    const files = await getFilesFromDataTransfer(e.dataTransfer);
+    if (files.length > 0) {
+      processFiles(files, progressContainer, statusMessage);
     }
   });
 }
+

frontend/templates/upload.html

@@ -12,12 +12,11 @@
       <div
         id="dropZone"
         class="border-4 border-dashed border-gray-300 rounded-lg p-6 sm:p-8 bg-white text-center w-full"
-        ondrop="handleDrop(event)"
         ondragover="handleDragOver(event)"
         ondragleave="handleDragLeave(event)"
       >
         <p class="text-gray-500 mb-4 hidden sm:block">
-          Drag &amp; drop files here, or click to select files.
+          Drag &amp; drop files or folders here, or click to select files.
         </p>
         <p class="text-gray-500 mb-4 sm:hidden">
           Tap to select files or use the camera button below.
@@ -42,7 +41,7 @@
         </button>
         <div class="text-sm text-gray-500 mt-2">
           <p>Allowed types: PDF, Office documents (Word, Excel, PowerPoint, etc.), Images</p>
-          <p>Maximum size: 500MB per file</p>
+          <p>Maximum size: 500MB per file &bull; Directories are uploaded recursively</p>
         </div>
       </div>
 
@@ -122,6 +121,13 @@
 {% endblock %}
 
 {% block scripts %}
+<script>
+  // Expose server-side upload throttling configuration to upload.js
+  window.uploadConfig = {
+    concurrency: {{ upload_concurrency }},
+    queueDelayMs: {{ upload_queue_delay_ms }}
+  };
+</script>
 <script src="/static/js/upload.js"></script>
 <script>
   // Drag-and-drop / file input logic
@@ -143,10 +149,12 @@
     dropZone.classList.remove("bg-gray-100");
   }
 
-  function handleDrop(e) {
+  dropZone.addEventListener("drop", async (e) => {
     e.preventDefault();
     dropZone.classList.remove("bg-gray-100");
-    if (e.dataTransfer.files.length) {
+
+    const files = await getFilesFromDataTransfer(e.dataTransfer);
+    if (files.length) {
       // Clear previous upload progress
       uploadProgress.innerHTML = "";
 
@@ -155,10 +163,10 @@
       progressContainer.className = "space-y-2";
       uploadProgress.appendChild(progressContainer);
 
-      // Use the shared processFiles function
-      processFiles(e.dataTransfer.files, progressContainer, statusMessage);
+      // Use the shared processFiles function (queue-based)
+      processFiles(files, progressContainer, statusMessage);
     }
-  }
+  });
 
   function handleFileSelect(e) {
     if (e.target.files.length) {
@@ -171,7 +179,7 @@
       uploadProgress.appendChild(progressContainer);
 
       // Use the shared processFiles function
-      processFiles(e.target.files, progressContainer, statusMessage);
+      processFiles(Array.from(e.target.files), progressContainer, statusMessage);
     }
   }