From 35d2ed05e9ed589cc70ced4d1b76dc3b9a85ca41 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Feb 2026 23:11:44 +0000 Subject: [PATCH] feat(upload): add directory traversal, queue throttling, and upload config settings Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- app/config.py | 16 ++ app/utils/settings_service.py | 22 +++ app/views/files.py | 24 +++ app/views/general.py | 11 +- frontend/static/js/upload.js | 296 ++++++++++++++++++++++++--------- frontend/templates/upload.html | 26 ++- 6 files changed, 311 insertions(+), 84 deletions(-) diff --git a/app/config.py b/app/config.py index 7f5ccbc8..c45f651b 100644 --- a/app/config.py +++ b/app/config.py @@ -232,6 +232,22 @@ class Settings(BaseSettings): description="Delay in seconds between each task submission when throttling in /processall", ) + # Client-side upload throttling settings (applied when uploading files via the web UI) + upload_concurrency: int = Field( + default=3, + description=( + "Maximum number of files uploaded simultaneously from the browser. " + "Limits parallel uploads to prevent API overload when dragging directories. Default: 3." + ), + ) + upload_queue_delay_ms: int = Field( + default=500, + description=( + "Delay in milliseconds between starting each upload slot when queue is active. " + "Staggers upload starts to smooth out server load. Default: 500 ms." + ), + ) + # Notification settings notification_urls: Union[List[str], str] = Field( default_factory=list, diff --git a/app/utils/settings_service.py b/app/utils/settings_service.py index 41faacc8..c27642a0 100644 --- a/app/utils/settings_service.py +++ b/app/utils/settings_service.py @@ -1057,6 +1057,28 @@ SETTING_METADATA = { "required": False, "restart_required": False, }, + "upload_concurrency": { + "category": "Processing", + "description": ( + "Maximum number of files uploaded simultaneously from the browser. " + "Limits parallel uploads to prevent API overload when dragging directories. Default: 3." + ), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "upload_queue_delay_ms": { + "category": "Processing", + "description": ( + "Delay in milliseconds between starting each upload slot when queue is active. " + "Staggers upload starts to smooth out server load. Default: 500 ms." + ), + "type": "integer", + "sensitive": False, + "required": False, + "restart_required": False, + }, "enable_text_quality_check": { "category": "Processing", "description": ( diff --git a/app/views/files.py b/app/views/files.py index d2395dc7..0644d28a 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -17,6 +17,26 @@ router = APIRouter() _FILE_NOT_FOUND = "File not found" +def _get_upload_concurrency() -> int: + """Return the configured upload concurrency (falls back to default on error).""" + try: + from app.config import settings + + return settings.upload_concurrency + except Exception: + return 3 + + +def _get_upload_queue_delay_ms() -> int: + """Return the configured upload queue delay in ms (falls back to default on error).""" + try: + from app.config import settings + + return settings.upload_queue_delay_ms + except Exception: + return 500 + + @router.get("/files") @require_login def files_page( @@ -111,6 +131,8 @@ def files_page( "mime_type": mime_type or "", "status": status or "", "mime_types": mime_types, + "upload_concurrency": _get_upload_concurrency(), + "upload_queue_delay_ms": _get_upload_queue_delay_ms(), }, ) except Exception as e: @@ -124,6 +146,8 @@ def files_page( "files": [], "pagination": {"page": 1, "per_page": per_page, "total_items": 0, "total_pages": 0}, "error": str(e), + "upload_concurrency": _get_upload_concurrency(), + "upload_queue_delay_ms": _get_upload_queue_delay_ms(), }, ) diff --git a/app/views/general.py b/app/views/general.py index 26c45580..e6d70f25 100644 --- a/app/views/general.py +++ b/app/views/general.py @@ -99,7 +99,16 @@ async def serve_imprint(request: Request): @require_login async def serve_upload(request: Request): """Serve the upload page.""" - return templates.TemplateResponse("upload.html", {"request": request}) + from app.config import settings + + return templates.TemplateResponse( + "upload.html", + { + "request": request, + "upload_concurrency": settings.upload_concurrency, + "upload_queue_delay_ms": settings.upload_queue_delay_ms, + }, + ) @router.get("/favicon.ico", include_in_schema=False) diff --git a/frontend/static/js/upload.js b/frontend/static/js/upload.js index 1cb92074..9212ca9a 100644 --- a/frontend/static/js/upload.js +++ b/frontend/static/js/upload.js @@ -4,6 +4,10 @@ // Configuration const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MB +// Upload throttling defaults (overridden by window.uploadConfig when available) +const DEFAULT_UPLOAD_CONCURRENCY = 3; +const DEFAULT_UPLOAD_QUEUE_DELAY_MS = 500; + // Allowed file types const ACCEPTED_TYPES = { // PDF files @@ -49,17 +53,101 @@ const ACCEPTED_EXTENSIONS = [ '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg', '.md' ]; +// --------------------------------------------------------------------------- +// Directory traversal helpers (FileSystem Access API) +// --------------------------------------------------------------------------- + /** - * Process a list of files for upload - * @param {FileList} files - Files to process + * Read all file entries from a DirectoryReader, handling the 100-entry limit + * by calling readEntries() repeatedly until it returns an empty batch. + * @param {FileSystemDirectoryReader} reader + * @returns {Promise} + */ +function readAllDirectoryEntries(reader) { + return new Promise((resolve, reject) => { + const entries = []; + function readBatch() { + reader.readEntries((batch) => { + if (batch.length === 0) { + resolve(entries); + } else { + entries.push(...batch); + readBatch(); + } + }, reject); + } + readBatch(); + }); +} + +/** + * Recursively collect all File objects from a FileSystemEntry tree. + * @param {FileSystemEntry} entry + * @param {File[]} files - accumulator array + * @returns {Promise} + */ +async function traverseFileEntry(entry, files) { + if (entry.isFile) { + await new Promise((resolve) => { + entry.file((file) => { files.push(file); resolve(); }, resolve); + }); + } else if (entry.isDirectory) { + const reader = entry.createReader(); + const subEntries = await readAllDirectoryEntries(reader); + for (const sub of subEntries) { + await traverseFileEntry(sub, files); + } + } +} + +/** + * Extract all File objects from a DataTransfer, recursively expanding any + * dropped directories. Falls back gracefully to dataTransfer.files when + * the FileSystem Entry API is unavailable. + * @param {DataTransfer} dataTransfer + * @returns {Promise} + */ +async function getFilesFromDataTransfer(dataTransfer) { + // Use the FileSystem Entry API when available (all modern browsers) + if (dataTransfer.items && dataTransfer.items.length > 0) { + const files = []; + const itemList = dataTransfer.items; + for (let i = 0; i < itemList.length; i++) { + const item = itemList[i]; + const entry = item.webkitGetAsEntry ? item.webkitGetAsEntry() : null; + if (entry) { + await traverseFileEntry(entry, files); + } else if (item.kind === 'file') { + const file = item.getAsFile(); + if (file) files.push(file); + } + } + return files; + } + // Fallback: plain FileList (no directory support) + return Array.from(dataTransfer.files || []); +} + +// --------------------------------------------------------------------------- +// Queue-based upload runner +// --------------------------------------------------------------------------- + +/** + * Upload a list of files using a concurrency-limited queue with a configurable + * delay between slot starts to prevent server overload. + * + * @param {File[]} files - Files to upload * @param {HTMLElement} progressContainer - Container element for progress display * @param {HTMLElement} statusMessage - Element for status message display */ function processFiles(files, progressContainer, statusMessage) { if (files.length === 0) return; + const concurrency = (window.uploadConfig && window.uploadConfig.concurrency) || DEFAULT_UPLOAD_CONCURRENCY; + const delayMs = (window.uploadConfig && window.uploadConfig.queueDelayMs) || DEFAULT_UPLOAD_QUEUE_DELAY_MS; + if (statusMessage) { - statusMessage.textContent = `Processing ${files.length} file(s)...`; + statusMessage.textContent = `Queued ${files.length} file(s) for upload…`; } // Clear previous upload progress @@ -67,15 +155,79 @@ function processFiles(files, progressContainer, statusMessage) { progressContainer.innerHTML = ""; } - // Process each file - for (let i = 0; i < files.length; i++) { - const file = files[i]; - validateAndUpload(file, progressContainer, statusMessage); + // Pre-create all progress elements so the user sees the full list immediately + const progressElements = files.map((file) => { + const fileProgress = document.createElement("div"); + fileProgress.className = "flex flex-col mb-2"; + fileProgress.innerHTML = ` +
+ ${file.name} + ${formatFileSize(file.size)} +
+
+
+
+
Queued
+ `; + if (progressContainer) progressContainer.appendChild(fileProgress); + return fileProgress; + }); + + // Queue runner + let index = 0; + let active = 0; + + function startNext() { + while (active < concurrency && index < files.length) { + const i = index++; + active++; + const progressBar = progressElements[i].querySelector(".file-progress-bar"); + const statusEl = progressElements[i].querySelector(".file-status"); + validateAndUploadQueued(files[i], progressBar, statusEl, statusMessage).finally(() => { + active--; + setTimeout(startNext, delayMs); + }); + } } + + startNext(); } /** - * Validate and upload a single file + * Validate and upload a single file (used by the queue runner). + * @param {File} file + * @param {HTMLElement} progressBar + * @param {HTMLElement} statusEl + * @param {HTMLElement} statusMessage + * @returns {Promise} + */ +function validateAndUploadQueued(file, progressBar, statusEl, statusMessage) { + // Validate file type by checking both MIME type and extension + const isValidMimeType = ACCEPTED_TYPES[file.type] || false; + const fileExtension = '.' + file.name.split('.').pop().toLowerCase(); + const isValidExtension = ACCEPTED_EXTENSIONS.includes(fileExtension); + + if (!isValidMimeType && !isValidExtension) { + statusEl.textContent = `Unsupported file type`; + statusEl.className = "text-xs text-red-500 mt-1"; + progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full"; + updateOverallStatus(statusMessage); + return Promise.resolve(); + } + + if (file.size > MAX_FILE_SIZE) { + statusEl.textContent = `Exceeds 500 MB limit`; + statusEl.className = "text-xs text-red-500 mt-1"; + progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full"; + updateOverallStatus(statusMessage); + return Promise.resolve(); + } + + return uploadFile(file, progressBar, statusEl, statusMessage); +} + +/** + * Validate and upload a single file (legacy entry point kept for compatibility). * @param {File} file - File to validate and upload * @param {HTMLElement} progressContainer - Container element for progress display * @param {HTMLElement} statusMessage - Element for status message display @@ -102,26 +254,7 @@ function validateAndUpload(file, progressContainer, statusMessage) { const progressBar = fileProgress.querySelector(".file-progress-bar"); const statusEl = fileProgress.querySelector(".file-status"); - // Validate file type by checking both MIME type and extension - const isValidMimeType = ACCEPTED_TYPES[file.type] || false; - const fileExtension = '.' + file.name.split('.').pop().toLowerCase(); - const isValidExtension = ACCEPTED_EXTENSIONS.includes(fileExtension); - - if (!isValidMimeType && !isValidExtension) { - statusEl.textContent = `Error: ${file.name} - Unsupported file type`; - statusEl.className = "text-xs text-red-500 mt-1"; - return; - } - - // Validate file size - if (file.size > MAX_FILE_SIZE) { - statusEl.textContent = `Error: ${file.name} - File size exceeds 500MB limit`; - statusEl.className = "text-xs text-red-500 mt-1"; - return; - } - - // Upload the file - uploadFile(file, progressBar, statusEl, statusMessage); + validateAndUploadQueued(file, progressBar, statusEl, statusMessage); } /** @@ -130,55 +263,68 @@ function validateAndUpload(file, progressContainer, statusMessage) { * @param {HTMLElement} progressBar - Progress bar element * @param {HTMLElement} statusEl - Status element * @param {HTMLElement} statusMessage - Overall status message element + * @returns {Promise} */ -async function uploadFile(file, progressBar, statusEl, statusMessage) { - statusEl.textContent = `Uploading...`; - try { - let formData = new FormData(); - formData.append("file", file); +function uploadFile(file, progressBar, statusEl, statusMessage) { + statusEl.textContent = `Uploading…`; + statusEl.className = "text-xs text-gray-600 mt-1"; + progressBar.className = "file-progress-bar bg-blue-500 h-2 rounded-full"; - const xhr = new XMLHttpRequest(); - xhr.open("POST", "/api/ui-upload", true); + return new Promise((resolve) => { + try { + const formData = new FormData(); + formData.append("file", file); - // Attach CSRF token so the server-side CSRF middleware accepts the request. - const csrfToken = typeof getCsrfToken === 'function' ? getCsrfToken() : ''; - if (csrfToken) { - xhr.setRequestHeader("X-CSRF-Token", csrfToken); - } + const xhr = new XMLHttpRequest(); + xhr.open("POST", "/api/ui-upload", true); - xhr.upload.onprogress = (e) => { - if (e.lengthComputable) { - const percentComplete = (e.loaded / e.total) * 100; - progressBar.style.width = percentComplete + "%"; - statusEl.textContent = `Uploading: ${Math.round(percentComplete)}%`; + // Attach CSRF token so the server-side CSRF middleware accepts the request. + const csrfToken = typeof getCsrfToken === 'function' ? getCsrfToken() : ''; + if (csrfToken) { + xhr.setRequestHeader("X-CSRF-Token", csrfToken); } - }; - xhr.onload = function() { - if (xhr.status === 200) { - const result = JSON.parse(xhr.responseText); - progressBar.style.width = "100%"; - progressBar.className = "file-progress-bar bg-green-500 h-2 rounded-full"; - statusEl.textContent = `Success: Task ID: ${result.task_id}`; - statusEl.className = "text-xs text-green-600 mt-1"; + xhr.upload.onprogress = (e) => { + if (e.lengthComputable) { + const percentComplete = (e.loaded / e.total) * 100; + progressBar.style.width = percentComplete + "%"; + statusEl.textContent = `Uploading: ${Math.round(percentComplete)}%`; + } + }; + + xhr.onload = function () { + if (xhr.status === 200) { + const result = JSON.parse(xhr.responseText); + progressBar.style.width = "100%"; + progressBar.className = "file-progress-bar bg-green-500 h-2 rounded-full"; + statusEl.textContent = `Success: Task ID: ${result.task_id}`; + statusEl.className = "text-xs text-green-600 mt-1"; + } else { + progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full"; + statusEl.textContent = `Error: Upload failed (HTTP ${xhr.status})`; + statusEl.className = "text-xs text-red-500 mt-1"; + } updateOverallStatus(statusMessage); - } else { - throw new Error(`Upload failed with status ${xhr.status}`); - } - }; + resolve(); + }; - xhr.onerror = function() { - throw new Error("Network error occurred"); - }; + xhr.onerror = function () { + progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full"; + statusEl.textContent = `Error: Network error`; + statusEl.className = "text-xs text-red-500 mt-1"; + updateOverallStatus(statusMessage); + resolve(); + }; - xhr.send(formData); - - } catch (err) { - statusEl.textContent = `Error: ${err.message}`; - statusEl.className = "text-xs text-red-500 mt-1"; - progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full"; - updateOverallStatus(statusMessage); - } + xhr.send(formData); + } catch (err) { + statusEl.textContent = `Error: ${err.message}`; + statusEl.className = "text-xs text-red-500 mt-1"; + progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full"; + updateOverallStatus(statusMessage); + resolve(); + } + }); } /** @@ -191,10 +337,10 @@ function updateOverallStatus(statusMessage) { // Count success/failure const fileStatuses = document.querySelectorAll('.file-status'); let completed = 0; - let total = fileStatuses.length; + const total = fileStatuses.length; fileStatuses.forEach(status => { - if (status.textContent.includes('Success') || status.textContent.includes('Error')) { + if (status.textContent.includes('Success') || status.textContent.includes('Error') || status.textContent.includes('Unsupported') || status.textContent.includes('Exceeds')) { completed++; } }); @@ -226,7 +372,7 @@ function formatFileSize(bytes) { } /** - * Initialize drag-and-drop on an element + * Initialize drag-and-drop on an element with directory support. * @param {HTMLElement} element - Element to enable drag-and-drop on * @param {HTMLElement} progressContainer - Container for progress display * @param {HTMLElement} statusMessage - Element for status messages @@ -260,7 +406,7 @@ function initDragAndDrop(element, progressContainer, statusMessage, options = {} } }); - element.addEventListener("drop", (e) => { + element.addEventListener("drop", async (e) => { e.preventDefault(); e.stopPropagation(); @@ -269,8 +415,10 @@ function initDragAndDrop(element, progressContainer, statusMessage, options = {} element.classList.remove(options.dragOverClass); } - if (e.dataTransfer.files.length) { - processFiles(e.dataTransfer.files, progressContainer, statusMessage); + const files = await getFilesFromDataTransfer(e.dataTransfer); + if (files.length > 0) { + processFiles(files, progressContainer, statusMessage); } }); } + diff --git a/frontend/templates/upload.html b/frontend/templates/upload.html index 10b731eb..4393f4bf 100644 --- a/frontend/templates/upload.html +++ b/frontend/templates/upload.html @@ -12,12 +12,11 @@

Tap to select files or use the camera button below. @@ -42,7 +41,7 @@

Allowed types: PDF, Office documents (Word, Excel, PowerPoint, etc.), Images

-

Maximum size: 500MB per file

+

Maximum size: 500MB per file • Directories are uploaded recursively

@@ -122,6 +121,13 @@ {% endblock %} {% block scripts %} +