diff --git a/.env.demo b/.env.demo index bdd7ff03..19ef9de1 100644 --- a/.env.demo +++ b/.env.demo @@ -16,6 +16,12 @@ ALLOW_FILE_DELETE=true # Allow deletion of file records PROCESSALL_THROTTLE_THRESHOLD=20 # Number of files above which throttling is applied (default: 20) PROCESSALL_THROTTLE_DELAY=3 # Delay in seconds between each task submission when throttling (default: 3) +# **Client-Side Upload Throttling** +# Controls pacing when the browser uploads files (especially large directory drops). +# The browser auto-detects rate-limit (HTTP 429) responses and backs off accordingly. +UPLOAD_CONCURRENCY=3 # Max simultaneous uploads from the browser (default: 3) +UPLOAD_QUEUE_DELAY_MS=500 # Delay (ms) between starting each upload slot (default: 500) + # **File Upload Size Limits** (Security - see SECURITY_AUDIT.md) # Maximum file upload size in bytes. Default: 1GB (1073741824 bytes) # Prevents resource exhaustion attacks. Adjust based on your server capacity. diff --git a/app/api/files.py b/app/api/files.py index d4014f55..dff86c6a 100644 --- a/app/api/files.py +++ b/app/api/files.py @@ -18,6 +18,7 @@ from app.database import get_db from app.models import FileRecord, ProcessingLog from app.tasks.convert_to_pdf import convert_to_pdf from app.tasks.process_document import process_document +from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, IMAGE_MIME_TYPES from app.utils.file_queries import apply_status_filter from app.utils.file_status import get_files_processing_status from app.utils.filename_utils import sanitize_filename @@ -1034,33 +1035,6 @@ async def ui_upload(request: Request, file: UploadFile = File(...)): logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'") file_size = written_size - # Same set of allowed file types as in the IMAP task - ALLOWED_MIME_TYPES = { - "application/pdf", - "application/msword", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "application/vnd.ms-excel", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.ms-powerpoint", - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - "text/plain", - "text/csv", - "application/rtf", - "text/rtf", - } - - # Image MIME types that need conversion - IMAGE_MIME_TYPES = { - "image/jpeg", - "image/jpg", - "image/png", - "image/gif", - "image/bmp", - "image/tiff", - "image/webp", - "image/svg+xml", - } - # Determine if the file is a PDF or needs conversion mime_type, _ = mimetypes.guess_type(target_path) file_ext = os.path.splitext(target_path)[1].lower() @@ -1114,32 +1088,24 @@ async def ui_upload(request: Request, file: UploadFile = File(...)): # If it's a PDF, process directly task = process_document.delay(target_path, original_filename=safe_filename) logger.info(f"Enqueued PDF for processing: {target_path}") - elif mime_type in IMAGE_MIME_TYPES or any( - file_ext.endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"] - ): + elif mime_type in IMAGE_MIME_TYPES or file_ext in { + ".jpg", + ".jpeg", + ".png", + ".gif", + ".bmp", + ".tiff", + ".tif", + ".webp", + ".svg", + }: # If it's an image, convert to PDF first task = convert_to_pdf.delay(target_path, original_filename=safe_filename) logger.info(f"Enqueued image for PDF conversion: {target_path}") - elif mime_type in ALLOWED_MIME_TYPES or any( - file_ext.endswith(ext) - for ext in [ - ".doc", - ".docx", - ".xls", - ".xlsx", - ".ppt", - ".pptx", - ".odt", - ".ods", - ".odp", - ".rtf", - ".txt", - ".csv", - ] - ): - # If it's an office document, convert to PDF first + elif mime_type in ALLOWED_MIME_TYPES or file_ext in ALLOWED_EXTENSIONS: + # Office document, HTML, Markdown, or other Gotenberg-supported format task = convert_to_pdf.delay(target_path, original_filename=safe_filename) - logger.info(f"Enqueued office document for PDF conversion: {target_path}") + logger.info(f"Enqueued document for PDF conversion: {target_path}") else: # For any other file type, attempt conversion but log a warning logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion") diff --git a/app/api/url_upload.py b/app/api/url_upload.py index bfc82dab..f8e34d7b 100644 --- a/app/api/url_upload.py +++ b/app/api/url_upload.py @@ -17,6 +17,7 @@ from pydantic import BaseModel, HttpUrl, field_validator from app.auth import require_login from app.config import settings from app.tasks.process_document import process_document +from app.utils.allowed_types import ALLOWED_MIME_TYPES from app.utils.filename_utils import sanitize_filename # Set up logging @@ -108,7 +109,7 @@ def validate_url_safety(url: str) -> None: def validate_file_type(content_type: str, filename: str) -> bool: """ - Validate that the file type is supported. + Validate that the file type is supported (i.e. processable by Gotenberg). Args: content_type: MIME type from response headers @@ -117,44 +118,18 @@ def validate_file_type(content_type: str, filename: str) -> bool: Returns: True if file type is allowed """ - # Same allowed types as regular upload - ALLOWED_MIME_TYPES = { - "application/pdf", - "application/msword", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "application/vnd.ms-excel", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.ms-powerpoint", - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - "text/plain", - "text/csv", - "application/rtf", - "text/rtf", - } - - IMAGE_MIME_TYPES = { - "image/jpeg", - "image/jpg", - "image/png", - "image/gif", - "image/bmp", - "image/tiff", - "image/webp", - "image/svg+xml", - } - # Check content type from header if content_type: # Handle content-type with charset (e.g., "application/pdf; charset=utf-8") base_content_type = content_type.split(";", maxsplit=1)[0].strip().lower() - if base_content_type in ALLOWED_MIME_TYPES or base_content_type in IMAGE_MIME_TYPES: + if base_content_type in ALLOWED_MIME_TYPES: return True # Also check by extension as fallback _, ext = os.path.splitext(filename) if ext: guessed_type, _ = mimetypes.guess_type(filename) - if guessed_type and (guessed_type in ALLOWED_MIME_TYPES or guessed_type in IMAGE_MIME_TYPES): + if guessed_type and guessed_type in ALLOWED_MIME_TYPES: return True return False diff --git a/app/tasks/imap_tasks.py b/app/tasks/imap_tasks.py index 7453e63f..cc380d88 100644 --- a/app/tasks/imap_tasks.py +++ b/app/tasks/imap_tasks.py @@ -13,6 +13,7 @@ from celery import shared_task from app.config import settings from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task from app.tasks.process_document import process_document # Updated import +from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES logger = logging.getLogger(__name__) @@ -268,20 +269,6 @@ def fetch_attachments_and_enqueue(email_message): Returns True if at least one allowed attachment was processed. """ - ALLOWED_MIME_TYPES = { - "application/pdf", - "application/msword", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "application/vnd.ms-excel", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.ms-powerpoint", - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - "text/plain", - "text/csv", - "application/rtf", - "text/rtf", - } - has_attachment = False for part in email_message.walk(): if part.get_content_maintype() == "multipart": @@ -295,8 +282,9 @@ def fetch_attachments_and_enqueue(email_message): is_pdf_by_extension = filename.lower().endswith(".pdf") mime_type = part.get_content_type() - # Accept file if it has an allowed MIME type OR it's a PDF by extension - if mime_type not in ALLOWED_MIME_TYPES and not is_pdf_by_extension: + file_ext = os.path.splitext(filename)[1].lower() + # Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension + if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension: logger.info("Skipping attachment %s with MIME type %s", filename, mime_type) continue diff --git a/app/utils/allowed_types.py b/app/utils/allowed_types.py new file mode 100644 index 00000000..733ff10e --- /dev/null +++ b/app/utils/allowed_types.py @@ -0,0 +1,133 @@ +""" +Canonical file-type lists for DocuElevate uploads. + +All file types listed here are supported by Gotenberg (the PDF-conversion service +used by DocuElevate) via its LibreOffice, Chromium, or Markdown routes. This +module is the single source of truth consumed by: + + - app/api/files.py (ui-upload endpoint) + - app/api/url_upload.py (URL-upload endpoint) + - app/tasks/imap_tasks.py (IMAP email-attachment ingestion) + - frontend/static/js/upload.js (client-side validation mirror) + +Keep in sync with the OFFICE_EXTENSIONS / IMAGE_EXTENSIONS sets defined in +app/tasks/convert_to_pdf.py. +""" + +# --------------------------------------------------------------------------- +# Document / office MIME types (converted via Gotenberg LibreOffice route) +# --------------------------------------------------------------------------- +DOCUMENT_MIME_TYPES: set[str] = { + # PDF + "application/pdf", + # Word + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "application/vnd.ms-word.document.macroEnabled.12", + "application/vnd.ms-word.template.macroEnabled.12", + # Excel + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.openxmlformats-officedocument.spreadsheetml.template", + "application/vnd.ms-excel.sheet.macroEnabled.12", + "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + # PowerPoint + "application/vnd.ms-powerpoint", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.openxmlformats-officedocument.presentationml.template", + "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + # OpenDocument (LibreOffice native) + "application/vnd.oasis.opendocument.text", + "application/vnd.oasis.opendocument.spreadsheet", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.oasis.opendocument.graphics", + "application/vnd.oasis.opendocument.formula", + # Plain text / data + "text/plain", + "text/csv", + "application/rtf", + "text/rtf", + # HTML (converted via Gotenberg Chromium route) + "text/html", + # Markdown (converted via Gotenberg Chromium/Markdown route) + "text/markdown", + "text/x-markdown", +} + +# --------------------------------------------------------------------------- +# Image MIME types (converted via Gotenberg LibreOffice route) +# --------------------------------------------------------------------------- +IMAGE_MIME_TYPES: set[str] = { + "image/jpeg", + "image/jpg", + "image/png", + "image/gif", + "image/bmp", + "image/tiff", + "image/webp", + "image/svg+xml", +} + +# --------------------------------------------------------------------------- +# Combined set – every MIME type accepted by the upload endpoints +# --------------------------------------------------------------------------- +ALLOWED_MIME_TYPES: set[str] = DOCUMENT_MIME_TYPES | IMAGE_MIME_TYPES + +# --------------------------------------------------------------------------- +# File extensions (lower-case, with leading dot) accepted by Gotenberg +# --------------------------------------------------------------------------- +ALLOWED_EXTENSIONS: set[str] = { + # PDF + ".pdf", + # Word + ".doc", + ".docx", + ".docm", + ".dot", + ".dotx", + ".dotm", + # Excel + ".xls", + ".xlsx", + ".xlsm", + ".xlsb", + ".xlt", + ".xltx", + ".xlw", + # PowerPoint + ".ppt", + ".pptx", + ".pptm", + ".pps", + ".ppsx", + ".pot", + ".potx", + # OpenDocument + ".odt", + ".ods", + ".odp", + ".odg", + ".odf", + # Text / data + ".rtf", + ".txt", + ".csv", + # Images + ".jpg", + ".jpeg", + ".png", + ".gif", + ".bmp", + ".tiff", + ".tif", + ".webp", + ".svg", + # Web + ".html", + ".htm", + # Markdown + ".md", + ".markdown", +} diff --git a/app/views/files.py b/app/views/files.py index 0644d28a..072de056 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -17,26 +17,6 @@ router = APIRouter() _FILE_NOT_FOUND = "File not found" -def _get_upload_concurrency() -> int: - """Return the configured upload concurrency (falls back to default on error).""" - try: - from app.config import settings - - return settings.upload_concurrency - except Exception: - return 3 - - -def _get_upload_queue_delay_ms() -> int: - """Return the configured upload queue delay in ms (falls back to default on error).""" - try: - from app.config import settings - - return settings.upload_queue_delay_ms - except Exception: - return 500 - - @router.get("/files") @require_login def files_page( @@ -53,6 +33,8 @@ def files_page( """ Return the 'files.html' template with server-side pagination, sorting, and filtering """ + from app.config import settings + try: # Import the model here to avoid circular imports from sqlalchemy import asc, desc @@ -131,8 +113,8 @@ def files_page( "mime_type": mime_type or "", "status": status or "", "mime_types": mime_types, - "upload_concurrency": _get_upload_concurrency(), - "upload_queue_delay_ms": _get_upload_queue_delay_ms(), + "upload_concurrency": settings.upload_concurrency, + "upload_queue_delay_ms": settings.upload_queue_delay_ms, }, ) except Exception as e: @@ -146,8 +128,8 @@ def files_page( "files": [], "pagination": {"page": 1, "per_page": per_page, "total_items": 0, "total_pages": 0}, "error": str(e), - "upload_concurrency": _get_upload_concurrency(), - "upload_queue_delay_ms": _get_upload_queue_delay_ms(), + "upload_concurrency": settings.upload_concurrency, + "upload_queue_delay_ms": settings.upload_queue_delay_ms, }, ) diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index 90c386df..dba9c2ae 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -31,6 +31,19 @@ Control how the `/processall` endpoint handles large batches of files to prevent - Total queue time: (25-1) × 3 = 72 seconds - Prevents API rate limit issues and ensures smooth processing +### Client-Side Upload Throttling + +Control how the web UI queues and paces file uploads to avoid overwhelming the backend, especially when dragging large directories (potentially thousands of files) onto the upload area. + +| **Variable** | **Description** | **Default** | +|----------------------------|-------------------------------------------------------------------------------------------------------------------------------|-------------| +| `UPLOAD_CONCURRENCY` | Maximum number of files uploaded simultaneously from the browser. | `3` | +| `UPLOAD_QUEUE_DELAY_MS` | Delay in milliseconds between starting each upload slot. Staggers upload starts to smooth out server load. | `500` | + +**Adaptive back-off**: The browser automatically slows down if the server responds with HTTP 429 (Too Many Requests). It reads the `Retry-After` header, pauses the queue for the indicated time, doubles the inter-slot delay (exponential back-off, capped at 30 s), and reduces concurrency to 1. After 5 consecutive successes it gradually recovers toward the configured values. + +**Example**: With `UPLOAD_CONCURRENCY=3` and `UPLOAD_QUEUE_DELAY_MS=500`, a directory of 5,000 files is uploaded ≈ 3 at a time with 500 ms pacing – the backend processes files at its own rate while the queue drains in the background without triggering API rate limits. + ### File Upload Size Limits **Security Feature**: Control file upload sizes to prevent resource exhaustion attacks. See [SECURITY_AUDIT.md](../SECURITY_AUDIT.md#5-file-upload-size-limits) for security details. diff --git a/frontend/static/js/upload.js b/frontend/static/js/upload.js index 9212ca9a..22f8332d 100644 --- a/frontend/static/js/upload.js +++ b/frontend/static/js/upload.js @@ -1,65 +1,179 @@ // frontend/static/js/upload.js // Reusable drag-and-drop upload functionality for DocuElevate -// Configuration -const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MB +// ── Configuration ───────────────────────────────────────────────────────────── +const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500 MB -// Upload throttling defaults (overridden by window.uploadConfig when available) +/** Fallback upload throttling values when window.uploadConfig is not set. */ const DEFAULT_UPLOAD_CONCURRENCY = 3; const DEFAULT_UPLOAD_QUEUE_DELAY_MS = 500; -// Allowed file types +/** Maximum number of 429 retries before a file is permanently marked failed. */ +const MAX_RATE_LIMIT_RETRIES = 5; + +// ── Accepted MIME types (mirrors app/utils/allowed_types.py) ───────────────── +// All types processable by Gotenberg (LibreOffice, Chromium, or Markdown routes). const ACCEPTED_TYPES = { - // PDF files + // PDF 'application/pdf': true, - - // Image formats - 'image/jpeg': true, 'image/jpg': true, 'image/png': true, - 'image/gif': true, 'image/bmp': true, 'image/tiff': true, - 'image/webp': true, 'image/svg+xml': true, - - // Office document formats - Word + // Word 'application/msword': true, 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': true, 'application/vnd.openxmlformats-officedocument.wordprocessingml.template': true, 'application/vnd.ms-word.document.macroEnabled.12': true, - + 'application/vnd.ms-word.template.macroEnabled.12': true, // Excel 'application/vnd.ms-excel': true, 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': true, 'application/vnd.openxmlformats-officedocument.spreadsheetml.template': true, 'application/vnd.ms-excel.sheet.macroEnabled.12': true, - + 'application/vnd.ms-excel.sheet.binary.macroEnabled.12': true, // PowerPoint 'application/vnd.ms-powerpoint': true, 'application/vnd.openxmlformats-officedocument.presentationml.presentation': true, 'application/vnd.openxmlformats-officedocument.presentationml.template': true, 'application/vnd.openxmlformats-officedocument.presentationml.slideshow': true, - - // Other common formats + 'application/vnd.ms-powerpoint.presentation.macroEnabled.12': true, + // OpenDocument (LibreOffice native) + 'application/vnd.oasis.opendocument.text': true, + 'application/vnd.oasis.opendocument.spreadsheet': true, + 'application/vnd.oasis.opendocument.presentation': true, + 'application/vnd.oasis.opendocument.graphics': true, + 'application/vnd.oasis.opendocument.formula': true, + // Images + 'image/jpeg': true, 'image/jpg': true, 'image/png': true, + 'image/gif': true, 'image/bmp': true, 'image/tiff': true, + 'image/webp': true, 'image/svg+xml': true, + // Text / data 'text/plain': true, 'text/csv': true, 'application/rtf': true, 'text/rtf': true, + // HTML (Gotenberg Chromium route) 'text/html': true, - 'application/xml': true, - 'text/xml': true + // Markdown (Gotenberg Chromium/Markdown route) + 'text/markdown': true, + 'text/x-markdown': true, }; -// File extensions that are always allowed (even if mime type is not recognized) -const ACCEPTED_EXTENSIONS = [ - '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', - '.odt', '.ods', '.odp', '.rtf', '.txt', '.csv', - '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg', '.md' -]; +// File extensions always accepted even when the browser reports no / wrong MIME type. +const ACCEPTED_EXTENSIONS = new Set([ + // PDF + '.pdf', + // Word + '.doc', '.docx', '.docm', '.dot', '.dotx', '.dotm', + // Excel + '.xls', '.xlsx', '.xlsm', '.xlsb', '.xlt', '.xltx', '.xlw', + // PowerPoint + '.ppt', '.pptx', '.pptm', '.pps', '.ppsx', '.pot', '.potx', + // OpenDocument + '.odt', '.ods', '.odp', '.odg', '.odf', + // Text / data + '.rtf', '.txt', '.csv', + // Images + '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif', '.webp', '.svg', + // Web + '.html', '.htm', + // Markdown + '.md', '.markdown', +]); -// --------------------------------------------------------------------------- -// Directory traversal helpers (FileSystem Access API) -// --------------------------------------------------------------------------- +// ── Adaptive throttle state ─────────────────────────────────────────────────── +// Module-level so the backoff state persists across multiple drop/select events +// on the same page (rate limits are per-user on the server). +const _adaptiveState = { + /** Current effective inter-slot delay (ms). null = use window.uploadConfig value. */ + delayMs: null, + /** Current effective concurrency. null = use window.uploadConfig value. */ + concurrency: null, + /** Consecutive successful uploads without a 429. Resets on each 429 or recovery step. */ + consecutiveOk: 0, + /** Date.now() timestamp after which the queue may resume (set on 429 backoff). */ + pauseUntil: 0, +}; + +function _cfgDelay() { + return (window.uploadConfig && window.uploadConfig.queueDelayMs != null) + ? window.uploadConfig.queueDelayMs + : DEFAULT_UPLOAD_QUEUE_DELAY_MS; +} + +function _cfgConcurrency() { + return (window.uploadConfig && window.uploadConfig.concurrency != null) + ? window.uploadConfig.concurrency + : DEFAULT_UPLOAD_CONCURRENCY; +} + +/** Effective delay between upload slot starts (increased during backoff). */ +function _effectiveDelay() { + return _adaptiveState.delayMs !== null ? _adaptiveState.delayMs : _cfgDelay(); +} + +/** Fallback backoff multiplier when no Retry-After header is present. */ +const FALLBACK_BACKOFF_MULTIPLIER = 4; +/** Minimum fallback pause duration (ms) when no Retry-After header is present. */ +const MIN_FALLBACK_BACKOFF_MS = 5000; +/** Maximum inter-slot delay after repeated exponential backoff (ms). */ +const MAX_BACKOFF_DELAY_MS = 30000; + +/** Effective concurrency (reduced to 1 during backoff). */ +function _effectiveConcurrency() { + return _adaptiveState.concurrency !== null ? _adaptiveState.concurrency : _cfgConcurrency(); +} /** - * Read all file entries from a DirectoryReader, handling the 100-entry limit - * by calling readEntries() repeatedly until it returns an empty batch. + * Called when an HTTP 429 Too Many Requests response is received. + * Pauses the queue and applies exponential backoff. + * @param {number} retryAfterSeconds - Value of the Retry-After header (0 = absent). + */ +function _onRateLimited(retryAfterSeconds) { + // Determine how long to pause – prefer the server's Retry-After; fall back to + // FALLBACK_BACKOFF_MULTIPLIER × the current delay (minimum MIN_FALLBACK_BACKOFF_MS). + const waitMs = retryAfterSeconds > 0 + ? retryAfterSeconds * 1000 + : Math.max(_effectiveDelay() * FALLBACK_BACKOFF_MULTIPLIER, MIN_FALLBACK_BACKOFF_MS); + + _adaptiveState.pauseUntil = Date.now() + waitMs; + // Exponential backoff on the inter-slot delay, capped at MAX_BACKOFF_DELAY_MS. + _adaptiveState.delayMs = Math.min(_effectiveDelay() * 2, MAX_BACKOFF_DELAY_MS); + // Serialize uploads while we recover. + _adaptiveState.concurrency = 1; + _adaptiveState.consecutiveOk = 0; + + console.warn( + `[DocuElevate] Rate limited. Pausing ${waitMs} ms. ` + + `New delay: ${_adaptiveState.delayMs} ms, concurrency: 1.` + ); +} + +/** + * Called after each successful (non-429) upload. + * After 5 consecutive successes, gently recovers toward the configured values. + */ +function _onUploadSuccess() { + _adaptiveState.consecutiveOk++; + if (_adaptiveState.consecutiveOk < 5) return; + + // One recovery step every 5 successes. + _adaptiveState.consecutiveOk = 0; + const cfgDelay = _cfgDelay(); + const cfgConc = _cfgConcurrency(); + + if (_adaptiveState.delayMs !== null && _adaptiveState.delayMs > cfgDelay) { + _adaptiveState.delayMs = Math.max(Math.round(_adaptiveState.delayMs * 0.75), cfgDelay); + if (_adaptiveState.delayMs <= cfgDelay) _adaptiveState.delayMs = null; // fully recovered + } + if (_adaptiveState.concurrency !== null && _adaptiveState.concurrency < cfgConc) { + _adaptiveState.concurrency = Math.min(_adaptiveState.concurrency + 1, cfgConc); + if (_adaptiveState.concurrency >= cfgConc) _adaptiveState.concurrency = null; // fully recovered + } +} + +// ── Directory traversal helpers ─────────────────────────────────────────────── + +/** + * Read all entries from a DirectoryReader, handling the browser's 100-entry + * per-batch limit by calling readEntries() repeatedly. * @param {FileSystemDirectoryReader} reader * @returns {Promise} */ @@ -68,12 +182,9 @@ function readAllDirectoryEntries(reader) { const entries = []; function readBatch() { reader.readEntries((batch) => { - if (batch.length === 0) { - resolve(entries); - } else { - entries.push(...batch); - readBatch(); - } + if (batch.length === 0) { resolve(entries); return; } + entries.push(...batch); + readBatch(); }, reject); } readBatch(); @@ -83,7 +194,7 @@ function readAllDirectoryEntries(reader) { /** * Recursively collect all File objects from a FileSystemEntry tree. * @param {FileSystemEntry} entry - * @param {File[]} files - accumulator array + * @param {File[]} files - accumulator * @returns {Promise} */ async function traverseFileEntry(entry, files) { @@ -92,8 +203,7 @@ async function traverseFileEntry(entry, files) { entry.file((file) => { files.push(file); resolve(); }, resolve); }); } else if (entry.isDirectory) { - const reader = entry.createReader(); - const subEntries = await readAllDirectoryEntries(reader); + const subEntries = await readAllDirectoryEntries(entry.createReader()); for (const sub of subEntries) { await traverseFileEntry(sub, files); } @@ -102,18 +212,16 @@ async function traverseFileEntry(entry, files) { /** * Extract all File objects from a DataTransfer, recursively expanding any - * dropped directories. Falls back gracefully to dataTransfer.files when - * the FileSystem Entry API is unavailable. + * dropped directories. Falls back gracefully to dataTransfer.files when the + * FileSystem Entry API is unavailable (Safari < 11.1, some mobile browsers). * @param {DataTransfer} dataTransfer * @returns {Promise} */ async function getFilesFromDataTransfer(dataTransfer) { - // Use the FileSystem Entry API when available (all modern browsers) if (dataTransfer.items && dataTransfer.items.length > 0) { const files = []; - const itemList = dataTransfer.items; - for (let i = 0; i < itemList.length; i++) { - const item = itemList[i]; + for (let i = 0; i < dataTransfer.items.length; i++) { + const item = dataTransfer.items[i]; const entry = item.webkitGetAsEntry ? item.webkitGetAsEntry() : null; if (entry) { await traverseFileEntry(entry, files); @@ -124,244 +232,261 @@ async function getFilesFromDataTransfer(dataTransfer) { } return files; } - // Fallback: plain FileList (no directory support) return Array.from(dataTransfer.files || []); } -// --------------------------------------------------------------------------- -// Queue-based upload runner -// --------------------------------------------------------------------------- +// ── Core queue runner ───────────────────────────────────────────────────────── /** - * Upload a list of files using a concurrency-limited queue with a configurable - * delay between slot starts to prevent server overload. + * Validate and queue files for upload with adaptive throttling. * - * @param {File[]} files - Files to upload - * @param {HTMLElement} progressContainer - Container element for progress display - * @param {HTMLElement} statusMessage - Element for status message display + * Files are pre-rendered as progress rows so the user immediately sees the + * full list. The queue runner respects the current effective concurrency and + * delay, slowing down automatically when the server signals rate limiting (429). + * + * @param {File[]|FileList} files + * @param {HTMLElement} progressContainer + * @param {HTMLElement} statusMessage */ function processFiles(files, progressContainer, statusMessage) { - if (files.length === 0) return; - - const concurrency = (window.uploadConfig && window.uploadConfig.concurrency) || DEFAULT_UPLOAD_CONCURRENCY; - const delayMs = (window.uploadConfig && window.uploadConfig.queueDelayMs) || DEFAULT_UPLOAD_QUEUE_DELAY_MS; + const fileArray = Array.from(files); + if (!fileArray.length) return; if (statusMessage) { - statusMessage.textContent = `Queued ${files.length} file(s) for upload…`; + statusMessage.textContent = `Queued ${fileArray.length} file(s) for upload…`; } + if (progressContainer) progressContainer.innerHTML = ''; - // Clear previous upload progress - if (progressContainer) { - progressContainer.innerHTML = ""; - } - - // Pre-create all progress elements so the user sees the full list immediately - const progressElements = files.map((file) => { - const fileProgress = document.createElement("div"); - fileProgress.className = "flex flex-col mb-2"; - fileProgress.innerHTML = ` + // Pre-create one progress row per file. + const queueItems = fileArray.map((file) => { + const row = document.createElement('div'); + row.className = 'flex flex-col mb-2'; + row.innerHTML = `
${file.name} ${formatFileSize(file.size)}
-
+
Queued
`; - if (progressContainer) progressContainer.appendChild(fileProgress); - return fileProgress; + if (progressContainer) progressContainer.appendChild(row); + return { + file, + progressBar: row.querySelector('.file-progress-bar'), + statusEl: row.querySelector('.file-status'), + retryCount: 0, + }; }); - // Queue runner - let index = 0; + // Mutable queue – rate-limited items are pushed back to the front. + const queue = [...queueItems]; let active = 0; - function startNext() { - while (active < concurrency && index < files.length) { - const i = index++; + function scheduleNext() { + // Respect global backoff pause. + const pauseRemaining = _adaptiveState.pauseUntil - Date.now(); + if (pauseRemaining > 0) { + setTimeout(scheduleNext, pauseRemaining + 50); + return; + } + + while (active < _effectiveConcurrency() && queue.length > 0) { + const item = queue.shift(); active++; - const progressBar = progressElements[i].querySelector(".file-progress-bar"); - const statusEl = progressElements[i].querySelector(".file-status"); - validateAndUploadQueued(files[i], progressBar, statusEl, statusMessage).finally(() => { + + // Validate before hitting the network. + if (!_isAcceptedFile(item.file)) { + item.progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full'; + item.statusEl.textContent = 'Unsupported file type'; + item.statusEl.className = 'text-xs text-red-500 mt-1'; active--; - setTimeout(startNext, delayMs); - }); + updateOverallStatus(statusMessage); + // No HTTP request – skip straight to next without adding delay. + scheduleNext(); + continue; + } + + if (item.file.size > MAX_FILE_SIZE) { + item.progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full'; + item.statusEl.textContent = 'Exceeds 500 MB limit'; + item.statusEl.className = 'text-xs text-red-500 mt-1'; + active--; + updateOverallStatus(statusMessage); + scheduleNext(); + continue; + } + + _uploadSingleFile(item.file, item.progressBar, item.statusEl, statusMessage) + .then((result) => { + active--; + if (result.rateLimited) { + _onRateLimited(result.retryAfterSeconds); + item.retryCount++; + if (item.retryCount < MAX_RATE_LIMIT_RETRIES) { + // Re-insert at the front of the queue to retry after the pause. + queue.unshift(item); + } else { + item.progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full'; + item.statusEl.textContent = 'Failed: rate limit retries exhausted'; + item.statusEl.className = 'text-xs text-red-500 mt-1'; + updateOverallStatus(statusMessage); + } + // Resume after the backoff window. + const wait = Math.max(_adaptiveState.pauseUntil - Date.now() + 50, 0); + setTimeout(scheduleNext, wait); + } else { + // Success or permanent error – wait the configured delay before next slot. + setTimeout(scheduleNext, _effectiveDelay()); + } + }); } } - startNext(); + scheduleNext(); } /** - * Validate and upload a single file (used by the queue runner). + * Check whether a file passes MIME type and extension validation. + * @param {File} file + * @returns {boolean} + */ +function _isAcceptedFile(file) { + if (ACCEPTED_TYPES[file.type]) return true; + const ext = '.' + file.name.split('.').pop().toLowerCase(); + return ACCEPTED_EXTENSIONS.has(ext); +} + +/** + * Upload a single file via XHR, returning a structured result. + * Detects HTTP 429 responses and reads the Retry-After / X-RateLimit-Reset + * headers so the caller can apply precise backoff. + * * @param {File} file * @param {HTMLElement} progressBar * @param {HTMLElement} statusEl * @param {HTMLElement} statusMessage - * @returns {Promise} + * @returns {Promise<{rateLimited: boolean, retryAfterSeconds: number}>} */ -function validateAndUploadQueued(file, progressBar, statusEl, statusMessage) { - // Validate file type by checking both MIME type and extension - const isValidMimeType = ACCEPTED_TYPES[file.type] || false; - const fileExtension = '.' + file.name.split('.').pop().toLowerCase(); - const isValidExtension = ACCEPTED_EXTENSIONS.includes(fileExtension); - - if (!isValidMimeType && !isValidExtension) { - statusEl.textContent = `Unsupported file type`; - statusEl.className = "text-xs text-red-500 mt-1"; - progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full"; - updateOverallStatus(statusMessage); - return Promise.resolve(); - } - - if (file.size > MAX_FILE_SIZE) { - statusEl.textContent = `Exceeds 500 MB limit`; - statusEl.className = "text-xs text-red-500 mt-1"; - progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full"; - updateOverallStatus(statusMessage); - return Promise.resolve(); - } - - return uploadFile(file, progressBar, statusEl, statusMessage); -} - -/** - * Validate and upload a single file (legacy entry point kept for compatibility). - * @param {File} file - File to validate and upload - * @param {HTMLElement} progressContainer - Container element for progress display - * @param {HTMLElement} statusMessage - Element for status message display - */ -function validateAndUpload(file, progressContainer, statusMessage) { - // Create progress element for this file - const fileProgress = document.createElement("div"); - fileProgress.className = "flex flex-col mb-2"; - fileProgress.innerHTML = ` -
- ${file.name} - ${formatFileSize(file.size)} -
-
-
-
-
Validating...
- `; - - if (progressContainer) { - progressContainer.appendChild(fileProgress); - } - - const progressBar = fileProgress.querySelector(".file-progress-bar"); - const statusEl = fileProgress.querySelector(".file-status"); - - validateAndUploadQueued(file, progressBar, statusEl, statusMessage); -} - -/** - * Upload a file to the server - * @param {File} file - File to upload - * @param {HTMLElement} progressBar - Progress bar element - * @param {HTMLElement} statusEl - Status element - * @param {HTMLElement} statusMessage - Overall status message element - * @returns {Promise} - */ -function uploadFile(file, progressBar, statusEl, statusMessage) { - statusEl.textContent = `Uploading…`; - statusEl.className = "text-xs text-gray-600 mt-1"; - progressBar.className = "file-progress-bar bg-blue-500 h-2 rounded-full"; +function _uploadSingleFile(file, progressBar, statusEl, statusMessage) { + statusEl.textContent = 'Uploading…'; + statusEl.className = 'text-xs text-gray-600 mt-1'; + progressBar.style.width = '0%'; + progressBar.className = 'file-progress-bar bg-blue-500 h-2 rounded-full'; return new Promise((resolve) => { - try { - const formData = new FormData(); - formData.append("file", file); + const formData = new FormData(); + formData.append('file', file); - const xhr = new XMLHttpRequest(); - xhr.open("POST", "/api/ui-upload", true); + const xhr = new XMLHttpRequest(); + xhr.open('POST', '/api/ui-upload', true); - // Attach CSRF token so the server-side CSRF middleware accepts the request. - const csrfToken = typeof getCsrfToken === 'function' ? getCsrfToken() : ''; - if (csrfToken) { - xhr.setRequestHeader("X-CSRF-Token", csrfToken); + const csrfToken = typeof getCsrfToken === 'function' ? getCsrfToken() : ''; + if (csrfToken) xhr.setRequestHeader('X-CSRF-Token', csrfToken); + + xhr.upload.onprogress = (e) => { + if (e.lengthComputable) { + const pct = Math.round((e.loaded / e.total) * 100); + progressBar.style.width = pct + '%'; + statusEl.textContent = `Uploading: ${pct}%`; } + }; - xhr.upload.onprogress = (e) => { - if (e.lengthComputable) { - const percentComplete = (e.loaded / e.total) * 100; - progressBar.style.width = percentComplete + "%"; - statusEl.textContent = `Uploading: ${Math.round(percentComplete)}%`; - } - }; - - xhr.onload = function () { - if (xhr.status === 200) { - const result = JSON.parse(xhr.responseText); - progressBar.style.width = "100%"; - progressBar.className = "file-progress-bar bg-green-500 h-2 rounded-full"; - statusEl.textContent = `Success: Task ID: ${result.task_id}`; - statusEl.className = "text-xs text-green-600 mt-1"; - } else { - progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full"; - statusEl.textContent = `Error: Upload failed (HTTP ${xhr.status})`; - statusEl.className = "text-xs text-red-500 mt-1"; - } + xhr.onload = () => { + if (xhr.status === 200) { + const result = JSON.parse(xhr.responseText); + progressBar.style.width = '100%'; + progressBar.className = 'file-progress-bar bg-green-500 h-2 rounded-full'; + statusEl.textContent = `Success: Task ID: ${result.task_id}`; + statusEl.className = 'text-xs text-green-600 mt-1'; + _onUploadSuccess(); updateOverallStatus(statusMessage); - resolve(); - }; + resolve({ rateLimited: false, retryAfterSeconds: 0 }); - xhr.onerror = function () { - progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full"; - statusEl.textContent = `Error: Network error`; - statusEl.className = "text-xs text-red-500 mt-1"; + } else if (xhr.status === 429) { + // Parse Retry-After (seconds integer). + let retryAfter = parseInt(xhr.getResponseHeader('Retry-After') || '0', 10); + if (!retryAfter) { + // Fall back to X-RateLimit-Reset (Unix timestamp). + const reset = parseInt(xhr.getResponseHeader('X-RateLimit-Reset') || '0', 10); + if (reset) retryAfter = Math.max(reset - Math.floor(Date.now() / 1000), 1); + } + progressBar.className = 'file-progress-bar bg-yellow-400 h-2 rounded-full'; + statusEl.textContent = 'Rate limited – queued to retry…'; + statusEl.className = 'text-xs text-yellow-600 mt-1'; + resolve({ rateLimited: true, retryAfterSeconds: retryAfter }); + + } else { + progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full'; + statusEl.textContent = `Error: HTTP ${xhr.status}`; + statusEl.className = 'text-xs text-red-500 mt-1'; updateOverallStatus(statusMessage); - resolve(); - }; + resolve({ rateLimited: false, retryAfterSeconds: 0 }); + } + }; - xhr.send(formData); - } catch (err) { - statusEl.textContent = `Error: ${err.message}`; - statusEl.className = "text-xs text-red-500 mt-1"; - progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full"; + xhr.onerror = () => { + progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full'; + statusEl.textContent = 'Error: Network error'; + statusEl.className = 'text-xs text-red-500 mt-1'; updateOverallStatus(statusMessage); - resolve(); - } + resolve({ rateLimited: false, retryAfterSeconds: 0 }); + }; + + xhr.send(formData); }); } +// ── Legacy single-file entry point (kept for backward compat) ───────────────── + /** - * Update the overall status message based on file statuses - * @param {HTMLElement} statusMessage - Status message element + * Validate and upload a single file (legacy path – wraps the queue runner). + * @param {File} file + * @param {HTMLElement} progressContainer + * @param {HTMLElement} statusMessage + */ +function validateAndUpload(file, progressContainer, statusMessage) { + processFiles([file], progressContainer, statusMessage); +} + +// ── Status helpers ──────────────────────────────────────────────────────────── + +/** + * Re-calculate and display the overall upload status. + * Fires 'allUploadsComplete' when every item has a terminal status. + * @param {HTMLElement} statusMessage */ function updateOverallStatus(statusMessage) { if (!statusMessage) return; - // Count success/failure const fileStatuses = document.querySelectorAll('.file-status'); - let completed = 0; + let done = 0; const total = fileStatuses.length; - fileStatuses.forEach(status => { - if (status.textContent.includes('Success') || status.textContent.includes('Error') || status.textContent.includes('Unsupported') || status.textContent.includes('Exceeds')) { - completed++; - } + fileStatuses.forEach((s) => { + const t = s.textContent; + if ( + t.startsWith('Success') || + t.startsWith('Error') || + t.startsWith('Unsupported') || + t.startsWith('Exceeds') || + t.startsWith('Failed:') + ) done++; }); - if (completed === total) { - statusMessage.textContent = `All uploads completed (${completed}/${total})`; - - // Trigger a custom event when all uploads are complete - const allUploadsComplete = new CustomEvent('allUploadsComplete', { - detail: { total: total, completed: completed } - }); - window.dispatchEvent(allUploadsComplete); + if (done === total && total > 0) { + statusMessage.textContent = `All uploads completed (${done}/${total})`; + window.dispatchEvent(new CustomEvent('allUploadsComplete', { detail: { total, completed: done } })); } else { - statusMessage.textContent = `Uploading files (${completed}/${total})`; + statusMessage.textContent = `Uploading files (${done}/${total})`; } } /** - * Format file size for display - * @param {number} bytes - File size in bytes - * @returns {string} Formatted file size + * Format a byte count for human-readable display. + * @param {number} bytes + * @returns {string} */ function formatFileSize(bytes) { if (bytes === 0) return '0 Bytes'; @@ -371,54 +496,43 @@ function formatFileSize(bytes) { return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; } +// ── Drag-and-drop initialiser ───────────────────────────────────────────────── + /** - * Initialize drag-and-drop on an element with directory support. - * @param {HTMLElement} element - Element to enable drag-and-drop on - * @param {HTMLElement} progressContainer - Container for progress display - * @param {HTMLElement} statusMessage - Element for status messages - * @param {Object} options - Additional options + * Wire up drag-and-drop on an element with full directory-traversal support. + * @param {HTMLElement} element + * @param {HTMLElement} progressContainer + * @param {HTMLElement} statusMessage + * @param {Object} [options] + * @param {string} [options.dragOverClass] - CSS class added during drag-over */ function initDragAndDrop(element, progressContainer, statusMessage, options = {}) { if (!element) { - console.error("Element not found for drag-and-drop initialization"); + console.error('[DocuElevate] Element not found for drag-and-drop initialization'); return; } - // Add event listeners for drag-and-drop - element.addEventListener("dragover", (e) => { + element.addEventListener('dragover', (e) => { e.preventDefault(); e.stopPropagation(); - e.dataTransfer.dropEffect = "copy"; - - // Add visual feedback - if (options.dragOverClass) { - element.classList.add(options.dragOverClass); - } + e.dataTransfer.dropEffect = 'copy'; + if (options.dragOverClass) element.classList.add(options.dragOverClass); }); - element.addEventListener("dragleave", (e) => { + element.addEventListener('dragleave', (e) => { e.preventDefault(); e.stopPropagation(); - - // Remove visual feedback - if (options.dragOverClass) { - element.classList.remove(options.dragOverClass); - } + if (options.dragOverClass) element.classList.remove(options.dragOverClass); }); - element.addEventListener("drop", async (e) => { + element.addEventListener('drop', async (e) => { e.preventDefault(); e.stopPropagation(); - - // Remove visual feedback - if (options.dragOverClass) { - element.classList.remove(options.dragOverClass); - } + if (options.dragOverClass) element.classList.remove(options.dragOverClass); const files = await getFilesFromDataTransfer(e.dataTransfer); - if (files.length > 0) { - processFiles(files, progressContainer, statusMessage); - } + if (files.length > 0) processFiles(files, progressContainer, statusMessage); }); } + diff --git a/frontend/templates/files.html b/frontend/templates/files.html index d4f13beb..cc4e62ae 100644 --- a/frontend/templates/files.html +++ b/frontend/templates/files.html @@ -3,6 +3,12 @@ {% block head_extra %} +