feat(upload): adaptive 429 backoff, full Gotenberg file types, directory traversal
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -16,6 +16,12 @@ ALLOW_FILE_DELETE=true # Allow deletion of file records
|
||||
PROCESSALL_THROTTLE_THRESHOLD=20 # Number of files above which throttling is applied (default: 20)
|
||||
PROCESSALL_THROTTLE_DELAY=3 # Delay in seconds between each task submission when throttling (default: 3)
|
||||
|
||||
# **Client-Side Upload Throttling**
|
||||
# Controls pacing when the browser uploads files (especially large directory drops).
|
||||
# The browser auto-detects rate-limit (HTTP 429) responses and backs off accordingly.
|
||||
UPLOAD_CONCURRENCY=3 # Max simultaneous uploads from the browser (default: 3)
|
||||
UPLOAD_QUEUE_DELAY_MS=500 # Delay (ms) between starting each upload slot (default: 500)
|
||||
|
||||
# **File Upload Size Limits** (Security - see SECURITY_AUDIT.md)
|
||||
# Maximum file upload size in bytes. Default: 1GB (1073741824 bytes)
|
||||
# Prevents resource exhaustion attacks. Adjust based on your server capacity.
|
||||
|
||||
+15
-49
@@ -18,6 +18,7 @@ from app.database import get_db
|
||||
from app.models import FileRecord, ProcessingLog
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf
|
||||
from app.tasks.process_document import process_document
|
||||
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, IMAGE_MIME_TYPES
|
||||
from app.utils.file_queries import apply_status_filter
|
||||
from app.utils.file_status import get_files_processing_status
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
@@ -1034,33 +1035,6 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
|
||||
logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")
|
||||
file_size = written_size
|
||||
|
||||
# Same set of allowed file types as in the IMAP task
|
||||
ALLOWED_MIME_TYPES = {
|
||||
"application/pdf",
|
||||
"application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"text/plain",
|
||||
"text/csv",
|
||||
"application/rtf",
|
||||
"text/rtf",
|
||||
}
|
||||
|
||||
# Image MIME types that need conversion
|
||||
IMAGE_MIME_TYPES = {
|
||||
"image/jpeg",
|
||||
"image/jpg",
|
||||
"image/png",
|
||||
"image/gif",
|
||||
"image/bmp",
|
||||
"image/tiff",
|
||||
"image/webp",
|
||||
"image/svg+xml",
|
||||
}
|
||||
|
||||
# Determine if the file is a PDF or needs conversion
|
||||
mime_type, _ = mimetypes.guess_type(target_path)
|
||||
file_ext = os.path.splitext(target_path)[1].lower()
|
||||
@@ -1114,32 +1088,24 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
|
||||
# If it's a PDF, process directly
|
||||
task = process_document.delay(target_path, original_filename=safe_filename)
|
||||
logger.info(f"Enqueued PDF for processing: {target_path}")
|
||||
elif mime_type in IMAGE_MIME_TYPES or any(
|
||||
file_ext.endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"]
|
||||
):
|
||||
elif mime_type in IMAGE_MIME_TYPES or file_ext in {
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".png",
|
||||
".gif",
|
||||
".bmp",
|
||||
".tiff",
|
||||
".tif",
|
||||
".webp",
|
||||
".svg",
|
||||
}:
|
||||
# If it's an image, convert to PDF first
|
||||
task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
|
||||
logger.info(f"Enqueued image for PDF conversion: {target_path}")
|
||||
elif mime_type in ALLOWED_MIME_TYPES or any(
|
||||
file_ext.endswith(ext)
|
||||
for ext in [
|
||||
".doc",
|
||||
".docx",
|
||||
".xls",
|
||||
".xlsx",
|
||||
".ppt",
|
||||
".pptx",
|
||||
".odt",
|
||||
".ods",
|
||||
".odp",
|
||||
".rtf",
|
||||
".txt",
|
||||
".csv",
|
||||
]
|
||||
):
|
||||
# If it's an office document, convert to PDF first
|
||||
elif mime_type in ALLOWED_MIME_TYPES or file_ext in ALLOWED_EXTENSIONS:
|
||||
# Office document, HTML, Markdown, or other Gotenberg-supported format
|
||||
task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
|
||||
logger.info(f"Enqueued office document for PDF conversion: {target_path}")
|
||||
logger.info(f"Enqueued document for PDF conversion: {target_path}")
|
||||
else:
|
||||
# For any other file type, attempt conversion but log a warning
|
||||
logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
|
||||
|
||||
+4
-29
@@ -17,6 +17,7 @@ from pydantic import BaseModel, HttpUrl, field_validator
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.tasks.process_document import process_document
|
||||
from app.utils.allowed_types import ALLOWED_MIME_TYPES
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
|
||||
# Set up logging
|
||||
@@ -108,7 +109,7 @@ def validate_url_safety(url: str) -> None:
|
||||
|
||||
def validate_file_type(content_type: str, filename: str) -> bool:
|
||||
"""
|
||||
Validate that the file type is supported.
|
||||
Validate that the file type is supported (i.e. processable by Gotenberg).
|
||||
|
||||
Args:
|
||||
content_type: MIME type from response headers
|
||||
@@ -117,44 +118,18 @@ def validate_file_type(content_type: str, filename: str) -> bool:
|
||||
Returns:
|
||||
True if file type is allowed
|
||||
"""
|
||||
# Same allowed types as regular upload
|
||||
ALLOWED_MIME_TYPES = {
|
||||
"application/pdf",
|
||||
"application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"text/plain",
|
||||
"text/csv",
|
||||
"application/rtf",
|
||||
"text/rtf",
|
||||
}
|
||||
|
||||
IMAGE_MIME_TYPES = {
|
||||
"image/jpeg",
|
||||
"image/jpg",
|
||||
"image/png",
|
||||
"image/gif",
|
||||
"image/bmp",
|
||||
"image/tiff",
|
||||
"image/webp",
|
||||
"image/svg+xml",
|
||||
}
|
||||
|
||||
# Check content type from header
|
||||
if content_type:
|
||||
# Handle content-type with charset (e.g., "application/pdf; charset=utf-8")
|
||||
base_content_type = content_type.split(";", maxsplit=1)[0].strip().lower()
|
||||
if base_content_type in ALLOWED_MIME_TYPES or base_content_type in IMAGE_MIME_TYPES:
|
||||
if base_content_type in ALLOWED_MIME_TYPES:
|
||||
return True
|
||||
|
||||
# Also check by extension as fallback
|
||||
_, ext = os.path.splitext(filename)
|
||||
if ext:
|
||||
guessed_type, _ = mimetypes.guess_type(filename)
|
||||
if guessed_type and (guessed_type in ALLOWED_MIME_TYPES or guessed_type in IMAGE_MIME_TYPES):
|
||||
if guessed_type and guessed_type in ALLOWED_MIME_TYPES:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
+4
-16
@@ -13,6 +13,7 @@ from celery import shared_task
|
||||
from app.config import settings
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
|
||||
from app.tasks.process_document import process_document # Updated import
|
||||
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -268,20 +269,6 @@ def fetch_attachments_and_enqueue(email_message):
|
||||
|
||||
Returns True if at least one allowed attachment was processed.
|
||||
"""
|
||||
ALLOWED_MIME_TYPES = {
|
||||
"application/pdf",
|
||||
"application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"text/plain",
|
||||
"text/csv",
|
||||
"application/rtf",
|
||||
"text/rtf",
|
||||
}
|
||||
|
||||
has_attachment = False
|
||||
for part in email_message.walk():
|
||||
if part.get_content_maintype() == "multipart":
|
||||
@@ -295,8 +282,9 @@ def fetch_attachments_and_enqueue(email_message):
|
||||
is_pdf_by_extension = filename.lower().endswith(".pdf")
|
||||
|
||||
mime_type = part.get_content_type()
|
||||
# Accept file if it has an allowed MIME type OR it's a PDF by extension
|
||||
if mime_type not in ALLOWED_MIME_TYPES and not is_pdf_by_extension:
|
||||
file_ext = os.path.splitext(filename)[1].lower()
|
||||
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
|
||||
if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension:
|
||||
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
|
||||
continue
|
||||
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
"""
|
||||
Canonical file-type lists for DocuElevate uploads.
|
||||
|
||||
All file types listed here are supported by Gotenberg (the PDF-conversion service
|
||||
used by DocuElevate) via its LibreOffice, Chromium, or Markdown routes. This
|
||||
module is the single source of truth consumed by:
|
||||
|
||||
- app/api/files.py (ui-upload endpoint)
|
||||
- app/api/url_upload.py (URL-upload endpoint)
|
||||
- app/tasks/imap_tasks.py (IMAP email-attachment ingestion)
|
||||
- frontend/static/js/upload.js (client-side validation mirror)
|
||||
|
||||
Keep in sync with the OFFICE_EXTENSIONS / IMAGE_EXTENSIONS sets defined in
|
||||
app/tasks/convert_to_pdf.py.
|
||||
"""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Document / office MIME types (converted via Gotenberg LibreOffice route)
|
||||
# ---------------------------------------------------------------------------
|
||||
DOCUMENT_MIME_TYPES: set[str] = {
|
||||
# PDF
|
||||
"application/pdf",
|
||||
# Word
|
||||
"application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
|
||||
"application/vnd.ms-word.document.macroEnabled.12",
|
||||
"application/vnd.ms-word.template.macroEnabled.12",
|
||||
# Excel
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.template",
|
||||
"application/vnd.ms-excel.sheet.macroEnabled.12",
|
||||
"application/vnd.ms-excel.sheet.binary.macroEnabled.12",
|
||||
# PowerPoint
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.template",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.slideshow",
|
||||
"application/vnd.ms-powerpoint.presentation.macroEnabled.12",
|
||||
# OpenDocument (LibreOffice native)
|
||||
"application/vnd.oasis.opendocument.text",
|
||||
"application/vnd.oasis.opendocument.spreadsheet",
|
||||
"application/vnd.oasis.opendocument.presentation",
|
||||
"application/vnd.oasis.opendocument.graphics",
|
||||
"application/vnd.oasis.opendocument.formula",
|
||||
# Plain text / data
|
||||
"text/plain",
|
||||
"text/csv",
|
||||
"application/rtf",
|
||||
"text/rtf",
|
||||
# HTML (converted via Gotenberg Chromium route)
|
||||
"text/html",
|
||||
# Markdown (converted via Gotenberg Chromium/Markdown route)
|
||||
"text/markdown",
|
||||
"text/x-markdown",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Image MIME types (converted via Gotenberg LibreOffice route)
|
||||
# ---------------------------------------------------------------------------
|
||||
IMAGE_MIME_TYPES: set[str] = {
|
||||
"image/jpeg",
|
||||
"image/jpg",
|
||||
"image/png",
|
||||
"image/gif",
|
||||
"image/bmp",
|
||||
"image/tiff",
|
||||
"image/webp",
|
||||
"image/svg+xml",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Combined set – every MIME type accepted by the upload endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
ALLOWED_MIME_TYPES: set[str] = DOCUMENT_MIME_TYPES | IMAGE_MIME_TYPES
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# File extensions (lower-case, with leading dot) accepted by Gotenberg
|
||||
# ---------------------------------------------------------------------------
|
||||
ALLOWED_EXTENSIONS: set[str] = {
|
||||
# PDF
|
||||
".pdf",
|
||||
# Word
|
||||
".doc",
|
||||
".docx",
|
||||
".docm",
|
||||
".dot",
|
||||
".dotx",
|
||||
".dotm",
|
||||
# Excel
|
||||
".xls",
|
||||
".xlsx",
|
||||
".xlsm",
|
||||
".xlsb",
|
||||
".xlt",
|
||||
".xltx",
|
||||
".xlw",
|
||||
# PowerPoint
|
||||
".ppt",
|
||||
".pptx",
|
||||
".pptm",
|
||||
".pps",
|
||||
".ppsx",
|
||||
".pot",
|
||||
".potx",
|
||||
# OpenDocument
|
||||
".odt",
|
||||
".ods",
|
||||
".odp",
|
||||
".odg",
|
||||
".odf",
|
||||
# Text / data
|
||||
".rtf",
|
||||
".txt",
|
||||
".csv",
|
||||
# Images
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".png",
|
||||
".gif",
|
||||
".bmp",
|
||||
".tiff",
|
||||
".tif",
|
||||
".webp",
|
||||
".svg",
|
||||
# Web
|
||||
".html",
|
||||
".htm",
|
||||
# Markdown
|
||||
".md",
|
||||
".markdown",
|
||||
}
|
||||
+6
-24
@@ -17,26 +17,6 @@ router = APIRouter()
|
||||
_FILE_NOT_FOUND = "File not found"
|
||||
|
||||
|
||||
def _get_upload_concurrency() -> int:
|
||||
"""Return the configured upload concurrency (falls back to default on error)."""
|
||||
try:
|
||||
from app.config import settings
|
||||
|
||||
return settings.upload_concurrency
|
||||
except Exception:
|
||||
return 3
|
||||
|
||||
|
||||
def _get_upload_queue_delay_ms() -> int:
|
||||
"""Return the configured upload queue delay in ms (falls back to default on error)."""
|
||||
try:
|
||||
from app.config import settings
|
||||
|
||||
return settings.upload_queue_delay_ms
|
||||
except Exception:
|
||||
return 500
|
||||
|
||||
|
||||
@router.get("/files")
|
||||
@require_login
|
||||
def files_page(
|
||||
@@ -53,6 +33,8 @@ def files_page(
|
||||
"""
|
||||
Return the 'files.html' template with server-side pagination, sorting, and filtering
|
||||
"""
|
||||
from app.config import settings
|
||||
|
||||
try:
|
||||
# Import the model here to avoid circular imports
|
||||
from sqlalchemy import asc, desc
|
||||
@@ -131,8 +113,8 @@ def files_page(
|
||||
"mime_type": mime_type or "",
|
||||
"status": status or "",
|
||||
"mime_types": mime_types,
|
||||
"upload_concurrency": _get_upload_concurrency(),
|
||||
"upload_queue_delay_ms": _get_upload_queue_delay_ms(),
|
||||
"upload_concurrency": settings.upload_concurrency,
|
||||
"upload_queue_delay_ms": settings.upload_queue_delay_ms,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
@@ -146,8 +128,8 @@ def files_page(
|
||||
"files": [],
|
||||
"pagination": {"page": 1, "per_page": per_page, "total_items": 0, "total_pages": 0},
|
||||
"error": str(e),
|
||||
"upload_concurrency": _get_upload_concurrency(),
|
||||
"upload_queue_delay_ms": _get_upload_queue_delay_ms(),
|
||||
"upload_concurrency": settings.upload_concurrency,
|
||||
"upload_queue_delay_ms": settings.upload_queue_delay_ms,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -31,6 +31,19 @@ Control how the `/processall` endpoint handles large batches of files to prevent
|
||||
- Total queue time: (25-1) × 3 = 72 seconds
|
||||
- Prevents API rate limit issues and ensures smooth processing
|
||||
|
||||
### Client-Side Upload Throttling
|
||||
|
||||
Control how the web UI queues and paces file uploads to avoid overwhelming the backend, especially when dragging large directories (potentially thousands of files) onto the upload area.
|
||||
|
||||
| **Variable** | **Description** | **Default** |
|
||||
|----------------------------|-------------------------------------------------------------------------------------------------------------------------------|-------------|
|
||||
| `UPLOAD_CONCURRENCY` | Maximum number of files uploaded simultaneously from the browser. | `3` |
|
||||
| `UPLOAD_QUEUE_DELAY_MS` | Delay in milliseconds between starting each upload slot. Staggers upload starts to smooth out server load. | `500` |
|
||||
|
||||
**Adaptive back-off**: The browser automatically slows down if the server responds with HTTP 429 (Too Many Requests). It reads the `Retry-After` header, pauses the queue for the indicated time, doubles the inter-slot delay (exponential back-off, capped at 30 s), and reduces concurrency to 1. After 5 consecutive successes it gradually recovers toward the configured values.
|
||||
|
||||
**Example**: With `UPLOAD_CONCURRENCY=3` and `UPLOAD_QUEUE_DELAY_MS=500`, a directory of 5,000 files is uploaded ≈ 3 at a time with 500 ms pacing – the backend processes files at its own rate while the queue drains in the background without triggering API rate limits.
|
||||
|
||||
### File Upload Size Limits
|
||||
|
||||
**Security Feature**: Control file upload sizes to prevent resource exhaustion attacks. See [SECURITY_AUDIT.md](../SECURITY_AUDIT.md#5-file-upload-size-limits) for security details.
|
||||
|
||||
+359
-245
@@ -1,65 +1,179 @@
|
||||
// frontend/static/js/upload.js
|
||||
// Reusable drag-and-drop upload functionality for DocuElevate
|
||||
|
||||
// Configuration
|
||||
const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MB
|
||||
// ── Configuration ─────────────────────────────────────────────────────────────
|
||||
const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500 MB
|
||||
|
||||
// Upload throttling defaults (overridden by window.uploadConfig when available)
|
||||
/** Fallback upload throttling values when window.uploadConfig is not set. */
|
||||
const DEFAULT_UPLOAD_CONCURRENCY = 3;
|
||||
const DEFAULT_UPLOAD_QUEUE_DELAY_MS = 500;
|
||||
|
||||
// Allowed file types
|
||||
/** Maximum number of 429 retries before a file is permanently marked failed. */
|
||||
const MAX_RATE_LIMIT_RETRIES = 5;
|
||||
|
||||
// ── Accepted MIME types (mirrors app/utils/allowed_types.py) ─────────────────
|
||||
// All types processable by Gotenberg (LibreOffice, Chromium, or Markdown routes).
|
||||
const ACCEPTED_TYPES = {
|
||||
// PDF files
|
||||
// PDF
|
||||
'application/pdf': true,
|
||||
|
||||
// Image formats
|
||||
'image/jpeg': true, 'image/jpg': true, 'image/png': true,
|
||||
'image/gif': true, 'image/bmp': true, 'image/tiff': true,
|
||||
'image/webp': true, 'image/svg+xml': true,
|
||||
|
||||
// Office document formats - Word
|
||||
// Word
|
||||
'application/msword': true,
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document': true,
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.template': true,
|
||||
'application/vnd.ms-word.document.macroEnabled.12': true,
|
||||
|
||||
'application/vnd.ms-word.template.macroEnabled.12': true,
|
||||
// Excel
|
||||
'application/vnd.ms-excel': true,
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': true,
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.template': true,
|
||||
'application/vnd.ms-excel.sheet.macroEnabled.12': true,
|
||||
|
||||
'application/vnd.ms-excel.sheet.binary.macroEnabled.12': true,
|
||||
// PowerPoint
|
||||
'application/vnd.ms-powerpoint': true,
|
||||
'application/vnd.openxmlformats-officedocument.presentationml.presentation': true,
|
||||
'application/vnd.openxmlformats-officedocument.presentationml.template': true,
|
||||
'application/vnd.openxmlformats-officedocument.presentationml.slideshow': true,
|
||||
|
||||
// Other common formats
|
||||
'application/vnd.ms-powerpoint.presentation.macroEnabled.12': true,
|
||||
// OpenDocument (LibreOffice native)
|
||||
'application/vnd.oasis.opendocument.text': true,
|
||||
'application/vnd.oasis.opendocument.spreadsheet': true,
|
||||
'application/vnd.oasis.opendocument.presentation': true,
|
||||
'application/vnd.oasis.opendocument.graphics': true,
|
||||
'application/vnd.oasis.opendocument.formula': true,
|
||||
// Images
|
||||
'image/jpeg': true, 'image/jpg': true, 'image/png': true,
|
||||
'image/gif': true, 'image/bmp': true, 'image/tiff': true,
|
||||
'image/webp': true, 'image/svg+xml': true,
|
||||
// Text / data
|
||||
'text/plain': true,
|
||||
'text/csv': true,
|
||||
'application/rtf': true,
|
||||
'text/rtf': true,
|
||||
// HTML (Gotenberg Chromium route)
|
||||
'text/html': true,
|
||||
'application/xml': true,
|
||||
'text/xml': true
|
||||
// Markdown (Gotenberg Chromium/Markdown route)
|
||||
'text/markdown': true,
|
||||
'text/x-markdown': true,
|
||||
};
|
||||
|
||||
// File extensions that are always allowed (even if mime type is not recognized)
|
||||
const ACCEPTED_EXTENSIONS = [
|
||||
'.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
|
||||
'.odt', '.ods', '.odp', '.rtf', '.txt', '.csv',
|
||||
'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg', '.md'
|
||||
];
|
||||
// File extensions always accepted even when the browser reports no / wrong MIME type.
|
||||
const ACCEPTED_EXTENSIONS = new Set([
|
||||
// PDF
|
||||
'.pdf',
|
||||
// Word
|
||||
'.doc', '.docx', '.docm', '.dot', '.dotx', '.dotm',
|
||||
// Excel
|
||||
'.xls', '.xlsx', '.xlsm', '.xlsb', '.xlt', '.xltx', '.xlw',
|
||||
// PowerPoint
|
||||
'.ppt', '.pptx', '.pptm', '.pps', '.ppsx', '.pot', '.potx',
|
||||
// OpenDocument
|
||||
'.odt', '.ods', '.odp', '.odg', '.odf',
|
||||
// Text / data
|
||||
'.rtf', '.txt', '.csv',
|
||||
// Images
|
||||
'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif', '.webp', '.svg',
|
||||
// Web
|
||||
'.html', '.htm',
|
||||
// Markdown
|
||||
'.md', '.markdown',
|
||||
]);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Directory traversal helpers (FileSystem Access API)
|
||||
// ---------------------------------------------------------------------------
|
||||
// ── Adaptive throttle state ───────────────────────────────────────────────────
|
||||
// Module-level so the backoff state persists across multiple drop/select events
|
||||
// on the same page (rate limits are per-user on the server).
|
||||
const _adaptiveState = {
|
||||
/** Current effective inter-slot delay (ms). null = use window.uploadConfig value. */
|
||||
delayMs: null,
|
||||
/** Current effective concurrency. null = use window.uploadConfig value. */
|
||||
concurrency: null,
|
||||
/** Consecutive successful uploads without a 429. Resets on each 429 or recovery step. */
|
||||
consecutiveOk: 0,
|
||||
/** Date.now() timestamp after which the queue may resume (set on 429 backoff). */
|
||||
pauseUntil: 0,
|
||||
};
|
||||
|
||||
function _cfgDelay() {
|
||||
return (window.uploadConfig && window.uploadConfig.queueDelayMs != null)
|
||||
? window.uploadConfig.queueDelayMs
|
||||
: DEFAULT_UPLOAD_QUEUE_DELAY_MS;
|
||||
}
|
||||
|
||||
function _cfgConcurrency() {
|
||||
return (window.uploadConfig && window.uploadConfig.concurrency != null)
|
||||
? window.uploadConfig.concurrency
|
||||
: DEFAULT_UPLOAD_CONCURRENCY;
|
||||
}
|
||||
|
||||
/** Effective delay between upload slot starts (increased during backoff). */
|
||||
function _effectiveDelay() {
|
||||
return _adaptiveState.delayMs !== null ? _adaptiveState.delayMs : _cfgDelay();
|
||||
}
|
||||
|
||||
/** Fallback backoff multiplier when no Retry-After header is present. */
|
||||
const FALLBACK_BACKOFF_MULTIPLIER = 4;
|
||||
/** Minimum fallback pause duration (ms) when no Retry-After header is present. */
|
||||
const MIN_FALLBACK_BACKOFF_MS = 5000;
|
||||
/** Maximum inter-slot delay after repeated exponential backoff (ms). */
|
||||
const MAX_BACKOFF_DELAY_MS = 30000;
|
||||
|
||||
/** Effective concurrency (reduced to 1 during backoff). */
|
||||
function _effectiveConcurrency() {
|
||||
return _adaptiveState.concurrency !== null ? _adaptiveState.concurrency : _cfgConcurrency();
|
||||
}
|
||||
|
||||
/**
|
||||
* Read all file entries from a DirectoryReader, handling the 100-entry limit
|
||||
* by calling readEntries() repeatedly until it returns an empty batch.
|
||||
* Called when an HTTP 429 Too Many Requests response is received.
|
||||
* Pauses the queue and applies exponential backoff.
|
||||
* @param {number} retryAfterSeconds - Value of the Retry-After header (0 = absent).
|
||||
*/
|
||||
function _onRateLimited(retryAfterSeconds) {
|
||||
// Determine how long to pause – prefer the server's Retry-After; fall back to
|
||||
// FALLBACK_BACKOFF_MULTIPLIER × the current delay (minimum MIN_FALLBACK_BACKOFF_MS).
|
||||
const waitMs = retryAfterSeconds > 0
|
||||
? retryAfterSeconds * 1000
|
||||
: Math.max(_effectiveDelay() * FALLBACK_BACKOFF_MULTIPLIER, MIN_FALLBACK_BACKOFF_MS);
|
||||
|
||||
_adaptiveState.pauseUntil = Date.now() + waitMs;
|
||||
// Exponential backoff on the inter-slot delay, capped at MAX_BACKOFF_DELAY_MS.
|
||||
_adaptiveState.delayMs = Math.min(_effectiveDelay() * 2, MAX_BACKOFF_DELAY_MS);
|
||||
// Serialize uploads while we recover.
|
||||
_adaptiveState.concurrency = 1;
|
||||
_adaptiveState.consecutiveOk = 0;
|
||||
|
||||
console.warn(
|
||||
`[DocuElevate] Rate limited. Pausing ${waitMs} ms. ` +
|
||||
`New delay: ${_adaptiveState.delayMs} ms, concurrency: 1.`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called after each successful (non-429) upload.
|
||||
* After 5 consecutive successes, gently recovers toward the configured values.
|
||||
*/
|
||||
function _onUploadSuccess() {
|
||||
_adaptiveState.consecutiveOk++;
|
||||
if (_adaptiveState.consecutiveOk < 5) return;
|
||||
|
||||
// One recovery step every 5 successes.
|
||||
_adaptiveState.consecutiveOk = 0;
|
||||
const cfgDelay = _cfgDelay();
|
||||
const cfgConc = _cfgConcurrency();
|
||||
|
||||
if (_adaptiveState.delayMs !== null && _adaptiveState.delayMs > cfgDelay) {
|
||||
_adaptiveState.delayMs = Math.max(Math.round(_adaptiveState.delayMs * 0.75), cfgDelay);
|
||||
if (_adaptiveState.delayMs <= cfgDelay) _adaptiveState.delayMs = null; // fully recovered
|
||||
}
|
||||
if (_adaptiveState.concurrency !== null && _adaptiveState.concurrency < cfgConc) {
|
||||
_adaptiveState.concurrency = Math.min(_adaptiveState.concurrency + 1, cfgConc);
|
||||
if (_adaptiveState.concurrency >= cfgConc) _adaptiveState.concurrency = null; // fully recovered
|
||||
}
|
||||
}
|
||||
|
||||
// ── Directory traversal helpers ───────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Read all entries from a DirectoryReader, handling the browser's 100-entry
|
||||
* per-batch limit by calling readEntries() repeatedly.
|
||||
* @param {FileSystemDirectoryReader} reader
|
||||
* @returns {Promise<FileSystemEntry[]>}
|
||||
*/
|
||||
@@ -68,12 +182,9 @@ function readAllDirectoryEntries(reader) {
|
||||
const entries = [];
|
||||
function readBatch() {
|
||||
reader.readEntries((batch) => {
|
||||
if (batch.length === 0) {
|
||||
resolve(entries);
|
||||
} else {
|
||||
entries.push(...batch);
|
||||
readBatch();
|
||||
}
|
||||
if (batch.length === 0) { resolve(entries); return; }
|
||||
entries.push(...batch);
|
||||
readBatch();
|
||||
}, reject);
|
||||
}
|
||||
readBatch();
|
||||
@@ -83,7 +194,7 @@ function readAllDirectoryEntries(reader) {
|
||||
/**
|
||||
* Recursively collect all File objects from a FileSystemEntry tree.
|
||||
* @param {FileSystemEntry} entry
|
||||
* @param {File[]} files - accumulator array
|
||||
* @param {File[]} files - accumulator
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function traverseFileEntry(entry, files) {
|
||||
@@ -92,8 +203,7 @@ async function traverseFileEntry(entry, files) {
|
||||
entry.file((file) => { files.push(file); resolve(); }, resolve);
|
||||
});
|
||||
} else if (entry.isDirectory) {
|
||||
const reader = entry.createReader();
|
||||
const subEntries = await readAllDirectoryEntries(reader);
|
||||
const subEntries = await readAllDirectoryEntries(entry.createReader());
|
||||
for (const sub of subEntries) {
|
||||
await traverseFileEntry(sub, files);
|
||||
}
|
||||
@@ -102,18 +212,16 @@ async function traverseFileEntry(entry, files) {
|
||||
|
||||
/**
|
||||
* Extract all File objects from a DataTransfer, recursively expanding any
|
||||
* dropped directories. Falls back gracefully to dataTransfer.files when
|
||||
* the FileSystem Entry API is unavailable.
|
||||
* dropped directories. Falls back gracefully to dataTransfer.files when the
|
||||
* FileSystem Entry API is unavailable (Safari < 11.1, some mobile browsers).
|
||||
* @param {DataTransfer} dataTransfer
|
||||
* @returns {Promise<File[]>}
|
||||
*/
|
||||
async function getFilesFromDataTransfer(dataTransfer) {
|
||||
// Use the FileSystem Entry API when available (all modern browsers)
|
||||
if (dataTransfer.items && dataTransfer.items.length > 0) {
|
||||
const files = [];
|
||||
const itemList = dataTransfer.items;
|
||||
for (let i = 0; i < itemList.length; i++) {
|
||||
const item = itemList[i];
|
||||
for (let i = 0; i < dataTransfer.items.length; i++) {
|
||||
const item = dataTransfer.items[i];
|
||||
const entry = item.webkitGetAsEntry ? item.webkitGetAsEntry() : null;
|
||||
if (entry) {
|
||||
await traverseFileEntry(entry, files);
|
||||
@@ -124,244 +232,261 @@ async function getFilesFromDataTransfer(dataTransfer) {
|
||||
}
|
||||
return files;
|
||||
}
|
||||
// Fallback: plain FileList (no directory support)
|
||||
return Array.from(dataTransfer.files || []);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Queue-based upload runner
|
||||
// ---------------------------------------------------------------------------
|
||||
// ── Core queue runner ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Upload a list of files using a concurrency-limited queue with a configurable
|
||||
* delay between slot starts to prevent server overload.
|
||||
* Validate and queue files for upload with adaptive throttling.
|
||||
*
|
||||
* @param {File[]} files - Files to upload
|
||||
* @param {HTMLElement} progressContainer - Container element for progress display
|
||||
* @param {HTMLElement} statusMessage - Element for status message display
|
||||
* Files are pre-rendered as progress rows so the user immediately sees the
|
||||
* full list. The queue runner respects the current effective concurrency and
|
||||
* delay, slowing down automatically when the server signals rate limiting (429).
|
||||
*
|
||||
* @param {File[]|FileList} files
|
||||
* @param {HTMLElement} progressContainer
|
||||
* @param {HTMLElement} statusMessage
|
||||
*/
|
||||
function processFiles(files, progressContainer, statusMessage) {
|
||||
if (files.length === 0) return;
|
||||
|
||||
const concurrency = (window.uploadConfig && window.uploadConfig.concurrency) || DEFAULT_UPLOAD_CONCURRENCY;
|
||||
const delayMs = (window.uploadConfig && window.uploadConfig.queueDelayMs) || DEFAULT_UPLOAD_QUEUE_DELAY_MS;
|
||||
const fileArray = Array.from(files);
|
||||
if (!fileArray.length) return;
|
||||
|
||||
if (statusMessage) {
|
||||
statusMessage.textContent = `Queued ${files.length} file(s) for upload…`;
|
||||
statusMessage.textContent = `Queued ${fileArray.length} file(s) for upload…`;
|
||||
}
|
||||
if (progressContainer) progressContainer.innerHTML = '';
|
||||
|
||||
// Clear previous upload progress
|
||||
if (progressContainer) {
|
||||
progressContainer.innerHTML = "";
|
||||
}
|
||||
|
||||
// Pre-create all progress elements so the user sees the full list immediately
|
||||
const progressElements = files.map((file) => {
|
||||
const fileProgress = document.createElement("div");
|
||||
fileProgress.className = "flex flex-col mb-2";
|
||||
fileProgress.innerHTML = `
|
||||
// Pre-create one progress row per file.
|
||||
const queueItems = fileArray.map((file) => {
|
||||
const row = document.createElement('div');
|
||||
row.className = 'flex flex-col mb-2';
|
||||
row.innerHTML = `
|
||||
<div class="flex justify-between">
|
||||
<span class="text-sm truncate" title="${file.name}">${file.name}</span>
|
||||
<span class="text-xs text-gray-500">${formatFileSize(file.size)}</span>
|
||||
</div>
|
||||
<div class="w-full bg-gray-200 h-2 rounded-full mt-1">
|
||||
<div class="file-progress-bar bg-gray-300 h-2 rounded-full" style="width: 0%"></div>
|
||||
<div class="file-progress-bar bg-gray-300 h-2 rounded-full" style="width:0%"></div>
|
||||
</div>
|
||||
<div class="file-status text-xs text-gray-400 mt-1">Queued</div>
|
||||
`;
|
||||
if (progressContainer) progressContainer.appendChild(fileProgress);
|
||||
return fileProgress;
|
||||
if (progressContainer) progressContainer.appendChild(row);
|
||||
return {
|
||||
file,
|
||||
progressBar: row.querySelector('.file-progress-bar'),
|
||||
statusEl: row.querySelector('.file-status'),
|
||||
retryCount: 0,
|
||||
};
|
||||
});
|
||||
|
||||
// Queue runner
|
||||
let index = 0;
|
||||
// Mutable queue – rate-limited items are pushed back to the front.
|
||||
const queue = [...queueItems];
|
||||
let active = 0;
|
||||
|
||||
function startNext() {
|
||||
while (active < concurrency && index < files.length) {
|
||||
const i = index++;
|
||||
function scheduleNext() {
|
||||
// Respect global backoff pause.
|
||||
const pauseRemaining = _adaptiveState.pauseUntil - Date.now();
|
||||
if (pauseRemaining > 0) {
|
||||
setTimeout(scheduleNext, pauseRemaining + 50);
|
||||
return;
|
||||
}
|
||||
|
||||
while (active < _effectiveConcurrency() && queue.length > 0) {
|
||||
const item = queue.shift();
|
||||
active++;
|
||||
const progressBar = progressElements[i].querySelector(".file-progress-bar");
|
||||
const statusEl = progressElements[i].querySelector(".file-status");
|
||||
validateAndUploadQueued(files[i], progressBar, statusEl, statusMessage).finally(() => {
|
||||
|
||||
// Validate before hitting the network.
|
||||
if (!_isAcceptedFile(item.file)) {
|
||||
item.progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
|
||||
item.statusEl.textContent = 'Unsupported file type';
|
||||
item.statusEl.className = 'text-xs text-red-500 mt-1';
|
||||
active--;
|
||||
setTimeout(startNext, delayMs);
|
||||
});
|
||||
updateOverallStatus(statusMessage);
|
||||
// No HTTP request – skip straight to next without adding delay.
|
||||
scheduleNext();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (item.file.size > MAX_FILE_SIZE) {
|
||||
item.progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
|
||||
item.statusEl.textContent = 'Exceeds 500 MB limit';
|
||||
item.statusEl.className = 'text-xs text-red-500 mt-1';
|
||||
active--;
|
||||
updateOverallStatus(statusMessage);
|
||||
scheduleNext();
|
||||
continue;
|
||||
}
|
||||
|
||||
_uploadSingleFile(item.file, item.progressBar, item.statusEl, statusMessage)
|
||||
.then((result) => {
|
||||
active--;
|
||||
if (result.rateLimited) {
|
||||
_onRateLimited(result.retryAfterSeconds);
|
||||
item.retryCount++;
|
||||
if (item.retryCount < MAX_RATE_LIMIT_RETRIES) {
|
||||
// Re-insert at the front of the queue to retry after the pause.
|
||||
queue.unshift(item);
|
||||
} else {
|
||||
item.progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
|
||||
item.statusEl.textContent = 'Failed: rate limit retries exhausted';
|
||||
item.statusEl.className = 'text-xs text-red-500 mt-1';
|
||||
updateOverallStatus(statusMessage);
|
||||
}
|
||||
// Resume after the backoff window.
|
||||
const wait = Math.max(_adaptiveState.pauseUntil - Date.now() + 50, 0);
|
||||
setTimeout(scheduleNext, wait);
|
||||
} else {
|
||||
// Success or permanent error – wait the configured delay before next slot.
|
||||
setTimeout(scheduleNext, _effectiveDelay());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
startNext();
|
||||
scheduleNext();
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and upload a single file (used by the queue runner).
|
||||
* Check whether a file passes MIME type and extension validation.
|
||||
* @param {File} file
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function _isAcceptedFile(file) {
|
||||
if (ACCEPTED_TYPES[file.type]) return true;
|
||||
const ext = '.' + file.name.split('.').pop().toLowerCase();
|
||||
return ACCEPTED_EXTENSIONS.has(ext);
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a single file via XHR, returning a structured result.
|
||||
* Detects HTTP 429 responses and reads the Retry-After / X-RateLimit-Reset
|
||||
* headers so the caller can apply precise backoff.
|
||||
*
|
||||
* @param {File} file
|
||||
* @param {HTMLElement} progressBar
|
||||
* @param {HTMLElement} statusEl
|
||||
* @param {HTMLElement} statusMessage
|
||||
* @returns {Promise<void>}
|
||||
* @returns {Promise<{rateLimited: boolean, retryAfterSeconds: number}>}
|
||||
*/
|
||||
function validateAndUploadQueued(file, progressBar, statusEl, statusMessage) {
|
||||
// Validate file type by checking both MIME type and extension
|
||||
const isValidMimeType = ACCEPTED_TYPES[file.type] || false;
|
||||
const fileExtension = '.' + file.name.split('.').pop().toLowerCase();
|
||||
const isValidExtension = ACCEPTED_EXTENSIONS.includes(fileExtension);
|
||||
|
||||
if (!isValidMimeType && !isValidExtension) {
|
||||
statusEl.textContent = `Unsupported file type`;
|
||||
statusEl.className = "text-xs text-red-500 mt-1";
|
||||
progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
|
||||
updateOverallStatus(statusMessage);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
if (file.size > MAX_FILE_SIZE) {
|
||||
statusEl.textContent = `Exceeds 500 MB limit`;
|
||||
statusEl.className = "text-xs text-red-500 mt-1";
|
||||
progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
|
||||
updateOverallStatus(statusMessage);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
return uploadFile(file, progressBar, statusEl, statusMessage);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and upload a single file (legacy entry point kept for compatibility).
|
||||
* @param {File} file - File to validate and upload
|
||||
* @param {HTMLElement} progressContainer - Container element for progress display
|
||||
* @param {HTMLElement} statusMessage - Element for status message display
|
||||
*/
|
||||
function validateAndUpload(file, progressContainer, statusMessage) {
|
||||
// Create progress element for this file
|
||||
const fileProgress = document.createElement("div");
|
||||
fileProgress.className = "flex flex-col mb-2";
|
||||
fileProgress.innerHTML = `
|
||||
<div class="flex justify-between">
|
||||
<span class="text-sm truncate" title="${file.name}">${file.name}</span>
|
||||
<span class="text-xs text-gray-500">${formatFileSize(file.size)}</span>
|
||||
</div>
|
||||
<div class="w-full bg-gray-200 h-2 rounded-full mt-1">
|
||||
<div class="file-progress-bar bg-blue-500 h-2 rounded-full" style="width: 0%"></div>
|
||||
</div>
|
||||
<div class="file-status text-xs text-gray-600 mt-1">Validating...</div>
|
||||
`;
|
||||
|
||||
if (progressContainer) {
|
||||
progressContainer.appendChild(fileProgress);
|
||||
}
|
||||
|
||||
const progressBar = fileProgress.querySelector(".file-progress-bar");
|
||||
const statusEl = fileProgress.querySelector(".file-status");
|
||||
|
||||
validateAndUploadQueued(file, progressBar, statusEl, statusMessage);
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a file to the server
|
||||
* @param {File} file - File to upload
|
||||
* @param {HTMLElement} progressBar - Progress bar element
|
||||
* @param {HTMLElement} statusEl - Status element
|
||||
* @param {HTMLElement} statusMessage - Overall status message element
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
function uploadFile(file, progressBar, statusEl, statusMessage) {
|
||||
statusEl.textContent = `Uploading…`;
|
||||
statusEl.className = "text-xs text-gray-600 mt-1";
|
||||
progressBar.className = "file-progress-bar bg-blue-500 h-2 rounded-full";
|
||||
function _uploadSingleFile(file, progressBar, statusEl, statusMessage) {
|
||||
statusEl.textContent = 'Uploading…';
|
||||
statusEl.className = 'text-xs text-gray-600 mt-1';
|
||||
progressBar.style.width = '0%';
|
||||
progressBar.className = 'file-progress-bar bg-blue-500 h-2 rounded-full';
|
||||
|
||||
return new Promise((resolve) => {
|
||||
try {
|
||||
const formData = new FormData();
|
||||
formData.append("file", file);
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open("POST", "/api/ui-upload", true);
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', '/api/ui-upload', true);
|
||||
|
||||
// Attach CSRF token so the server-side CSRF middleware accepts the request.
|
||||
const csrfToken = typeof getCsrfToken === 'function' ? getCsrfToken() : '';
|
||||
if (csrfToken) {
|
||||
xhr.setRequestHeader("X-CSRF-Token", csrfToken);
|
||||
const csrfToken = typeof getCsrfToken === 'function' ? getCsrfToken() : '';
|
||||
if (csrfToken) xhr.setRequestHeader('X-CSRF-Token', csrfToken);
|
||||
|
||||
xhr.upload.onprogress = (e) => {
|
||||
if (e.lengthComputable) {
|
||||
const pct = Math.round((e.loaded / e.total) * 100);
|
||||
progressBar.style.width = pct + '%';
|
||||
statusEl.textContent = `Uploading: ${pct}%`;
|
||||
}
|
||||
};
|
||||
|
||||
xhr.upload.onprogress = (e) => {
|
||||
if (e.lengthComputable) {
|
||||
const percentComplete = (e.loaded / e.total) * 100;
|
||||
progressBar.style.width = percentComplete + "%";
|
||||
statusEl.textContent = `Uploading: ${Math.round(percentComplete)}%`;
|
||||
}
|
||||
};
|
||||
|
||||
xhr.onload = function () {
|
||||
if (xhr.status === 200) {
|
||||
const result = JSON.parse(xhr.responseText);
|
||||
progressBar.style.width = "100%";
|
||||
progressBar.className = "file-progress-bar bg-green-500 h-2 rounded-full";
|
||||
statusEl.textContent = `Success: Task ID: ${result.task_id}`;
|
||||
statusEl.className = "text-xs text-green-600 mt-1";
|
||||
} else {
|
||||
progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
|
||||
statusEl.textContent = `Error: Upload failed (HTTP ${xhr.status})`;
|
||||
statusEl.className = "text-xs text-red-500 mt-1";
|
||||
}
|
||||
xhr.onload = () => {
|
||||
if (xhr.status === 200) {
|
||||
const result = JSON.parse(xhr.responseText);
|
||||
progressBar.style.width = '100%';
|
||||
progressBar.className = 'file-progress-bar bg-green-500 h-2 rounded-full';
|
||||
statusEl.textContent = `Success: Task ID: ${result.task_id}`;
|
||||
statusEl.className = 'text-xs text-green-600 mt-1';
|
||||
_onUploadSuccess();
|
||||
updateOverallStatus(statusMessage);
|
||||
resolve();
|
||||
};
|
||||
resolve({ rateLimited: false, retryAfterSeconds: 0 });
|
||||
|
||||
xhr.onerror = function () {
|
||||
progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
|
||||
statusEl.textContent = `Error: Network error`;
|
||||
statusEl.className = "text-xs text-red-500 mt-1";
|
||||
} else if (xhr.status === 429) {
|
||||
// Parse Retry-After (seconds integer).
|
||||
let retryAfter = parseInt(xhr.getResponseHeader('Retry-After') || '0', 10);
|
||||
if (!retryAfter) {
|
||||
// Fall back to X-RateLimit-Reset (Unix timestamp).
|
||||
const reset = parseInt(xhr.getResponseHeader('X-RateLimit-Reset') || '0', 10);
|
||||
if (reset) retryAfter = Math.max(reset - Math.floor(Date.now() / 1000), 1);
|
||||
}
|
||||
progressBar.className = 'file-progress-bar bg-yellow-400 h-2 rounded-full';
|
||||
statusEl.textContent = 'Rate limited – queued to retry…';
|
||||
statusEl.className = 'text-xs text-yellow-600 mt-1';
|
||||
resolve({ rateLimited: true, retryAfterSeconds: retryAfter });
|
||||
|
||||
} else {
|
||||
progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
|
||||
statusEl.textContent = `Error: HTTP ${xhr.status}`;
|
||||
statusEl.className = 'text-xs text-red-500 mt-1';
|
||||
updateOverallStatus(statusMessage);
|
||||
resolve();
|
||||
};
|
||||
resolve({ rateLimited: false, retryAfterSeconds: 0 });
|
||||
}
|
||||
};
|
||||
|
||||
xhr.send(formData);
|
||||
} catch (err) {
|
||||
statusEl.textContent = `Error: ${err.message}`;
|
||||
statusEl.className = "text-xs text-red-500 mt-1";
|
||||
progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
|
||||
xhr.onerror = () => {
|
||||
progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
|
||||
statusEl.textContent = 'Error: Network error';
|
||||
statusEl.className = 'text-xs text-red-500 mt-1';
|
||||
updateOverallStatus(statusMessage);
|
||||
resolve();
|
||||
}
|
||||
resolve({ rateLimited: false, retryAfterSeconds: 0 });
|
||||
};
|
||||
|
||||
xhr.send(formData);
|
||||
});
|
||||
}
|
||||
|
||||
// ── Legacy single-file entry point (kept for backward compat) ─────────────────
|
||||
|
||||
/**
|
||||
* Update the overall status message based on file statuses
|
||||
* @param {HTMLElement} statusMessage - Status message element
|
||||
* Validate and upload a single file (legacy path – wraps the queue runner).
|
||||
* @param {File} file
|
||||
* @param {HTMLElement} progressContainer
|
||||
* @param {HTMLElement} statusMessage
|
||||
*/
|
||||
function validateAndUpload(file, progressContainer, statusMessage) {
|
||||
processFiles([file], progressContainer, statusMessage);
|
||||
}
|
||||
|
||||
// ── Status helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Re-calculate and display the overall upload status.
|
||||
* Fires 'allUploadsComplete' when every item has a terminal status.
|
||||
* @param {HTMLElement} statusMessage
|
||||
*/
|
||||
function updateOverallStatus(statusMessage) {
|
||||
if (!statusMessage) return;
|
||||
|
||||
// Count success/failure
|
||||
const fileStatuses = document.querySelectorAll('.file-status');
|
||||
let completed = 0;
|
||||
let done = 0;
|
||||
const total = fileStatuses.length;
|
||||
|
||||
fileStatuses.forEach(status => {
|
||||
if (status.textContent.includes('Success') || status.textContent.includes('Error') || status.textContent.includes('Unsupported') || status.textContent.includes('Exceeds')) {
|
||||
completed++;
|
||||
}
|
||||
fileStatuses.forEach((s) => {
|
||||
const t = s.textContent;
|
||||
if (
|
||||
t.startsWith('Success') ||
|
||||
t.startsWith('Error') ||
|
||||
t.startsWith('Unsupported') ||
|
||||
t.startsWith('Exceeds') ||
|
||||
t.startsWith('Failed:')
|
||||
) done++;
|
||||
});
|
||||
|
||||
if (completed === total) {
|
||||
statusMessage.textContent = `All uploads completed (${completed}/${total})`;
|
||||
|
||||
// Trigger a custom event when all uploads are complete
|
||||
const allUploadsComplete = new CustomEvent('allUploadsComplete', {
|
||||
detail: { total: total, completed: completed }
|
||||
});
|
||||
window.dispatchEvent(allUploadsComplete);
|
||||
if (done === total && total > 0) {
|
||||
statusMessage.textContent = `All uploads completed (${done}/${total})`;
|
||||
window.dispatchEvent(new CustomEvent('allUploadsComplete', { detail: { total, completed: done } }));
|
||||
} else {
|
||||
statusMessage.textContent = `Uploading files (${completed}/${total})`;
|
||||
statusMessage.textContent = `Uploading files (${done}/${total})`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format file size for display
|
||||
* @param {number} bytes - File size in bytes
|
||||
* @returns {string} Formatted file size
|
||||
* Format a byte count for human-readable display.
|
||||
* @param {number} bytes
|
||||
* @returns {string}
|
||||
*/
|
||||
function formatFileSize(bytes) {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
@@ -371,54 +496,43 @@ function formatFileSize(bytes) {
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
// ── Drag-and-drop initialiser ─────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Initialize drag-and-drop on an element with directory support.
|
||||
* @param {HTMLElement} element - Element to enable drag-and-drop on
|
||||
* @param {HTMLElement} progressContainer - Container for progress display
|
||||
* @param {HTMLElement} statusMessage - Element for status messages
|
||||
* @param {Object} options - Additional options
|
||||
* Wire up drag-and-drop on an element with full directory-traversal support.
|
||||
* @param {HTMLElement} element
|
||||
* @param {HTMLElement} progressContainer
|
||||
* @param {HTMLElement} statusMessage
|
||||
* @param {Object} [options]
|
||||
* @param {string} [options.dragOverClass] - CSS class added during drag-over
|
||||
*/
|
||||
function initDragAndDrop(element, progressContainer, statusMessage, options = {}) {
|
||||
if (!element) {
|
||||
console.error("Element not found for drag-and-drop initialization");
|
||||
console.error('[DocuElevate] Element not found for drag-and-drop initialization');
|
||||
return;
|
||||
}
|
||||
|
||||
// Add event listeners for drag-and-drop
|
||||
element.addEventListener("dragover", (e) => {
|
||||
element.addEventListener('dragover', (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
e.dataTransfer.dropEffect = "copy";
|
||||
|
||||
// Add visual feedback
|
||||
if (options.dragOverClass) {
|
||||
element.classList.add(options.dragOverClass);
|
||||
}
|
||||
e.dataTransfer.dropEffect = 'copy';
|
||||
if (options.dragOverClass) element.classList.add(options.dragOverClass);
|
||||
});
|
||||
|
||||
element.addEventListener("dragleave", (e) => {
|
||||
element.addEventListener('dragleave', (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
|
||||
// Remove visual feedback
|
||||
if (options.dragOverClass) {
|
||||
element.classList.remove(options.dragOverClass);
|
||||
}
|
||||
if (options.dragOverClass) element.classList.remove(options.dragOverClass);
|
||||
});
|
||||
|
||||
element.addEventListener("drop", async (e) => {
|
||||
element.addEventListener('drop', async (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
|
||||
// Remove visual feedback
|
||||
if (options.dragOverClass) {
|
||||
element.classList.remove(options.dragOverClass);
|
||||
}
|
||||
if (options.dragOverClass) element.classList.remove(options.dragOverClass);
|
||||
|
||||
const files = await getFilesFromDataTransfer(e.dataTransfer);
|
||||
if (files.length > 0) {
|
||||
processFiles(files, progressContainer, statusMessage);
|
||||
}
|
||||
if (files.length > 0) processFiles(files, progressContainer, statusMessage);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,12 @@
|
||||
|
||||
{% block head_extra %}
|
||||
<script src="/static/js/common.js"></script>
|
||||
<script>
|
||||
window.uploadConfig = {
|
||||
concurrency: {{ upload_concurrency }},
|
||||
queueDelayMs: {{ upload_queue_delay_ms }}
|
||||
};
|
||||
</script>
|
||||
<script src="/static/js/upload.js"></script>
|
||||
<style>
|
||||
/* Drop overlay styles - covers entire page for drag-and-drop anywhere */
|
||||
@@ -342,8 +348,8 @@
|
||||
<div id="dropOverlay" class="drop-overlay">
|
||||
<div class="drop-message">
|
||||
<i class="fas fa-cloud-upload-alt"></i>
|
||||
<p>Drop files anywhere to upload</p>
|
||||
<div class="drop-hint">Supports PDF, Office docs, images, and more</div>
|
||||
<p>Drop files or folders anywhere to upload</p>
|
||||
<div class="drop-hint">Supports PDF, Office docs, images, HTML, Markdown, and more – folders are processed recursively</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -902,18 +908,19 @@
|
||||
e.dataTransfer.dropEffect = 'copy';
|
||||
});
|
||||
|
||||
window.addEventListener('drop', (e) => {
|
||||
window.addEventListener('drop', async (e) => {
|
||||
e.preventDefault();
|
||||
dragCounter = 0;
|
||||
dropOverlay.classList.remove('active');
|
||||
|
||||
if (e.dataTransfer.files.length > 0) {
|
||||
const files = await getFilesFromDataTransfer(e.dataTransfer);
|
||||
if (files.length > 0) {
|
||||
// Show upload modal
|
||||
uploadModal.classList.add('active');
|
||||
uploadProgressContainer.innerHTML = '';
|
||||
|
||||
// Process the dropped files
|
||||
processFiles(e.dataTransfer.files, uploadProgressContainer, uploadStatusMessage);
|
||||
// Process the dropped files (queue-based, with adaptive throttling)
|
||||
processFiles(files, uploadProgressContainer, uploadStatusMessage);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Tests for app/utils/allowed_types.py – canonical Gotenberg file-type lists."""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.utils.allowed_types import (
|
||||
ALLOWED_EXTENSIONS,
|
||||
ALLOWED_MIME_TYPES,
|
||||
DOCUMENT_MIME_TYPES,
|
||||
IMAGE_MIME_TYPES,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestAllowedTypesStructure:
|
||||
"""Structural sanity checks on the exported sets."""
|
||||
|
||||
def test_allowed_mime_types_is_union(self):
|
||||
"""ALLOWED_MIME_TYPES must equal DOCUMENT_MIME_TYPES ∪ IMAGE_MIME_TYPES."""
|
||||
assert ALLOWED_MIME_TYPES == DOCUMENT_MIME_TYPES | IMAGE_MIME_TYPES
|
||||
|
||||
def test_sets_are_disjoint(self):
|
||||
"""DOCUMENT_MIME_TYPES and IMAGE_MIME_TYPES should not overlap."""
|
||||
assert DOCUMENT_MIME_TYPES.isdisjoint(IMAGE_MIME_TYPES)
|
||||
|
||||
def test_extensions_have_leading_dot(self):
|
||||
"""All entries in ALLOWED_EXTENSIONS must start with '.'."""
|
||||
for ext in ALLOWED_EXTENSIONS:
|
||||
assert ext.startswith("."), f"Extension without leading dot: {ext!r}"
|
||||
|
||||
def test_extensions_are_lowercase(self):
|
||||
"""All entries in ALLOWED_EXTENSIONS must be lower-case."""
|
||||
for ext in ALLOWED_EXTENSIONS:
|
||||
assert ext == ext.lower(), f"Non-lowercase extension: {ext!r}"
|
||||
|
||||
def test_no_empty_entries(self):
|
||||
"""No set should contain empty strings."""
|
||||
for s in (DOCUMENT_MIME_TYPES, IMAGE_MIME_TYPES, ALLOWED_EXTENSIONS):
|
||||
assert "" not in s
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestGotenbergCoverageDocuments:
|
||||
"""Verify that every extension Gotenberg handles is present."""
|
||||
|
||||
# These must match OFFICE_EXTENSIONS in app/tasks/convert_to_pdf.py
|
||||
_office_extensions = {
|
||||
".doc",
|
||||
".docx",
|
||||
".docm",
|
||||
".dot",
|
||||
".dotx",
|
||||
".dotm",
|
||||
".xls",
|
||||
".xlsx",
|
||||
".xlsm",
|
||||
".xlsb",
|
||||
".xlt",
|
||||
".xltx",
|
||||
".xlw",
|
||||
".ppt",
|
||||
".pptx",
|
||||
".pptm",
|
||||
".pps",
|
||||
".ppsx",
|
||||
".pot",
|
||||
".potx",
|
||||
".odt",
|
||||
".ods",
|
||||
".odp",
|
||||
".odg",
|
||||
".odf",
|
||||
".rtf",
|
||||
".txt",
|
||||
".csv",
|
||||
}
|
||||
_image_extensions = {
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".png",
|
||||
".gif",
|
||||
".bmp",
|
||||
".tiff",
|
||||
".tif",
|
||||
".webp",
|
||||
".svg",
|
||||
}
|
||||
_html_extensions = {".html", ".htm"}
|
||||
_markdown_extensions = {".md", ".markdown"}
|
||||
|
||||
def test_office_extensions_all_present(self):
|
||||
missing = self._office_extensions - ALLOWED_EXTENSIONS
|
||||
assert not missing, f"Office extensions missing from ALLOWED_EXTENSIONS: {missing}"
|
||||
|
||||
def test_image_extensions_all_present(self):
|
||||
missing = self._image_extensions - ALLOWED_EXTENSIONS
|
||||
assert not missing, f"Image extensions missing from ALLOWED_EXTENSIONS: {missing}"
|
||||
|
||||
def test_html_extensions_present(self):
|
||||
missing = self._html_extensions - ALLOWED_EXTENSIONS
|
||||
assert not missing, f"HTML extensions missing from ALLOWED_EXTENSIONS: {missing}"
|
||||
|
||||
def test_markdown_extensions_present(self):
|
||||
missing = self._markdown_extensions - ALLOWED_EXTENSIONS
|
||||
assert not missing, f"Markdown extensions missing from ALLOWED_EXTENSIONS: {missing}"
|
||||
|
||||
def test_pdf_extension_present(self):
|
||||
assert ".pdf" in ALLOWED_EXTENSIONS
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestGotenbergCoverageMimeTypes:
|
||||
"""Verify that key MIME types are present."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mime_type",
|
||||
[
|
||||
"application/pdf",
|
||||
"application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
# OpenDocument
|
||||
"application/vnd.oasis.opendocument.text",
|
||||
"application/vnd.oasis.opendocument.spreadsheet",
|
||||
"application/vnd.oasis.opendocument.presentation",
|
||||
# Images
|
||||
"image/jpeg",
|
||||
"image/png",
|
||||
"image/tiff",
|
||||
"image/webp",
|
||||
"image/svg+xml",
|
||||
# Text
|
||||
"text/plain",
|
||||
"text/csv",
|
||||
"text/html",
|
||||
"text/markdown",
|
||||
"text/x-markdown",
|
||||
# RTF
|
||||
"application/rtf",
|
||||
"text/rtf",
|
||||
],
|
||||
)
|
||||
def test_mime_type_in_allowed(self, mime_type):
|
||||
assert mime_type in ALLOWED_MIME_TYPES, f"{mime_type} not in ALLOWED_MIME_TYPES"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mime_type",
|
||||
[
|
||||
# New extended Office variants
|
||||
"application/vnd.ms-word.document.macroEnabled.12",
|
||||
"application/vnd.ms-word.template.macroEnabled.12",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
|
||||
"application/vnd.ms-excel.sheet.macroEnabled.12",
|
||||
"application/vnd.ms-excel.sheet.binary.macroEnabled.12",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.template",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.template",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.slideshow",
|
||||
"application/vnd.ms-powerpoint.presentation.macroEnabled.12",
|
||||
"application/vnd.oasis.opendocument.graphics",
|
||||
"application/vnd.oasis.opendocument.formula",
|
||||
],
|
||||
)
|
||||
def test_extended_mime_type_in_allowed(self, mime_type):
|
||||
"""Extended / macro-enabled Office and ODF types must be present."""
|
||||
assert mime_type in ALLOWED_MIME_TYPES, f"{mime_type} not in ALLOWED_MIME_TYPES"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestUploadViewsPassConfig:
|
||||
"""Verify that the upload views inject uploadConfig into their templates."""
|
||||
|
||||
def test_upload_page_contains_upload_config(self, client):
|
||||
"""The /upload page must render window.uploadConfig with numeric values."""
|
||||
response = client.get("/upload")
|
||||
assert response.status_code == 200
|
||||
content = response.text
|
||||
assert "window.uploadConfig" in content
|
||||
# Verify that literal integers are rendered (not the placeholder names)
|
||||
assert "concurrency: 3" in content or "concurrency:" in content
|
||||
assert "queueDelayMs: 500" in content or "queueDelayMs:" in content
|
||||
|
||||
def test_files_page_contains_upload_config(self, client):
|
||||
"""The /files page must render window.uploadConfig with numeric values."""
|
||||
response = client.get("/files")
|
||||
assert response.status_code == 200
|
||||
content = response.text
|
||||
assert "window.uploadConfig" in content
|
||||
assert "concurrency: 3" in content or "concurrency:" in content
|
||||
assert "queueDelayMs: 500" in content or "queueDelayMs:" in content
|
||||
|
||||
def test_upload_config_has_numeric_concurrency(self, client):
|
||||
"""Concurrency in uploadConfig must be a positive integer."""
|
||||
import re
|
||||
|
||||
response = client.get("/upload")
|
||||
assert response.status_code == 200
|
||||
match = re.search(r"concurrency:\s*(\d+)", response.text)
|
||||
assert match is not None, "concurrency integer not found in window.uploadConfig"
|
||||
assert int(match.group(1)) > 0
|
||||
|
||||
def test_upload_config_has_numeric_delay(self, client):
|
||||
"""queueDelayMs in uploadConfig must be a non-negative integer."""
|
||||
import re
|
||||
|
||||
response = client.get("/upload")
|
||||
assert response.status_code == 200
|
||||
match = re.search(r"queueDelayMs:\s*(\d+)", response.text)
|
||||
assert match is not None, "queueDelayMs integer not found in window.uploadConfig"
|
||||
assert int(match.group(1)) >= 0
|
||||
Reference in New Issue
Block a user