feat(upload): adaptive 429 backoff, full Gotenberg file types, directory traversal
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
+359
-245
@@ -1,65 +1,179 @@
|
||||
// frontend/static/js/upload.js
|
||||
// Reusable drag-and-drop upload functionality for DocuElevate
|
||||
|
||||
// Configuration
|
||||
const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MB
|
||||
// ── Configuration ─────────────────────────────────────────────────────────────
|
||||
const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500 MB
|
||||
|
||||
// Upload throttling defaults (overridden by window.uploadConfig when available)
|
||||
/** Fallback upload throttling values when window.uploadConfig is not set. */
|
||||
const DEFAULT_UPLOAD_CONCURRENCY = 3;
|
||||
const DEFAULT_UPLOAD_QUEUE_DELAY_MS = 500;
|
||||
|
||||
// Allowed file types
|
||||
/** Maximum number of 429 retries before a file is permanently marked failed. */
|
||||
const MAX_RATE_LIMIT_RETRIES = 5;
|
||||
|
||||
// ── Accepted MIME types (mirrors app/utils/allowed_types.py) ─────────────────
|
||||
// All types processable by Gotenberg (LibreOffice, Chromium, or Markdown routes).
|
||||
const ACCEPTED_TYPES = {
|
||||
// PDF files
|
||||
// PDF
|
||||
'application/pdf': true,
|
||||
|
||||
// Image formats
|
||||
'image/jpeg': true, 'image/jpg': true, 'image/png': true,
|
||||
'image/gif': true, 'image/bmp': true, 'image/tiff': true,
|
||||
'image/webp': true, 'image/svg+xml': true,
|
||||
|
||||
// Office document formats - Word
|
||||
// Word
|
||||
'application/msword': true,
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document': true,
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.template': true,
|
||||
'application/vnd.ms-word.document.macroEnabled.12': true,
|
||||
|
||||
'application/vnd.ms-word.template.macroEnabled.12': true,
|
||||
// Excel
|
||||
'application/vnd.ms-excel': true,
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': true,
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.template': true,
|
||||
'application/vnd.ms-excel.sheet.macroEnabled.12': true,
|
||||
|
||||
'application/vnd.ms-excel.sheet.binary.macroEnabled.12': true,
|
||||
// PowerPoint
|
||||
'application/vnd.ms-powerpoint': true,
|
||||
'application/vnd.openxmlformats-officedocument.presentationml.presentation': true,
|
||||
'application/vnd.openxmlformats-officedocument.presentationml.template': true,
|
||||
'application/vnd.openxmlformats-officedocument.presentationml.slideshow': true,
|
||||
|
||||
// Other common formats
|
||||
'application/vnd.ms-powerpoint.presentation.macroEnabled.12': true,
|
||||
// OpenDocument (LibreOffice native)
|
||||
'application/vnd.oasis.opendocument.text': true,
|
||||
'application/vnd.oasis.opendocument.spreadsheet': true,
|
||||
'application/vnd.oasis.opendocument.presentation': true,
|
||||
'application/vnd.oasis.opendocument.graphics': true,
|
||||
'application/vnd.oasis.opendocument.formula': true,
|
||||
// Images
|
||||
'image/jpeg': true, 'image/jpg': true, 'image/png': true,
|
||||
'image/gif': true, 'image/bmp': true, 'image/tiff': true,
|
||||
'image/webp': true, 'image/svg+xml': true,
|
||||
// Text / data
|
||||
'text/plain': true,
|
||||
'text/csv': true,
|
||||
'application/rtf': true,
|
||||
'text/rtf': true,
|
||||
// HTML (Gotenberg Chromium route)
|
||||
'text/html': true,
|
||||
'application/xml': true,
|
||||
'text/xml': true
|
||||
// Markdown (Gotenberg Chromium/Markdown route)
|
||||
'text/markdown': true,
|
||||
'text/x-markdown': true,
|
||||
};
|
||||
|
||||
// File extensions that are always allowed (even if mime type is not recognized)
|
||||
const ACCEPTED_EXTENSIONS = [
|
||||
'.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
|
||||
'.odt', '.ods', '.odp', '.rtf', '.txt', '.csv',
|
||||
'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg', '.md'
|
||||
];
|
||||
// File extensions always accepted even when the browser reports no / wrong MIME type.
|
||||
const ACCEPTED_EXTENSIONS = new Set([
|
||||
// PDF
|
||||
'.pdf',
|
||||
// Word
|
||||
'.doc', '.docx', '.docm', '.dot', '.dotx', '.dotm',
|
||||
// Excel
|
||||
'.xls', '.xlsx', '.xlsm', '.xlsb', '.xlt', '.xltx', '.xlw',
|
||||
// PowerPoint
|
||||
'.ppt', '.pptx', '.pptm', '.pps', '.ppsx', '.pot', '.potx',
|
||||
// OpenDocument
|
||||
'.odt', '.ods', '.odp', '.odg', '.odf',
|
||||
// Text / data
|
||||
'.rtf', '.txt', '.csv',
|
||||
// Images
|
||||
'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif', '.webp', '.svg',
|
||||
// Web
|
||||
'.html', '.htm',
|
||||
// Markdown
|
||||
'.md', '.markdown',
|
||||
]);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Directory traversal helpers (FileSystem Access API)
|
||||
// ---------------------------------------------------------------------------
|
||||
// ── Adaptive throttle state ───────────────────────────────────────────────────
|
||||
// Module-level so the backoff state persists across multiple drop/select events
|
||||
// on the same page (rate limits are per-user on the server).
|
||||
const _adaptiveState = {
|
||||
/** Current effective inter-slot delay (ms). null = use window.uploadConfig value. */
|
||||
delayMs: null,
|
||||
/** Current effective concurrency. null = use window.uploadConfig value. */
|
||||
concurrency: null,
|
||||
/** Consecutive successful uploads without a 429. Resets on each 429 or recovery step. */
|
||||
consecutiveOk: 0,
|
||||
/** Date.now() timestamp after which the queue may resume (set on 429 backoff). */
|
||||
pauseUntil: 0,
|
||||
};
|
||||
|
||||
function _cfgDelay() {
|
||||
return (window.uploadConfig && window.uploadConfig.queueDelayMs != null)
|
||||
? window.uploadConfig.queueDelayMs
|
||||
: DEFAULT_UPLOAD_QUEUE_DELAY_MS;
|
||||
}
|
||||
|
||||
function _cfgConcurrency() {
|
||||
return (window.uploadConfig && window.uploadConfig.concurrency != null)
|
||||
? window.uploadConfig.concurrency
|
||||
: DEFAULT_UPLOAD_CONCURRENCY;
|
||||
}
|
||||
|
||||
/** Effective delay between upload slot starts (increased during backoff). */
|
||||
function _effectiveDelay() {
|
||||
return _adaptiveState.delayMs !== null ? _adaptiveState.delayMs : _cfgDelay();
|
||||
}
|
||||
|
||||
/** Fallback backoff multiplier when no Retry-After header is present. */
|
||||
const FALLBACK_BACKOFF_MULTIPLIER = 4;
|
||||
/** Minimum fallback pause duration (ms) when no Retry-After header is present. */
|
||||
const MIN_FALLBACK_BACKOFF_MS = 5000;
|
||||
/** Maximum inter-slot delay after repeated exponential backoff (ms). */
|
||||
const MAX_BACKOFF_DELAY_MS = 30000;
|
||||
|
||||
/** Effective concurrency (reduced to 1 during backoff). */
|
||||
function _effectiveConcurrency() {
|
||||
return _adaptiveState.concurrency !== null ? _adaptiveState.concurrency : _cfgConcurrency();
|
||||
}
|
||||
|
||||
/**
|
||||
* Read all file entries from a DirectoryReader, handling the 100-entry limit
|
||||
* by calling readEntries() repeatedly until it returns an empty batch.
|
||||
* Called when an HTTP 429 Too Many Requests response is received.
|
||||
* Pauses the queue and applies exponential backoff.
|
||||
* @param {number} retryAfterSeconds - Value of the Retry-After header (0 = absent).
|
||||
*/
|
||||
function _onRateLimited(retryAfterSeconds) {
|
||||
// Determine how long to pause – prefer the server's Retry-After; fall back to
|
||||
// FALLBACK_BACKOFF_MULTIPLIER × the current delay (minimum MIN_FALLBACK_BACKOFF_MS).
|
||||
const waitMs = retryAfterSeconds > 0
|
||||
? retryAfterSeconds * 1000
|
||||
: Math.max(_effectiveDelay() * FALLBACK_BACKOFF_MULTIPLIER, MIN_FALLBACK_BACKOFF_MS);
|
||||
|
||||
_adaptiveState.pauseUntil = Date.now() + waitMs;
|
||||
// Exponential backoff on the inter-slot delay, capped at MAX_BACKOFF_DELAY_MS.
|
||||
_adaptiveState.delayMs = Math.min(_effectiveDelay() * 2, MAX_BACKOFF_DELAY_MS);
|
||||
// Serialize uploads while we recover.
|
||||
_adaptiveState.concurrency = 1;
|
||||
_adaptiveState.consecutiveOk = 0;
|
||||
|
||||
console.warn(
|
||||
`[DocuElevate] Rate limited. Pausing ${waitMs} ms. ` +
|
||||
`New delay: ${_adaptiveState.delayMs} ms, concurrency: 1.`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called after each successful (non-429) upload.
|
||||
* After 5 consecutive successes, gently recovers toward the configured values.
|
||||
*/
|
||||
function _onUploadSuccess() {
|
||||
_adaptiveState.consecutiveOk++;
|
||||
if (_adaptiveState.consecutiveOk < 5) return;
|
||||
|
||||
// One recovery step every 5 successes.
|
||||
_adaptiveState.consecutiveOk = 0;
|
||||
const cfgDelay = _cfgDelay();
|
||||
const cfgConc = _cfgConcurrency();
|
||||
|
||||
if (_adaptiveState.delayMs !== null && _adaptiveState.delayMs > cfgDelay) {
|
||||
_adaptiveState.delayMs = Math.max(Math.round(_adaptiveState.delayMs * 0.75), cfgDelay);
|
||||
if (_adaptiveState.delayMs <= cfgDelay) _adaptiveState.delayMs = null; // fully recovered
|
||||
}
|
||||
if (_adaptiveState.concurrency !== null && _adaptiveState.concurrency < cfgConc) {
|
||||
_adaptiveState.concurrency = Math.min(_adaptiveState.concurrency + 1, cfgConc);
|
||||
if (_adaptiveState.concurrency >= cfgConc) _adaptiveState.concurrency = null; // fully recovered
|
||||
}
|
||||
}
|
||||
|
||||
// ── Directory traversal helpers ───────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Read all entries from a DirectoryReader, handling the browser's 100-entry
|
||||
* per-batch limit by calling readEntries() repeatedly.
|
||||
* @param {FileSystemDirectoryReader} reader
|
||||
* @returns {Promise<FileSystemEntry[]>}
|
||||
*/
|
||||
@@ -68,12 +182,9 @@ function readAllDirectoryEntries(reader) {
|
||||
const entries = [];
|
||||
function readBatch() {
|
||||
reader.readEntries((batch) => {
|
||||
if (batch.length === 0) {
|
||||
resolve(entries);
|
||||
} else {
|
||||
entries.push(...batch);
|
||||
readBatch();
|
||||
}
|
||||
if (batch.length === 0) { resolve(entries); return; }
|
||||
entries.push(...batch);
|
||||
readBatch();
|
||||
}, reject);
|
||||
}
|
||||
readBatch();
|
||||
@@ -83,7 +194,7 @@ function readAllDirectoryEntries(reader) {
|
||||
/**
|
||||
* Recursively collect all File objects from a FileSystemEntry tree.
|
||||
* @param {FileSystemEntry} entry
|
||||
* @param {File[]} files - accumulator array
|
||||
* @param {File[]} files - accumulator
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function traverseFileEntry(entry, files) {
|
||||
@@ -92,8 +203,7 @@ async function traverseFileEntry(entry, files) {
|
||||
entry.file((file) => { files.push(file); resolve(); }, resolve);
|
||||
});
|
||||
} else if (entry.isDirectory) {
|
||||
const reader = entry.createReader();
|
||||
const subEntries = await readAllDirectoryEntries(reader);
|
||||
const subEntries = await readAllDirectoryEntries(entry.createReader());
|
||||
for (const sub of subEntries) {
|
||||
await traverseFileEntry(sub, files);
|
||||
}
|
||||
@@ -102,18 +212,16 @@ async function traverseFileEntry(entry, files) {
|
||||
|
||||
/**
|
||||
* Extract all File objects from a DataTransfer, recursively expanding any
|
||||
* dropped directories. Falls back gracefully to dataTransfer.files when
|
||||
* the FileSystem Entry API is unavailable.
|
||||
* dropped directories. Falls back gracefully to dataTransfer.files when the
|
||||
* FileSystem Entry API is unavailable (Safari < 11.1, some mobile browsers).
|
||||
* @param {DataTransfer} dataTransfer
|
||||
* @returns {Promise<File[]>}
|
||||
*/
|
||||
async function getFilesFromDataTransfer(dataTransfer) {
|
||||
// Use the FileSystem Entry API when available (all modern browsers)
|
||||
if (dataTransfer.items && dataTransfer.items.length > 0) {
|
||||
const files = [];
|
||||
const itemList = dataTransfer.items;
|
||||
for (let i = 0; i < itemList.length; i++) {
|
||||
const item = itemList[i];
|
||||
for (let i = 0; i < dataTransfer.items.length; i++) {
|
||||
const item = dataTransfer.items[i];
|
||||
const entry = item.webkitGetAsEntry ? item.webkitGetAsEntry() : null;
|
||||
if (entry) {
|
||||
await traverseFileEntry(entry, files);
|
||||
@@ -124,244 +232,261 @@ async function getFilesFromDataTransfer(dataTransfer) {
|
||||
}
|
||||
return files;
|
||||
}
|
||||
// Fallback: plain FileList (no directory support)
|
||||
return Array.from(dataTransfer.files || []);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Queue-based upload runner
|
||||
// ---------------------------------------------------------------------------
|
||||
// ── Core queue runner ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Upload a list of files using a concurrency-limited queue with a configurable
|
||||
* delay between slot starts to prevent server overload.
|
||||
* Validate and queue files for upload with adaptive throttling.
|
||||
*
|
||||
* @param {File[]} files - Files to upload
|
||||
* @param {HTMLElement} progressContainer - Container element for progress display
|
||||
* @param {HTMLElement} statusMessage - Element for status message display
|
||||
* Files are pre-rendered as progress rows so the user immediately sees the
|
||||
* full list. The queue runner respects the current effective concurrency and
|
||||
* delay, slowing down automatically when the server signals rate limiting (429).
|
||||
*
|
||||
* @param {File[]|FileList} files
|
||||
* @param {HTMLElement} progressContainer
|
||||
* @param {HTMLElement} statusMessage
|
||||
*/
|
||||
function processFiles(files, progressContainer, statusMessage) {
|
||||
if (files.length === 0) return;
|
||||
|
||||
const concurrency = (window.uploadConfig && window.uploadConfig.concurrency) || DEFAULT_UPLOAD_CONCURRENCY;
|
||||
const delayMs = (window.uploadConfig && window.uploadConfig.queueDelayMs) || DEFAULT_UPLOAD_QUEUE_DELAY_MS;
|
||||
const fileArray = Array.from(files);
|
||||
if (!fileArray.length) return;
|
||||
|
||||
if (statusMessage) {
|
||||
statusMessage.textContent = `Queued ${files.length} file(s) for upload…`;
|
||||
statusMessage.textContent = `Queued ${fileArray.length} file(s) for upload…`;
|
||||
}
|
||||
if (progressContainer) progressContainer.innerHTML = '';
|
||||
|
||||
// Clear previous upload progress
|
||||
if (progressContainer) {
|
||||
progressContainer.innerHTML = "";
|
||||
}
|
||||
|
||||
// Pre-create all progress elements so the user sees the full list immediately
|
||||
const progressElements = files.map((file) => {
|
||||
const fileProgress = document.createElement("div");
|
||||
fileProgress.className = "flex flex-col mb-2";
|
||||
fileProgress.innerHTML = `
|
||||
// Pre-create one progress row per file.
|
||||
const queueItems = fileArray.map((file) => {
|
||||
const row = document.createElement('div');
|
||||
row.className = 'flex flex-col mb-2';
|
||||
row.innerHTML = `
|
||||
<div class="flex justify-between">
|
||||
<span class="text-sm truncate" title="${file.name}">${file.name}</span>
|
||||
<span class="text-xs text-gray-500">${formatFileSize(file.size)}</span>
|
||||
</div>
|
||||
<div class="w-full bg-gray-200 h-2 rounded-full mt-1">
|
||||
<div class="file-progress-bar bg-gray-300 h-2 rounded-full" style="width: 0%"></div>
|
||||
<div class="file-progress-bar bg-gray-300 h-2 rounded-full" style="width:0%"></div>
|
||||
</div>
|
||||
<div class="file-status text-xs text-gray-400 mt-1">Queued</div>
|
||||
`;
|
||||
if (progressContainer) progressContainer.appendChild(fileProgress);
|
||||
return fileProgress;
|
||||
if (progressContainer) progressContainer.appendChild(row);
|
||||
return {
|
||||
file,
|
||||
progressBar: row.querySelector('.file-progress-bar'),
|
||||
statusEl: row.querySelector('.file-status'),
|
||||
retryCount: 0,
|
||||
};
|
||||
});
|
||||
|
||||
// Queue runner
|
||||
let index = 0;
|
||||
// Mutable queue – rate-limited items are pushed back to the front.
|
||||
const queue = [...queueItems];
|
||||
let active = 0;
|
||||
|
||||
function startNext() {
|
||||
while (active < concurrency && index < files.length) {
|
||||
const i = index++;
|
||||
function scheduleNext() {
|
||||
// Respect global backoff pause.
|
||||
const pauseRemaining = _adaptiveState.pauseUntil - Date.now();
|
||||
if (pauseRemaining > 0) {
|
||||
setTimeout(scheduleNext, pauseRemaining + 50);
|
||||
return;
|
||||
}
|
||||
|
||||
while (active < _effectiveConcurrency() && queue.length > 0) {
|
||||
const item = queue.shift();
|
||||
active++;
|
||||
const progressBar = progressElements[i].querySelector(".file-progress-bar");
|
||||
const statusEl = progressElements[i].querySelector(".file-status");
|
||||
validateAndUploadQueued(files[i], progressBar, statusEl, statusMessage).finally(() => {
|
||||
|
||||
// Validate before hitting the network.
|
||||
if (!_isAcceptedFile(item.file)) {
|
||||
item.progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
|
||||
item.statusEl.textContent = 'Unsupported file type';
|
||||
item.statusEl.className = 'text-xs text-red-500 mt-1';
|
||||
active--;
|
||||
setTimeout(startNext, delayMs);
|
||||
});
|
||||
updateOverallStatus(statusMessage);
|
||||
// No HTTP request – skip straight to next without adding delay.
|
||||
scheduleNext();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (item.file.size > MAX_FILE_SIZE) {
|
||||
item.progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
|
||||
item.statusEl.textContent = 'Exceeds 500 MB limit';
|
||||
item.statusEl.className = 'text-xs text-red-500 mt-1';
|
||||
active--;
|
||||
updateOverallStatus(statusMessage);
|
||||
scheduleNext();
|
||||
continue;
|
||||
}
|
||||
|
||||
_uploadSingleFile(item.file, item.progressBar, item.statusEl, statusMessage)
|
||||
.then((result) => {
|
||||
active--;
|
||||
if (result.rateLimited) {
|
||||
_onRateLimited(result.retryAfterSeconds);
|
||||
item.retryCount++;
|
||||
if (item.retryCount < MAX_RATE_LIMIT_RETRIES) {
|
||||
// Re-insert at the front of the queue to retry after the pause.
|
||||
queue.unshift(item);
|
||||
} else {
|
||||
item.progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
|
||||
item.statusEl.textContent = 'Failed: rate limit retries exhausted';
|
||||
item.statusEl.className = 'text-xs text-red-500 mt-1';
|
||||
updateOverallStatus(statusMessage);
|
||||
}
|
||||
// Resume after the backoff window.
|
||||
const wait = Math.max(_adaptiveState.pauseUntil - Date.now() + 50, 0);
|
||||
setTimeout(scheduleNext, wait);
|
||||
} else {
|
||||
// Success or permanent error – wait the configured delay before next slot.
|
||||
setTimeout(scheduleNext, _effectiveDelay());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
startNext();
|
||||
scheduleNext();
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and upload a single file (used by the queue runner).
|
||||
* Check whether a file passes MIME type and extension validation.
|
||||
* @param {File} file
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function _isAcceptedFile(file) {
|
||||
if (ACCEPTED_TYPES[file.type]) return true;
|
||||
const ext = '.' + file.name.split('.').pop().toLowerCase();
|
||||
return ACCEPTED_EXTENSIONS.has(ext);
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a single file via XHR, returning a structured result.
|
||||
* Detects HTTP 429 responses and reads the Retry-After / X-RateLimit-Reset
|
||||
* headers so the caller can apply precise backoff.
|
||||
*
|
||||
* @param {File} file
|
||||
* @param {HTMLElement} progressBar
|
||||
* @param {HTMLElement} statusEl
|
||||
* @param {HTMLElement} statusMessage
|
||||
* @returns {Promise<void>}
|
||||
* @returns {Promise<{rateLimited: boolean, retryAfterSeconds: number}>}
|
||||
*/
|
||||
function validateAndUploadQueued(file, progressBar, statusEl, statusMessage) {
|
||||
// Validate file type by checking both MIME type and extension
|
||||
const isValidMimeType = ACCEPTED_TYPES[file.type] || false;
|
||||
const fileExtension = '.' + file.name.split('.').pop().toLowerCase();
|
||||
const isValidExtension = ACCEPTED_EXTENSIONS.includes(fileExtension);
|
||||
|
||||
if (!isValidMimeType && !isValidExtension) {
|
||||
statusEl.textContent = `Unsupported file type`;
|
||||
statusEl.className = "text-xs text-red-500 mt-1";
|
||||
progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
|
||||
updateOverallStatus(statusMessage);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
if (file.size > MAX_FILE_SIZE) {
|
||||
statusEl.textContent = `Exceeds 500 MB limit`;
|
||||
statusEl.className = "text-xs text-red-500 mt-1";
|
||||
progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
|
||||
updateOverallStatus(statusMessage);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
return uploadFile(file, progressBar, statusEl, statusMessage);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and upload a single file (legacy entry point kept for compatibility).
|
||||
* @param {File} file - File to validate and upload
|
||||
* @param {HTMLElement} progressContainer - Container element for progress display
|
||||
* @param {HTMLElement} statusMessage - Element for status message display
|
||||
*/
|
||||
function validateAndUpload(file, progressContainer, statusMessage) {
|
||||
// Create progress element for this file
|
||||
const fileProgress = document.createElement("div");
|
||||
fileProgress.className = "flex flex-col mb-2";
|
||||
fileProgress.innerHTML = `
|
||||
<div class="flex justify-between">
|
||||
<span class="text-sm truncate" title="${file.name}">${file.name}</span>
|
||||
<span class="text-xs text-gray-500">${formatFileSize(file.size)}</span>
|
||||
</div>
|
||||
<div class="w-full bg-gray-200 h-2 rounded-full mt-1">
|
||||
<div class="file-progress-bar bg-blue-500 h-2 rounded-full" style="width: 0%"></div>
|
||||
</div>
|
||||
<div class="file-status text-xs text-gray-600 mt-1">Validating...</div>
|
||||
`;
|
||||
|
||||
if (progressContainer) {
|
||||
progressContainer.appendChild(fileProgress);
|
||||
}
|
||||
|
||||
const progressBar = fileProgress.querySelector(".file-progress-bar");
|
||||
const statusEl = fileProgress.querySelector(".file-status");
|
||||
|
||||
validateAndUploadQueued(file, progressBar, statusEl, statusMessage);
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a file to the server
|
||||
* @param {File} file - File to upload
|
||||
* @param {HTMLElement} progressBar - Progress bar element
|
||||
* @param {HTMLElement} statusEl - Status element
|
||||
* @param {HTMLElement} statusMessage - Overall status message element
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
function uploadFile(file, progressBar, statusEl, statusMessage) {
|
||||
statusEl.textContent = `Uploading…`;
|
||||
statusEl.className = "text-xs text-gray-600 mt-1";
|
||||
progressBar.className = "file-progress-bar bg-blue-500 h-2 rounded-full";
|
||||
function _uploadSingleFile(file, progressBar, statusEl, statusMessage) {
|
||||
statusEl.textContent = 'Uploading…';
|
||||
statusEl.className = 'text-xs text-gray-600 mt-1';
|
||||
progressBar.style.width = '0%';
|
||||
progressBar.className = 'file-progress-bar bg-blue-500 h-2 rounded-full';
|
||||
|
||||
return new Promise((resolve) => {
|
||||
try {
|
||||
const formData = new FormData();
|
||||
formData.append("file", file);
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open("POST", "/api/ui-upload", true);
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', '/api/ui-upload', true);
|
||||
|
||||
// Attach CSRF token so the server-side CSRF middleware accepts the request.
|
||||
const csrfToken = typeof getCsrfToken === 'function' ? getCsrfToken() : '';
|
||||
if (csrfToken) {
|
||||
xhr.setRequestHeader("X-CSRF-Token", csrfToken);
|
||||
const csrfToken = typeof getCsrfToken === 'function' ? getCsrfToken() : '';
|
||||
if (csrfToken) xhr.setRequestHeader('X-CSRF-Token', csrfToken);
|
||||
|
||||
xhr.upload.onprogress = (e) => {
|
||||
if (e.lengthComputable) {
|
||||
const pct = Math.round((e.loaded / e.total) * 100);
|
||||
progressBar.style.width = pct + '%';
|
||||
statusEl.textContent = `Uploading: ${pct}%`;
|
||||
}
|
||||
};
|
||||
|
||||
xhr.upload.onprogress = (e) => {
|
||||
if (e.lengthComputable) {
|
||||
const percentComplete = (e.loaded / e.total) * 100;
|
||||
progressBar.style.width = percentComplete + "%";
|
||||
statusEl.textContent = `Uploading: ${Math.round(percentComplete)}%`;
|
||||
}
|
||||
};
|
||||
|
||||
xhr.onload = function () {
|
||||
if (xhr.status === 200) {
|
||||
const result = JSON.parse(xhr.responseText);
|
||||
progressBar.style.width = "100%";
|
||||
progressBar.className = "file-progress-bar bg-green-500 h-2 rounded-full";
|
||||
statusEl.textContent = `Success: Task ID: ${result.task_id}`;
|
||||
statusEl.className = "text-xs text-green-600 mt-1";
|
||||
} else {
|
||||
progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
|
||||
statusEl.textContent = `Error: Upload failed (HTTP ${xhr.status})`;
|
||||
statusEl.className = "text-xs text-red-500 mt-1";
|
||||
}
|
||||
xhr.onload = () => {
|
||||
if (xhr.status === 200) {
|
||||
const result = JSON.parse(xhr.responseText);
|
||||
progressBar.style.width = '100%';
|
||||
progressBar.className = 'file-progress-bar bg-green-500 h-2 rounded-full';
|
||||
statusEl.textContent = `Success: Task ID: ${result.task_id}`;
|
||||
statusEl.className = 'text-xs text-green-600 mt-1';
|
||||
_onUploadSuccess();
|
||||
updateOverallStatus(statusMessage);
|
||||
resolve();
|
||||
};
|
||||
resolve({ rateLimited: false, retryAfterSeconds: 0 });
|
||||
|
||||
xhr.onerror = function () {
|
||||
progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
|
||||
statusEl.textContent = `Error: Network error`;
|
||||
statusEl.className = "text-xs text-red-500 mt-1";
|
||||
} else if (xhr.status === 429) {
|
||||
// Parse Retry-After (seconds integer).
|
||||
let retryAfter = parseInt(xhr.getResponseHeader('Retry-After') || '0', 10);
|
||||
if (!retryAfter) {
|
||||
// Fall back to X-RateLimit-Reset (Unix timestamp).
|
||||
const reset = parseInt(xhr.getResponseHeader('X-RateLimit-Reset') || '0', 10);
|
||||
if (reset) retryAfter = Math.max(reset - Math.floor(Date.now() / 1000), 1);
|
||||
}
|
||||
progressBar.className = 'file-progress-bar bg-yellow-400 h-2 rounded-full';
|
||||
statusEl.textContent = 'Rate limited – queued to retry…';
|
||||
statusEl.className = 'text-xs text-yellow-600 mt-1';
|
||||
resolve({ rateLimited: true, retryAfterSeconds: retryAfter });
|
||||
|
||||
} else {
|
||||
progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
|
||||
statusEl.textContent = `Error: HTTP ${xhr.status}`;
|
||||
statusEl.className = 'text-xs text-red-500 mt-1';
|
||||
updateOverallStatus(statusMessage);
|
||||
resolve();
|
||||
};
|
||||
resolve({ rateLimited: false, retryAfterSeconds: 0 });
|
||||
}
|
||||
};
|
||||
|
||||
xhr.send(formData);
|
||||
} catch (err) {
|
||||
statusEl.textContent = `Error: ${err.message}`;
|
||||
statusEl.className = "text-xs text-red-500 mt-1";
|
||||
progressBar.className = "file-progress-bar bg-red-500 h-2 rounded-full";
|
||||
xhr.onerror = () => {
|
||||
progressBar.className = 'file-progress-bar bg-red-500 h-2 rounded-full';
|
||||
statusEl.textContent = 'Error: Network error';
|
||||
statusEl.className = 'text-xs text-red-500 mt-1';
|
||||
updateOverallStatus(statusMessage);
|
||||
resolve();
|
||||
}
|
||||
resolve({ rateLimited: false, retryAfterSeconds: 0 });
|
||||
};
|
||||
|
||||
xhr.send(formData);
|
||||
});
|
||||
}
|
||||
|
||||
// ── Legacy single-file entry point (kept for backward compat) ─────────────────
|
||||
|
||||
/**
|
||||
* Update the overall status message based on file statuses
|
||||
* @param {HTMLElement} statusMessage - Status message element
|
||||
* Validate and upload a single file (legacy path – wraps the queue runner).
|
||||
* @param {File} file
|
||||
* @param {HTMLElement} progressContainer
|
||||
* @param {HTMLElement} statusMessage
|
||||
*/
|
||||
function validateAndUpload(file, progressContainer, statusMessage) {
|
||||
processFiles([file], progressContainer, statusMessage);
|
||||
}
|
||||
|
||||
// ── Status helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Re-calculate and display the overall upload status.
|
||||
* Fires 'allUploadsComplete' when every item has a terminal status.
|
||||
* @param {HTMLElement} statusMessage
|
||||
*/
|
||||
function updateOverallStatus(statusMessage) {
|
||||
if (!statusMessage) return;
|
||||
|
||||
// Count success/failure
|
||||
const fileStatuses = document.querySelectorAll('.file-status');
|
||||
let completed = 0;
|
||||
let done = 0;
|
||||
const total = fileStatuses.length;
|
||||
|
||||
fileStatuses.forEach(status => {
|
||||
if (status.textContent.includes('Success') || status.textContent.includes('Error') || status.textContent.includes('Unsupported') || status.textContent.includes('Exceeds')) {
|
||||
completed++;
|
||||
}
|
||||
fileStatuses.forEach((s) => {
|
||||
const t = s.textContent;
|
||||
if (
|
||||
t.startsWith('Success') ||
|
||||
t.startsWith('Error') ||
|
||||
t.startsWith('Unsupported') ||
|
||||
t.startsWith('Exceeds') ||
|
||||
t.startsWith('Failed:')
|
||||
) done++;
|
||||
});
|
||||
|
||||
if (completed === total) {
|
||||
statusMessage.textContent = `All uploads completed (${completed}/${total})`;
|
||||
|
||||
// Trigger a custom event when all uploads are complete
|
||||
const allUploadsComplete = new CustomEvent('allUploadsComplete', {
|
||||
detail: { total: total, completed: completed }
|
||||
});
|
||||
window.dispatchEvent(allUploadsComplete);
|
||||
if (done === total && total > 0) {
|
||||
statusMessage.textContent = `All uploads completed (${done}/${total})`;
|
||||
window.dispatchEvent(new CustomEvent('allUploadsComplete', { detail: { total, completed: done } }));
|
||||
} else {
|
||||
statusMessage.textContent = `Uploading files (${completed}/${total})`;
|
||||
statusMessage.textContent = `Uploading files (${done}/${total})`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format file size for display
|
||||
* @param {number} bytes - File size in bytes
|
||||
* @returns {string} Formatted file size
|
||||
* Format a byte count for human-readable display.
|
||||
* @param {number} bytes
|
||||
* @returns {string}
|
||||
*/
|
||||
function formatFileSize(bytes) {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
@@ -371,54 +496,43 @@ function formatFileSize(bytes) {
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
// ── Drag-and-drop initialiser ─────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Initialize drag-and-drop on an element with directory support.
|
||||
* @param {HTMLElement} element - Element to enable drag-and-drop on
|
||||
* @param {HTMLElement} progressContainer - Container for progress display
|
||||
* @param {HTMLElement} statusMessage - Element for status messages
|
||||
* @param {Object} options - Additional options
|
||||
* Wire up drag-and-drop on an element with full directory-traversal support.
|
||||
* @param {HTMLElement} element
|
||||
* @param {HTMLElement} progressContainer
|
||||
* @param {HTMLElement} statusMessage
|
||||
* @param {Object} [options]
|
||||
* @param {string} [options.dragOverClass] - CSS class added during drag-over
|
||||
*/
|
||||
function initDragAndDrop(element, progressContainer, statusMessage, options = {}) {
|
||||
if (!element) {
|
||||
console.error("Element not found for drag-and-drop initialization");
|
||||
console.error('[DocuElevate] Element not found for drag-and-drop initialization');
|
||||
return;
|
||||
}
|
||||
|
||||
// Add event listeners for drag-and-drop
|
||||
element.addEventListener("dragover", (e) => {
|
||||
element.addEventListener('dragover', (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
e.dataTransfer.dropEffect = "copy";
|
||||
|
||||
// Add visual feedback
|
||||
if (options.dragOverClass) {
|
||||
element.classList.add(options.dragOverClass);
|
||||
}
|
||||
e.dataTransfer.dropEffect = 'copy';
|
||||
if (options.dragOverClass) element.classList.add(options.dragOverClass);
|
||||
});
|
||||
|
||||
element.addEventListener("dragleave", (e) => {
|
||||
element.addEventListener('dragleave', (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
|
||||
// Remove visual feedback
|
||||
if (options.dragOverClass) {
|
||||
element.classList.remove(options.dragOverClass);
|
||||
}
|
||||
if (options.dragOverClass) element.classList.remove(options.dragOverClass);
|
||||
});
|
||||
|
||||
element.addEventListener("drop", async (e) => {
|
||||
element.addEventListener('drop', async (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
|
||||
// Remove visual feedback
|
||||
if (options.dragOverClass) {
|
||||
element.classList.remove(options.dragOverClass);
|
||||
}
|
||||
if (options.dragOverClass) element.classList.remove(options.dragOverClass);
|
||||
|
||||
const files = await getFilesFromDataTransfer(e.dataTransfer);
|
||||
if (files.length > 0) {
|
||||
processFiles(files, progressContainer, statusMessage);
|
||||
}
|
||||
if (files.length > 0) processFiles(files, progressContainer, statusMessage);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,12 @@
|
||||
|
||||
{% block head_extra %}
|
||||
<script src="/static/js/common.js"></script>
|
||||
<script>
|
||||
window.uploadConfig = {
|
||||
concurrency: {{ upload_concurrency }},
|
||||
queueDelayMs: {{ upload_queue_delay_ms }}
|
||||
};
|
||||
</script>
|
||||
<script src="/static/js/upload.js"></script>
|
||||
<style>
|
||||
/* Drop overlay styles - covers entire page for drag-and-drop anywhere */
|
||||
@@ -342,8 +348,8 @@
|
||||
<div id="dropOverlay" class="drop-overlay">
|
||||
<div class="drop-message">
|
||||
<i class="fas fa-cloud-upload-alt"></i>
|
||||
<p>Drop files anywhere to upload</p>
|
||||
<div class="drop-hint">Supports PDF, Office docs, images, and more</div>
|
||||
<p>Drop files or folders anywhere to upload</p>
|
||||
<div class="drop-hint">Supports PDF, Office docs, images, HTML, Markdown, and more – folders are processed recursively</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -902,18 +908,19 @@
|
||||
e.dataTransfer.dropEffect = 'copy';
|
||||
});
|
||||
|
||||
window.addEventListener('drop', (e) => {
|
||||
window.addEventListener('drop', async (e) => {
|
||||
e.preventDefault();
|
||||
dragCounter = 0;
|
||||
dropOverlay.classList.remove('active');
|
||||
|
||||
if (e.dataTransfer.files.length > 0) {
|
||||
const files = await getFilesFromDataTransfer(e.dataTransfer);
|
||||
if (files.length > 0) {
|
||||
// Show upload modal
|
||||
uploadModal.classList.add('active');
|
||||
uploadProgressContainer.innerHTML = '';
|
||||
|
||||
// Process the dropped files
|
||||
processFiles(e.dataTransfer.files, uploadProgressContainer, uploadStatusMessage);
|
||||
// Process the dropped files (queue-based, with adaptive throttling)
|
||||
processFiles(files, uploadProgressContainer, uploadStatusMessage);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user