fix(upload): reject exact duplicates at upload time and prevent duplicate mobile uploads

- Move duplicate check before task enqueue in ui_upload endpoint
- Clean up temp file and return status "duplicate" for exact duplicates
- Add URI-level dedup guard in mobile UploadScreen to prevent repeated uploads
- Improve ShareContext URI normalization (collapse slashes, decode percent-encoding)
- Guard +not-found.tsx effect against re-firing for the same pathname
- Update mobile UploadResponse type and handlers for duplicate status
- Update web frontend upload.js to show duplicate status
- Update API and Configuration docs

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-19 14:07:43 +00:00
parent cde966012c
commit d5c18ccf07
9 changed files with 264 additions and 69 deletions
+28 -12
View File
@@ -1268,7 +1268,12 @@ async def _save_upload_file_chunks(file: UploadFile, target_path: str, max_size:
def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: str) -> dict | None:
"""Check for an exact duplicate of the uploaded file and return a warning if found."""
"""Check for an exact duplicate of the uploaded file.
Returns a dict with duplicate info when the file's SHA-256 hash matches an
already-processed document, or ``None`` when no duplicate is found (or
deduplication is disabled).
"""
if not settings.enable_deduplication:
return None
@@ -1287,8 +1292,8 @@ def _check_for_exact_duplicate(db: DbSession, target_path: str, safe_filename: s
"original_file_id": existing.id,
"original_filename": existing.original_filename,
"message": (
"This file appears to be an exact duplicate of an already-processed document. "
"It will still be queued but will be flagged as a duplicate."
"This file is an exact duplicate of an already-processed document. "
"It has not been queued for processing again."
),
}
except Exception as e:
@@ -1390,6 +1395,25 @@ async def ui_upload(
logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")
file_size = written_size
# ── Early duplicate rejection ──────────────────────────────────────────
# Check for exact duplicates (same SHA-256 hash) BEFORE enqueuing a
# processing task. When deduplication is enabled and the file already
# exists, we skip processing entirely, clean up the temp file, and
# return the existing file's information to the caller.
exact_duplicate = _check_for_exact_duplicate(db, target_path, safe_filename)
if exact_duplicate:
# Remove the just-saved temp file — it's a duplicate.
try:
os.remove(target_path)
except OSError:
pass
return {
"status": "duplicate",
"original_filename": safe_filename,
"stored_filename": target_filename,
"duplicate_of": exact_duplicate,
}
# Determine if the file is a PDF or needs conversion
mime_type, _ = mimetypes.guess_type(target_path)
file_ext = os.path.splitext(target_path)[1].lower()
@@ -1466,20 +1490,12 @@ async def ui_upload(
logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
task = convert_to_pdf.delay(target_path, original_filename=safe_filename, owner_id=upload_owner_id)
# Check for exact duplicates (same SHA-256 hash) before returning.
# This gives the caller an immediate warning without waiting for the pipeline.
# Only performed when deduplication is enabled in settings.
exact_duplicate_warning = _check_for_exact_duplicate(db, target_path, safe_filename)
response: dict = {
return {
"task_id": task.id,
"status": "queued",
"original_filename": safe_filename,
"stored_filename": target_filename,
}
if exact_duplicate_warning:
response["duplicate_warning"] = exact_duplicate_warning
return response
# ---------------------------------------------------------------------------