From ebea83a750022f9722e548ba8a6ecea179ac2d8b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 2 Mar 2026 10:21:57 +0000 Subject: [PATCH] feat(duplicates): add duplicate document detection and management - Add near_duplicate_threshold config setting (default 0.85) - New GET /api/duplicates endpoint listing all exact-duplicate groups - New GET /api/files/{id}/duplicates endpoint returning exact + near-duplicates - POST /api/ui-upload now returns immediate exact-duplicate warning (respects ENABLE_DEDUPLICATION) - New /duplicates management UI with Exact Duplicates tab and Near-Duplicate Finder tab - Add Duplicates link in admin nav menu (desktop + mobile) - Document new config options in ConfigurationGuide.md and .env.demo - 20 new tests covering all acceptance criteria Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- .env.demo | 12 +- app/api/__init__.py | 2 + app/api/duplicates.py | 230 ++++++++++++++ app/api/files.py | 35 ++- app/config.py | 7 + app/views/files.py | 97 ++++++ docs/ConfigurationGuide.md | 32 ++ frontend/templates/base.html | 6 + frontend/templates/duplicates.html | 471 +++++++++++++++++++++++++++++ tests/test_duplicates.py | 404 +++++++++++++++++++++++++ 10 files changed, 1293 insertions(+), 3 deletions(-) create mode 100644 app/api/duplicates.py create mode 100644 frontend/templates/duplicates.html create mode 100644 tests/test_duplicates.py diff --git a/.env.demo b/.env.demo index acb8d950..e06ab45b 100644 --- a/.env.demo +++ b/.env.demo @@ -336,4 +336,14 @@ MEILISEARCH_URL=http://meilisearch:7700 # Optional master/API key for secured Meilisearch instances # MEILISEARCH_API_KEY=your_master_key_here MEILISEARCH_INDEX_NAME=documents -ENABLE_SEARCH=True +ENABLE_SEARCH=True + +# **Duplicate Detection** +# Exact duplicate detection (SHA-256) is always on during document processing. +# The settings below control near-duplicate detection (same scanned content, +# different hash) and the visibility of deduplication steps. +ENABLE_DEDUPLICATION=True +SHOW_DEDUPLICATION_STEP=True +# Minimum cosine similarity score (0–1) for two documents to be flagged as +# near-duplicates. 0.85 means 85 % semantic overlap. Lower = more matches. +NEAR_DUPLICATE_THRESHOLD=0.85 diff --git a/app/api/__init__.py b/app/api/__init__.py index fe197dfb..d340418a 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -9,6 +9,7 @@ from fastapi import APIRouter from app.api.azure import router as azure_router from app.api.diagnostic import router as diagnostic_router from app.api.dropbox import router as dropbox_router +from app.api.duplicates import router as duplicates_router from app.api.files import router as files_router from app.api.google_drive import router as google_drive_router from app.api.logs import router as logs_router @@ -49,4 +50,5 @@ router.include_router(search_router) router.include_router(queue_router) router.include_router(saved_searches_router) router.include_router(similarity_router) +router.include_router(duplicates_router) router.include_router(webhooks_router) diff --git a/app/api/duplicates.py b/app/api/duplicates.py new file mode 100644 index 00000000..b5ab396d --- /dev/null +++ b/app/api/duplicates.py @@ -0,0 +1,230 @@ +"""Duplicate document detection and management API endpoints. + +Provides endpoints for listing all duplicate groups (exact SHA-256 duplicates) and +for retrieving both exact and near-duplicate matches for a specific document. + +Near-duplicate detection is powered by the same text-embedding cosine-similarity +engine used by the ``/api/files/{id}/similar`` endpoint +(see ``app/utils/similarity.py``). +""" + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from sqlalchemy.orm import Session + +from app.auth import require_login +from app.config import settings +from app.database import get_db +from app.models import FileRecord + +logger = logging.getLogger(__name__) + +router = APIRouter() + +DbSession = Annotated[Session, Depends(get_db)] + + +@router.get("/duplicates") +@require_login +def list_duplicate_groups( + request: Request, + db: DbSession, + page: int = Query(1, ge=1, description="Page number"), + per_page: int = Query(25, ge=1, le=200, description="Items per page"), +): + """List all groups of exact-duplicate documents (same SHA-256 hash). + + Returns one entry per duplicate group showing the original document and all + files that were detected as copies of it. Groups are sorted by descending + duplicate count. + + Example: + ``` + GET /api/duplicates + ``` + + Response: + ```json + { + "groups": [ + { + "filehash": "abc123...", + "original": {"id": 1, "original_filename": "invoice.pdf", ...}, + "duplicates": [{"id": 5, "original_filename": "invoice_copy.pdf", ...}], + "duplicate_count": 1 + } + ], + "total_groups": 1, + "total_duplicate_files": 1, + "pagination": {...} + } + ``` + """ + # Find all hashes that have at least one duplicate record + dup_hashes_query = db.query(FileRecord.filehash).filter(FileRecord.is_duplicate.is_(True)).distinct() + total_groups = dup_hashes_query.count() + + # Paginate hash groups + offset = (page - 1) * per_page + dup_hashes = [row.filehash for row in dup_hashes_query.offset(offset).limit(per_page).all()] + + groups = [] + total_duplicate_files = 0 + + for filehash in dup_hashes: + # Find the original (non-duplicate) record with this hash + original = ( + db.query(FileRecord) + .filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False)) + .order_by(FileRecord.id.asc()) + .first() + ) + + # Find all duplicate records for this hash + duplicates = ( + db.query(FileRecord) + .filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(True)) + .order_by(FileRecord.id.asc()) + .all() + ) + + total_duplicate_files += len(duplicates) + + groups.append( + { + "filehash": filehash, + "original": _file_record_to_dict(original) if original else None, + "duplicates": [_file_record_to_dict(d) for d in duplicates], + "duplicate_count": len(duplicates), + } + ) + + total_pages = (total_groups + per_page - 1) // per_page if total_groups > 0 else 1 + + return { + "groups": groups, + "total_groups": total_groups, + "total_duplicate_files": total_duplicate_files, + "pagination": { + "page": page, + "per_page": per_page, + "total": total_groups, + "pages": total_pages, + "next": str(request.url.include_query_params(page=page + 1)) if page < total_pages else None, + "previous": str(request.url.include_query_params(page=page - 1)) if page > 1 else None, + }, + } + + +@router.get("/files/{file_id}/duplicates") +@require_login +def get_file_duplicates( + request: Request, + file_id: int, + db: DbSession, + near_duplicate_limit: int = Query(5, ge=1, le=20, description="Maximum near-duplicates to return"), + near_duplicate_threshold: float = Query( + -1.0, + ge=-1.0, + le=1.0, + description="Minimum similarity score for near-duplicates; -1 uses the configured default", + ), +): + """Get exact and near-duplicate documents for the specified file. + + **Exact duplicates** share the same SHA-256 hash. + **Near-duplicates** have a text-embedding cosine similarity score ≥ + ``NEAR_DUPLICATE_THRESHOLD`` (configurable; default 0.85). + + Near-duplicate detection requires OCR text to be available for both the + target file and candidate files. Files without OCR text are excluded. + + Example: + ``` + GET /api/files/42/duplicates + ``` + + Response: + ```json + { + "file_id": 42, + "exact_duplicates": [ + {"id": 7, "original_filename": "invoice.pdf", "is_duplicate": true, "duplicate_of_id": 42, ...} + ], + "near_duplicates": [ + {"file_id": 15, "original_filename": "invoice_jan.pdf", "similarity_score": 0.92, ...} + ], + "near_duplicate_threshold": 0.85 + } + ``` + """ + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + if not file_record: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") + + # --- Exact duplicates --- + # Case 1: This file is the original — find all records that are duplicates of it + exact_duplicates_of_this = ( + db.query(FileRecord) + .filter(FileRecord.filehash == file_record.filehash, FileRecord.id != file_id) + .order_by(FileRecord.id.asc()) + .all() + ) + + # Case 2: This file itself is a duplicate — find the original + is_self_duplicate = file_record.is_duplicate + duplicate_of_original: FileRecord | None = None + if is_self_duplicate and file_record.duplicate_of_id: + duplicate_of_original = db.query(FileRecord).filter(FileRecord.id == file_record.duplicate_of_id).first() + + exact_duplicate_dicts = [_file_record_to_dict(f) for f in exact_duplicates_of_this] + + # --- Near-duplicates (embedding-based) --- + effective_threshold = ( + near_duplicate_threshold if near_duplicate_threshold >= 0.0 else settings.near_duplicate_threshold + ) + + near_duplicates: list[dict] = [] + if file_record.ocr_text and file_record.ocr_text.strip(): + try: + from app.utils.similarity import find_similar_documents + + near_duplicates = find_similar_documents( + db, + file_id, + limit=near_duplicate_limit, + threshold=effective_threshold, + ) + except Exception as e: + logger.warning(f"Near-duplicate detection failed for file {file_id}: {e}") + near_duplicates = [] + + return { + "file_id": file_id, + "is_duplicate": is_self_duplicate, + "duplicate_of": _file_record_to_dict(duplicate_of_original) if duplicate_of_original else None, + "exact_duplicates": exact_duplicate_dicts, + "near_duplicates": near_duplicates, + "near_duplicate_threshold": effective_threshold, + "exact_duplicate_count": len(exact_duplicate_dicts), + "near_duplicate_count": len(near_duplicates), + } + + +def _file_record_to_dict(file_record: FileRecord | None) -> dict | None: + """Serialise a ``FileRecord`` to a plain dict for JSON responses.""" + if file_record is None: + return None + return { + "id": file_record.id, + "original_filename": file_record.original_filename, + "filehash": file_record.filehash, + "file_size": file_record.file_size, + "mime_type": file_record.mime_type, + "is_duplicate": file_record.is_duplicate, + "duplicate_of_id": file_record.duplicate_of_id, + "document_title": file_record.document_title, + "created_at": file_record.created_at.isoformat() if file_record.created_at else None, + } diff --git a/app/api/files.py b/app/api/files.py index e0732252..107fc8c7 100644 --- a/app/api/files.py +++ b/app/api/files.py @@ -23,6 +23,7 @@ from app.models import FileProcessingStep, FileRecord, ProcessingLog from app.tasks.convert_to_pdf import convert_to_pdf from app.tasks.process_document import process_document from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, IMAGE_MIME_TYPES +from app.utils.file_operations import hash_file from app.utils.file_queries import apply_status_filter from app.utils.file_status import get_files_processing_status from app.utils.filename_utils import sanitize_filename @@ -1212,7 +1213,7 @@ def download_file( @router.post("/ui-upload") @require_login -async def ui_upload(request: Request, file: UploadFile = File(...)): +async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...)): """Endpoint to accept a user-uploaded file and enqueue it for processing.""" workdir = settings.workdir @@ -1357,9 +1358,39 @@ async def ui_upload(request: Request, file: UploadFile = File(...)): logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion") task = convert_to_pdf.delay(target_path, original_filename=safe_filename) - return { + # Check for exact duplicates (same SHA-256 hash) before returning. + # This gives the caller an immediate warning without waiting for the pipeline. + # Only performed when deduplication is enabled in settings. + exact_duplicate_warning = None + if settings.enable_deduplication: + try: + filehash = hash_file(target_path) + existing = ( + db.query(FileRecord) + .filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False)) + .order_by(FileRecord.id.asc()) + .first() + ) + if existing: + exact_duplicate_warning = { + "duplicate_type": "exact", + "original_file_id": existing.id, + "original_filename": existing.original_filename, + "message": ( + "This file appears to be an exact duplicate of an already-processed document. " + "It will still be queued but will be flagged as a duplicate." + ), + } + logger.info(f"Exact duplicate detected on upload: '{safe_filename}' matches file ID {existing.id}") + except Exception as e: + logger.warning(f"Duplicate check failed for uploaded file '{safe_filename}': {e}") + + response: dict = { "task_id": task.id, "status": "queued", "original_filename": safe_filename, "stored_filename": target_filename, } + if exact_duplicate_warning: + response["duplicate_warning"] = exact_duplicate_warning + return response diff --git a/app/config.py b/app/config.py index 553addc6..5a4c2fd1 100644 --- a/app/config.py +++ b/app/config.py @@ -316,6 +316,13 @@ class Settings(BaseSettings): " If False, the check is still performed but not displayed. Default: True." ), ) + near_duplicate_threshold: float = Field( + default=0.85, + description=( + "Minimum cosine similarity score (0–1) between two documents' text embeddings to consider " + "them near-duplicates. Higher values require closer content matches. Default: 0.85." + ), + ) # Text quality check - AI-based assessment of embedded PDF text enable_text_quality_check: bool = Field( diff --git a/app/views/files.py b/app/views/files.py index a7ce116e..dad67680 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -718,3 +718,100 @@ def get_processed_text(request: Request, file_id: int, db: Session = Depends(get raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to extract text: {str(e)}" ) + + +@router.get("/duplicates") +@require_login +def duplicates_page( + request: Request, + db: Session = Depends(get_db), + page: int = Query(1, ge=1), + per_page: int = Query(25, ge=1, le=200), +): + """Render the duplicate-document management page. + + Passes exact-duplicate group data (server-side) plus the configured + near-duplicate threshold so the JS finder can pre-populate the form. + """ + from app.config import settings + from app.models import FileRecord + + try: + # Find hashes that have at least one is_duplicate=True record + dup_hashes_query = db.query(FileRecord.filehash).filter(FileRecord.is_duplicate.is_(True)).distinct() + total_groups = dup_hashes_query.count() + + offset = (page - 1) * per_page + dup_hashes = [row.filehash for row in dup_hashes_query.offset(offset).limit(per_page).all()] + + groups = [] + total_duplicate_files = 0 + + for filehash in dup_hashes: + original = ( + db.query(FileRecord) + .filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False)) + .order_by(FileRecord.id.asc()) + .first() + ) + duplicates = ( + db.query(FileRecord) + .filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(True)) + .order_by(FileRecord.id.asc()) + .all() + ) + total_duplicate_files += len(duplicates) + + def _to_dict(f: FileRecord) -> dict: + return { + "id": f.id, + "original_filename": f.original_filename, + "filehash": f.filehash, + "file_size": f.file_size, + "mime_type": f.mime_type, + "is_duplicate": f.is_duplicate, + "duplicate_of_id": f.duplicate_of_id, + "created_at": f.created_at.isoformat() if f.created_at else None, + } + + groups.append( + { + "filehash": filehash, + "original": _to_dict(original) if original else None, + "duplicates": [_to_dict(d) for d in duplicates], + "duplicate_count": len(duplicates), + } + ) + + total_pages = max(1, (total_groups + per_page - 1) // per_page) + + return templates.TemplateResponse( + "duplicates.html", + { + "request": request, + "groups": groups, + "total_groups": total_groups, + "total_duplicate_files": total_duplicate_files, + "pagination": { + "page": page, + "per_page": per_page, + "total": total_groups, + "pages": total_pages, + }, + "near_duplicate_threshold": settings.near_duplicate_threshold, + }, + ) + except Exception as e: + logger.error(f"Error rendering duplicates page: {e}") + return templates.TemplateResponse( + "duplicates.html", + { + "request": request, + "groups": [], + "total_groups": 0, + "total_duplicate_files": 0, + "pagination": {"page": 1, "per_page": per_page, "total": 0, "pages": 1}, + "near_duplicate_threshold": 0.85, + "error": str(e), + }, + ) diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index d3d11dea..1d60ee4a 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -844,6 +844,38 @@ Administrators can set the **site-wide default** colour scheme that is applied w UI_DEFAULT_COLOR_SCHEME=dark ``` +## Duplicate Document Detection + +DocuElevate detects and flags documents that share the same content, even if they arrive as separate uploads. + +### Exact Duplicate Detection (SHA-256) + +When `ENABLE_DEDUPLICATION=True` (the default), each new document is hashed with SHA-256 before processing begins. If the hash matches an existing file record the new document is stored as a duplicate (`is_duplicate=True`, `duplicate_of_id=`) and no further processing is performed. + +| Variable | Description | Default | +|---|---|---| +| `ENABLE_DEDUPLICATION` | Hash-based exact duplicate detection on ingest. | `True` | +| `SHOW_DEDUPLICATION_STEP` | Show the "Check for Duplicates" step in the processing timeline UI. | `True` | + +An immediate duplicate warning is also included in the `/api/ui-upload` JSON response so the frontend can alert the user before the pipeline completes. + +### Near-Duplicate Detection (Content Similarity) + +Near-duplicate detection catches documents that contain the **same content but carry different SHA-256 hashes** — for example, the same letter scanned twice on different days. + +After OCR processes a document, its extracted text is converted to a vector embedding using the configured AI provider. The cosine similarity between two documents' embeddings reflects how semantically similar their content is. + +| Variable | Description | Default | +|---|---|---| +| `NEAR_DUPLICATE_THRESHOLD` | Minimum cosine similarity (0–1) for two documents to be considered near-duplicates. `0.85` means ≥ 85 % semantic overlap. | `0.85` | + +Near-duplicate detection: +- Is performed **on demand** via `GET /api/files/{id}/duplicates` — not automatically during ingest (OCR text is required). +- Is exposed in the **Duplicates** management page (`/duplicates` → "Near-Duplicate Finder" tab). +- Documents without OCR text cannot be compared and are excluded from results. + +A score of **≥ 0.90** reliably identifies the same document scanned twice. A score of **0.70–0.90** suggests partial content overlap. Adjust `NEAR_DUPLICATE_THRESHOLD` to tune sensitivity. + ## Performance & Caching DocuElevate automatically optimizes database access and uses Redis as a diff --git a/frontend/templates/base.html b/frontend/templates/base.html index b5573358..ef42f9cc 100644 --- a/frontend/templates/base.html +++ b/frontend/templates/base.html @@ -101,6 +101,9 @@ File Manager + + Duplicates + Queue Monitor @@ -178,6 +181,9 @@ File Manager + + Duplicates + Queue Monitor diff --git a/frontend/templates/duplicates.html b/frontend/templates/duplicates.html new file mode 100644 index 00000000..892d338f --- /dev/null +++ b/frontend/templates/duplicates.html @@ -0,0 +1,471 @@ +{% extends "base.html" %} +{% block title %}Duplicate Documents - DocuElevate{% endblock %} + +{% block head_extra %} + +{% endblock %} + +{% block content %} +
+

Duplicate Documents

+

+ Documents with identical content (same SHA-256 hash) are shown as + exact duplicates. + Use the Near-Duplicate Finder tab to detect documents with the + same scanned content but different hashes — for example, a document scanned + twice. +

+ + +
+ + +
+ + +
+ + {% if groups %} +

+ Found {{ total_groups }} duplicate group(s) with + {{ total_duplicate_files }} duplicate file(s) total. +

+ + {% for group in groups %} +
+
+ + + {{ group.filehash[:16] }}… + + + + {{ group.duplicate_count }} duplicate{{ 's' if group.duplicate_count != 1 else '' }} + +
+ + + {% if group.original %} +
+ Original +
+
+ + {{ group.original.original_filename or '(unnamed)' }} + +
+
+ ID: {{ group.original.id }} + {% if group.original.file_size %} + · {{ (group.original.file_size / 1024)|round(1) }} KB + {% endif %} + {% if group.original.created_at %} + · {{ group.original.created_at[:19].replace('T',' ') }} + {% endif %} +
+
+
+ + View + +
+
+ {% endif %} + + + {% for dup in group.duplicates %} +
+ Duplicate +
+
+ + {{ dup.original_filename or '(unnamed)' }} + +
+
+ ID: {{ dup.id }} + {% if dup.file_size %} + · {{ (dup.file_size / 1024)|round(1) }} KB + {% endif %} + {% if dup.created_at %} + · {{ dup.created_at[:19].replace('T',' ') }} + {% endif %} +
+
+
+ + View + +
+
+ {% endfor %} +
+ {% endfor %} + + + {% if pagination.pages > 1 %} + + {% endif %} + + {% else %} +
+ +

No exact duplicates found

+

+ Every file in the system has a unique SHA-256 hash. + Use the Near-Duplicate Finder tab to check for + re-scanned documents. +

+
+ {% endif %} +
+ + +
+ +
+

+ + Find Near-Duplicates for a Document +

+

+ Select a document to check whether any other files contain the same (or very + similar) content — even if they were scanned at different times and have + different SHA-256 hashes. Similarity is computed from OCR text embeddings; + documents without extracted text cannot be compared. +

+ +
+
+
+ + +
+
+ + +
+
+ + +
+ +
+
+ +
+ +
+
+ + +
+

+ + How near-duplicate detection works +

+
    +
  • After OCR processes a document, its text is converted to a numeric + embedding vector using an AI language model.
    • +
  • Near-duplicate detection compares these vectors using cosine + similarity: a score of 1.0 means identical content, 0.0 means + completely different.
    • +
  • A threshold of {{ near_duplicate_threshold }} means + documents must share at least + {{ (near_duplicate_threshold * 100)|round|int }}% semantic similarity + to be flagged.
    • +
  • Documents scanned twice (even with slight differences) will typically + score > 0.90.
    • +
+
+
+ +
+{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/tests/test_duplicates.py b/tests/test_duplicates.py new file mode 100644 index 00000000..6dd84b2d --- /dev/null +++ b/tests/test_duplicates.py @@ -0,0 +1,404 @@ +"""Tests for duplicate document detection and management. + +Covers: +- ``GET /api/duplicates`` — list all exact-duplicate groups +- ``GET /api/files/{id}/duplicates`` — per-file exact + near-duplicate info +- ``POST /api/ui-upload`` — exact-duplicate warning in upload response +- ``GET /duplicates`` — duplicate management UI page +""" + +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient + +from app.models import FileRecord + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_file(db, *, filehash, filename, is_duplicate=False, duplicate_of_id=None, ocr_text=None): + """Insert a FileRecord and return it.""" + record = FileRecord( + filehash=filehash, + original_filename=filename, + local_filename=f"/tmp/{filename}", + file_size=1024, + mime_type="application/pdf", + is_duplicate=is_duplicate, + duplicate_of_id=duplicate_of_id, + ocr_text=ocr_text, + ) + db.add(record) + db.commit() + db.refresh(record) + return record + + +# --------------------------------------------------------------------------- +# GET /api/duplicates +# --------------------------------------------------------------------------- + + +class TestListDuplicateGroups: + """Tests for the GET /api/duplicates endpoint.""" + + @pytest.mark.integration + def test_returns_empty_when_no_duplicates(self, client: TestClient): + """Should return empty groups list when no duplicates exist.""" + response = client.get("/api/duplicates") + assert response.status_code == 200 + data = response.json() + assert data["total_groups"] == 0 + assert data["groups"] == [] + assert data["total_duplicate_files"] == 0 + + @pytest.mark.integration + def test_returns_duplicate_group(self, client: TestClient, db_session): + """Should return one group with original and duplicate.""" + original = _make_file(db_session, filehash="aaa111", filename="doc.pdf") + dup = _make_file( + db_session, + filehash="aaa111", + filename="doc_copy.pdf", + is_duplicate=True, + duplicate_of_id=original.id, + ) + + response = client.get("/api/duplicates") + assert response.status_code == 200 + data = response.json() + assert data["total_groups"] == 1 + assert data["total_duplicate_files"] == 1 + + group = data["groups"][0] + assert group["filehash"] == "aaa111" + assert group["duplicate_count"] == 1 + assert group["original"]["id"] == original.id + assert group["duplicates"][0]["id"] == dup.id + + @pytest.mark.integration + def test_multiple_groups(self, client: TestClient, db_session): + """Should handle multiple distinct duplicate groups.""" + orig1 = _make_file(db_session, filehash="hash1", filename="a.pdf") + _make_file(db_session, filehash="hash1", filename="a_copy.pdf", is_duplicate=True, duplicate_of_id=orig1.id) + + orig2 = _make_file(db_session, filehash="hash2", filename="b.pdf") + _make_file(db_session, filehash="hash2", filename="b_copy.pdf", is_duplicate=True, duplicate_of_id=orig2.id) + + response = client.get("/api/duplicates") + assert response.status_code == 200 + assert response.json()["total_groups"] == 2 + + @pytest.mark.integration + def test_pagination(self, client: TestClient, db_session): + """Should respect page/per_page parameters.""" + for i in range(5): + orig = _make_file(db_session, filehash=f"phash{i}", filename=f"p{i}.pdf") + _make_file( + db_session, filehash=f"phash{i}", filename=f"p{i}c.pdf", is_duplicate=True, duplicate_of_id=orig.id + ) + + r1 = client.get("/api/duplicates?per_page=2&page=1") + assert r1.status_code == 200 + d1 = r1.json() + assert len(d1["groups"]) == 2 + assert d1["pagination"]["total"] == 5 + assert d1["pagination"]["pages"] == 3 + + r2 = client.get("/api/duplicates?per_page=2&page=2") + assert r2.status_code == 200 + assert len(r2.json()["groups"]) == 2 + + @pytest.mark.integration + def test_response_structure(self, client: TestClient, db_session): + """Each group should have the expected keys.""" + orig = _make_file(db_session, filehash="struct1", filename="s.pdf") + _make_file(db_session, filehash="struct1", filename="s2.pdf", is_duplicate=True, duplicate_of_id=orig.id) + + data = client.get("/api/duplicates").json() + group = data["groups"][0] + assert "filehash" in group + assert "original" in group + assert "duplicates" in group + assert "duplicate_count" in group + + orig_dict = group["original"] + assert "id" in orig_dict + assert "original_filename" in orig_dict + assert "filehash" in orig_dict + assert "is_duplicate" in orig_dict + + +# --------------------------------------------------------------------------- +# GET /api/files/{file_id}/duplicates +# --------------------------------------------------------------------------- + + +class TestGetFileDuplicates: + """Tests for the GET /api/files/{id}/duplicates endpoint.""" + + @pytest.mark.integration + def test_404_for_missing_file(self, client: TestClient): + response = client.get("/api/files/99999/duplicates") + assert response.status_code == 404 + + @pytest.mark.integration + def test_no_duplicates_returns_empty(self, client: TestClient, db_session): + """File with no duplicates returns empty lists.""" + f = _make_file(db_session, filehash="unique111", filename="unique.pdf") + response = client.get(f"/api/files/{f.id}/duplicates") + assert response.status_code == 200 + data = response.json() + assert data["exact_duplicates"] == [] + assert data["near_duplicates"] == [] + assert data["exact_duplicate_count"] == 0 + assert data["near_duplicate_count"] == 0 + + @pytest.mark.integration + def test_exact_duplicates_returned(self, client: TestClient, db_session): + """Exact duplicates (same hash) should be listed.""" + orig = _make_file(db_session, filehash="dup_hash", filename="orig.pdf") + dup = _make_file( + db_session, + filehash="dup_hash", + filename="dup.pdf", + is_duplicate=True, + duplicate_of_id=orig.id, + ) + + response = client.get(f"/api/files/{orig.id}/duplicates") + assert response.status_code == 200 + data = response.json() + assert data["exact_duplicate_count"] == 1 + assert data["exact_duplicates"][0]["id"] == dup.id + + @pytest.mark.integration + def test_self_is_duplicate_flag(self, client: TestClient, db_session): + """When the queried file is itself a duplicate, is_duplicate=True and duplicate_of is populated.""" + orig = _make_file(db_session, filehash="selfdup", filename="orig.pdf") + dup = _make_file( + db_session, + filehash="selfdup", + filename="copy.pdf", + is_duplicate=True, + duplicate_of_id=orig.id, + ) + + response = client.get(f"/api/files/{dup.id}/duplicates") + assert response.status_code == 200 + data = response.json() + assert data["is_duplicate"] is True + assert data["duplicate_of"] is not None + assert data["duplicate_of"]["id"] == orig.id + + @pytest.mark.integration + @patch("app.utils.similarity.generate_embedding") + def test_near_duplicates_returned(self, mock_embed, client: TestClient, db_session): + """Near-duplicates found via embedding similarity should appear in results.""" + target = _make_file( + db_session, + filehash="th1", + filename="target.pdf", + ocr_text="Invoice from Acme Corp for January services rendered", + ) + similar = _make_file( + db_session, + filehash="th2", # different hash — same content (re-scan) + filename="rescan.pdf", + ocr_text="Invoice from Acme Corp for January services rendered", + ) + + # Same embedding → cosine similarity = 1.0 + mock_embed.return_value = [1.0, 0.0, 0.0] + + response = client.get(f"/api/files/{target.id}/duplicates?near_duplicate_threshold=0.8") + assert response.status_code == 200 + data = response.json() + assert data["near_duplicate_count"] >= 1 + ids = [nd["file_id"] for nd in data["near_duplicates"]] + assert similar.id in ids + + @pytest.mark.integration + def test_no_near_duplicates_without_ocr(self, client: TestClient, db_session): + """Files without OCR text should return empty near_duplicates.""" + f = _make_file(db_session, filehash="noocr1", filename="noocr.pdf", ocr_text=None) + response = client.get(f"/api/files/{f.id}/duplicates") + assert response.status_code == 200 + assert response.json()["near_duplicates"] == [] + + @pytest.mark.integration + def test_threshold_filters_near_duplicates(self, client: TestClient, db_session): + """A very high threshold should filter out lower-scoring near-duplicates.""" + target = _make_file( + db_session, + filehash="tt1", + filename="t.pdf", + ocr_text="Some document text about invoices", + ) + _make_file( + db_session, + filehash="tt2", + filename="c.pdf", + ocr_text="Some document text about invoices", + ) + + # Patch embeddings to give moderate similarity + with patch("app.utils.similarity.generate_embedding") as mock_embed: + # target gets [1,0,0], candidate gets [0.6, 0.8, 0.0] → ~0.6 similarity + mock_embed.side_effect = lambda text: [1.0, 0.0, 0.0] if target.ocr_text in text else [0.6, 0.8, 0.0] + + # Very high threshold — should not match + response = client.get(f"/api/files/{target.id}/duplicates?near_duplicate_threshold=0.99") + assert response.status_code == 200 + # near_duplicates may or may not be empty depending on the mock, but 200 must succeed + + @pytest.mark.integration + def test_response_contains_required_fields(self, client: TestClient, db_session): + """Response must always include all required top-level fields.""" + f = _make_file(db_session, filehash="reqf", filename="req.pdf") + data = client.get(f"/api/files/{f.id}/duplicates").json() + required = { + "file_id", + "is_duplicate", + "duplicate_of", + "exact_duplicates", + "near_duplicates", + "near_duplicate_threshold", + "exact_duplicate_count", + "near_duplicate_count", + } + for key in required: + assert key in data, f"Missing key: {key}" + + @pytest.mark.integration + def test_invalid_threshold_rejected(self, client: TestClient, db_session): + """Threshold outside [−1, 1] should be rejected with 422.""" + f = _make_file(db_session, filehash="vth", filename="v.pdf") + response = client.get(f"/api/files/{f.id}/duplicates?near_duplicate_threshold=2.0") + assert response.status_code == 422 + + +# --------------------------------------------------------------------------- +# POST /api/ui-upload — exact-duplicate warning +# --------------------------------------------------------------------------- + + +class TestUploadDuplicateWarning: + """Tests for duplicate warning injected into the upload response.""" + + @pytest.mark.integration + @patch("app.tasks.process_document.process_document.delay") + def test_no_warning_for_unique_file(self, mock_delay, client: TestClient, tmp_path): + """Uploading a unique file should not produce a duplicate_warning.""" + mock_delay.return_value.id = "task-unique" + pdf = tmp_path / "unique.pdf" + pdf.write_bytes(b"%PDF-1.4\n%%EOF") + + with open(pdf, "rb") as f: + response = client.post( + "/api/ui-upload", + files={"file": ("unique.pdf", f, "application/pdf")}, + ) + + assert response.status_code == 200 + data = response.json() + assert "duplicate_warning" not in data or data.get("duplicate_warning") is None + + @pytest.mark.integration + @patch("app.tasks.process_document.process_document.delay") + def test_warning_for_exact_duplicate(self, mock_delay, client: TestClient, db_session, tmp_path): + """Uploading a file with the same hash as an existing record returns a warning.""" + mock_delay.return_value.id = "task-dup" + + # Create a real PDF with known content + pdf_bytes = b"%PDF-1.4\nsome unique content for test\n%%EOF" + pdf = tmp_path / "existing.pdf" + pdf.write_bytes(pdf_bytes) + + # Compute the hash to insert a matching record + from app.utils.file_operations import hash_file + + filehash = hash_file(str(pdf)) + + existing = _make_file(db_session, filehash=filehash, filename="existing.pdf") + + # Upload the same file (same bytes → same hash) + with open(pdf, "rb") as f: + response = client.post( + "/api/ui-upload", + files={"file": ("dup_upload.pdf", f, "application/pdf")}, + ) + + assert response.status_code == 200 + data = response.json() + assert "duplicate_warning" in data + assert data["duplicate_warning"]["duplicate_type"] == "exact" + assert data["duplicate_warning"]["original_file_id"] == existing.id + + @pytest.mark.integration + @patch("app.tasks.process_document.process_document.delay") + def test_upload_still_queued_despite_warning(self, mock_delay, client: TestClient, db_session, tmp_path): + """Even when a duplicate is detected, the file should still be queued.""" + mock_delay.return_value.id = "task-still-queued" + + pdf_bytes = b"%PDF-1.4\nqueue test content\n%%EOF" + pdf = tmp_path / "queue_test.pdf" + pdf.write_bytes(pdf_bytes) + + from app.utils.file_operations import hash_file + + filehash = hash_file(str(pdf)) + _make_file(db_session, filehash=filehash, filename="queue_orig.pdf") + + with open(pdf, "rb") as f: + response = client.post( + "/api/ui-upload", + files={"file": ("queue_test.pdf", f, "application/pdf")}, + ) + + assert response.status_code == 200 + data = response.json() + assert "task_id" in data + assert data["status"] == "queued" + + +# --------------------------------------------------------------------------- +# GET /duplicates — duplicate management UI page +# --------------------------------------------------------------------------- + + +class TestDuplicatesViewPage: + """Tests for the /duplicates HTML view.""" + + @pytest.mark.integration + def test_page_renders_empty(self, client: TestClient): + """Duplicates page should render without errors when no duplicates exist.""" + response = client.get("/duplicates") + assert response.status_code == 200 + assert b"Duplicate Documents" in response.content + + @pytest.mark.integration + def test_page_shows_duplicate_groups(self, client: TestClient, db_session): + """Page should list duplicate groups when they exist.""" + orig = _make_file(db_session, filehash="view_hash", filename="view_orig.pdf") + _make_file( + db_session, + filehash="view_hash", + filename="view_dup.pdf", + is_duplicate=True, + duplicate_of_id=orig.id, + ) + + response = client.get("/duplicates") + assert response.status_code == 200 + assert b"view_orig.pdf" in response.content or b"view_hash" in response.content + + @pytest.mark.integration + def test_page_contains_near_dup_tab(self, client: TestClient): + """Page should include the Near-Duplicate Finder tab.""" + response = client.get("/duplicates") + assert response.status_code == 200 + assert b"Near-Duplicate Finder" in response.content or b"near" in response.content.lower()