feat(duplicates): add duplicate document detection and management
- Add near_duplicate_threshold config setting (default 0.85)
- New GET /api/duplicates endpoint listing all exact-duplicate groups
- New GET /api/files/{id}/duplicates endpoint returning exact + near-duplicates
- POST /api/ui-upload now returns immediate exact-duplicate warning (respects ENABLE_DEDUPLICATION)
- New /duplicates management UI with Exact Duplicates tab and Near-Duplicate Finder tab
- Add Duplicates link in admin nav menu (desktop + mobile)
- Document new config options in ConfigurationGuide.md and .env.demo
- 20 new tests covering all acceptance criteria
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -336,4 +336,14 @@ MEILISEARCH_URL=http://meilisearch:7700
|
||||
# Optional master/API key for secured Meilisearch instances
|
||||
# MEILISEARCH_API_KEY=your_master_key_here
|
||||
MEILISEARCH_INDEX_NAME=documents
|
||||
ENABLE_SEARCH=True
|
||||
ENABLE_SEARCH=True
|
||||
|
||||
# **Duplicate Detection**
|
||||
# Exact duplicate detection (SHA-256) is always on during document processing.
|
||||
# The settings below control near-duplicate detection (same scanned content,
|
||||
# different hash) and the visibility of deduplication steps.
|
||||
ENABLE_DEDUPLICATION=True
|
||||
SHOW_DEDUPLICATION_STEP=True
|
||||
# Minimum cosine similarity score (0–1) for two documents to be flagged as
|
||||
# near-duplicates. 0.85 means 85 % semantic overlap. Lower = more matches.
|
||||
NEAR_DUPLICATE_THRESHOLD=0.85
|
||||
|
||||
@@ -9,6 +9,7 @@ from fastapi import APIRouter
|
||||
from app.api.azure import router as azure_router
|
||||
from app.api.diagnostic import router as diagnostic_router
|
||||
from app.api.dropbox import router as dropbox_router
|
||||
from app.api.duplicates import router as duplicates_router
|
||||
from app.api.files import router as files_router
|
||||
from app.api.google_drive import router as google_drive_router
|
||||
from app.api.logs import router as logs_router
|
||||
@@ -49,4 +50,5 @@ router.include_router(search_router)
|
||||
router.include_router(queue_router)
|
||||
router.include_router(saved_searches_router)
|
||||
router.include_router(similarity_router)
|
||||
router.include_router(duplicates_router)
|
||||
router.include_router(webhooks_router)
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
"""Duplicate document detection and management API endpoints.
|
||||
|
||||
Provides endpoints for listing all duplicate groups (exact SHA-256 duplicates) and
|
||||
for retrieving both exact and near-duplicate matches for a specific document.
|
||||
|
||||
Near-duplicate detection is powered by the same text-embedding cosine-similarity
|
||||
engine used by the ``/api/files/{id}/similar`` endpoint
|
||||
(see ``app/utils/similarity.py``).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import FileRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
@router.get("/duplicates")
|
||||
@require_login
|
||||
def list_duplicate_groups(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
page: int = Query(1, ge=1, description="Page number"),
|
||||
per_page: int = Query(25, ge=1, le=200, description="Items per page"),
|
||||
):
|
||||
"""List all groups of exact-duplicate documents (same SHA-256 hash).
|
||||
|
||||
Returns one entry per duplicate group showing the original document and all
|
||||
files that were detected as copies of it. Groups are sorted by descending
|
||||
duplicate count.
|
||||
|
||||
Example:
|
||||
```
|
||||
GET /api/duplicates
|
||||
```
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"groups": [
|
||||
{
|
||||
"filehash": "abc123...",
|
||||
"original": {"id": 1, "original_filename": "invoice.pdf", ...},
|
||||
"duplicates": [{"id": 5, "original_filename": "invoice_copy.pdf", ...}],
|
||||
"duplicate_count": 1
|
||||
}
|
||||
],
|
||||
"total_groups": 1,
|
||||
"total_duplicate_files": 1,
|
||||
"pagination": {...}
|
||||
}
|
||||
```
|
||||
"""
|
||||
# Find all hashes that have at least one duplicate record
|
||||
dup_hashes_query = db.query(FileRecord.filehash).filter(FileRecord.is_duplicate.is_(True)).distinct()
|
||||
total_groups = dup_hashes_query.count()
|
||||
|
||||
# Paginate hash groups
|
||||
offset = (page - 1) * per_page
|
||||
dup_hashes = [row.filehash for row in dup_hashes_query.offset(offset).limit(per_page).all()]
|
||||
|
||||
groups = []
|
||||
total_duplicate_files = 0
|
||||
|
||||
for filehash in dup_hashes:
|
||||
# Find the original (non-duplicate) record with this hash
|
||||
original = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
|
||||
.order_by(FileRecord.id.asc())
|
||||
.first()
|
||||
)
|
||||
|
||||
# Find all duplicate records for this hash
|
||||
duplicates = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(True))
|
||||
.order_by(FileRecord.id.asc())
|
||||
.all()
|
||||
)
|
||||
|
||||
total_duplicate_files += len(duplicates)
|
||||
|
||||
groups.append(
|
||||
{
|
||||
"filehash": filehash,
|
||||
"original": _file_record_to_dict(original) if original else None,
|
||||
"duplicates": [_file_record_to_dict(d) for d in duplicates],
|
||||
"duplicate_count": len(duplicates),
|
||||
}
|
||||
)
|
||||
|
||||
total_pages = (total_groups + per_page - 1) // per_page if total_groups > 0 else 1
|
||||
|
||||
return {
|
||||
"groups": groups,
|
||||
"total_groups": total_groups,
|
||||
"total_duplicate_files": total_duplicate_files,
|
||||
"pagination": {
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"total": total_groups,
|
||||
"pages": total_pages,
|
||||
"next": str(request.url.include_query_params(page=page + 1)) if page < total_pages else None,
|
||||
"previous": str(request.url.include_query_params(page=page - 1)) if page > 1 else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/files/{file_id}/duplicates")
|
||||
@require_login
|
||||
def get_file_duplicates(
|
||||
request: Request,
|
||||
file_id: int,
|
||||
db: DbSession,
|
||||
near_duplicate_limit: int = Query(5, ge=1, le=20, description="Maximum near-duplicates to return"),
|
||||
near_duplicate_threshold: float = Query(
|
||||
-1.0,
|
||||
ge=-1.0,
|
||||
le=1.0,
|
||||
description="Minimum similarity score for near-duplicates; -1 uses the configured default",
|
||||
),
|
||||
):
|
||||
"""Get exact and near-duplicate documents for the specified file.
|
||||
|
||||
**Exact duplicates** share the same SHA-256 hash.
|
||||
**Near-duplicates** have a text-embedding cosine similarity score ≥
|
||||
``NEAR_DUPLICATE_THRESHOLD`` (configurable; default 0.85).
|
||||
|
||||
Near-duplicate detection requires OCR text to be available for both the
|
||||
target file and candidate files. Files without OCR text are excluded.
|
||||
|
||||
Example:
|
||||
```
|
||||
GET /api/files/42/duplicates
|
||||
```
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"file_id": 42,
|
||||
"exact_duplicates": [
|
||||
{"id": 7, "original_filename": "invoice.pdf", "is_duplicate": true, "duplicate_of_id": 42, ...}
|
||||
],
|
||||
"near_duplicates": [
|
||||
{"file_id": 15, "original_filename": "invoice_jan.pdf", "similarity_score": 0.92, ...}
|
||||
],
|
||||
"near_duplicate_threshold": 0.85
|
||||
}
|
||||
```
|
||||
"""
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
|
||||
|
||||
# --- Exact duplicates ---
|
||||
# Case 1: This file is the original — find all records that are duplicates of it
|
||||
exact_duplicates_of_this = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == file_record.filehash, FileRecord.id != file_id)
|
||||
.order_by(FileRecord.id.asc())
|
||||
.all()
|
||||
)
|
||||
|
||||
# Case 2: This file itself is a duplicate — find the original
|
||||
is_self_duplicate = file_record.is_duplicate
|
||||
duplicate_of_original: FileRecord | None = None
|
||||
if is_self_duplicate and file_record.duplicate_of_id:
|
||||
duplicate_of_original = db.query(FileRecord).filter(FileRecord.id == file_record.duplicate_of_id).first()
|
||||
|
||||
exact_duplicate_dicts = [_file_record_to_dict(f) for f in exact_duplicates_of_this]
|
||||
|
||||
# --- Near-duplicates (embedding-based) ---
|
||||
effective_threshold = (
|
||||
near_duplicate_threshold if near_duplicate_threshold >= 0.0 else settings.near_duplicate_threshold
|
||||
)
|
||||
|
||||
near_duplicates: list[dict] = []
|
||||
if file_record.ocr_text and file_record.ocr_text.strip():
|
||||
try:
|
||||
from app.utils.similarity import find_similar_documents
|
||||
|
||||
near_duplicates = find_similar_documents(
|
||||
db,
|
||||
file_id,
|
||||
limit=near_duplicate_limit,
|
||||
threshold=effective_threshold,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Near-duplicate detection failed for file {file_id}: {e}")
|
||||
near_duplicates = []
|
||||
|
||||
return {
|
||||
"file_id": file_id,
|
||||
"is_duplicate": is_self_duplicate,
|
||||
"duplicate_of": _file_record_to_dict(duplicate_of_original) if duplicate_of_original else None,
|
||||
"exact_duplicates": exact_duplicate_dicts,
|
||||
"near_duplicates": near_duplicates,
|
||||
"near_duplicate_threshold": effective_threshold,
|
||||
"exact_duplicate_count": len(exact_duplicate_dicts),
|
||||
"near_duplicate_count": len(near_duplicates),
|
||||
}
|
||||
|
||||
|
||||
def _file_record_to_dict(file_record: FileRecord | None) -> dict | None:
|
||||
"""Serialise a ``FileRecord`` to a plain dict for JSON responses."""
|
||||
if file_record is None:
|
||||
return None
|
||||
return {
|
||||
"id": file_record.id,
|
||||
"original_filename": file_record.original_filename,
|
||||
"filehash": file_record.filehash,
|
||||
"file_size": file_record.file_size,
|
||||
"mime_type": file_record.mime_type,
|
||||
"is_duplicate": file_record.is_duplicate,
|
||||
"duplicate_of_id": file_record.duplicate_of_id,
|
||||
"document_title": file_record.document_title,
|
||||
"created_at": file_record.created_at.isoformat() if file_record.created_at else None,
|
||||
}
|
||||
+33
-2
@@ -23,6 +23,7 @@ from app.models import FileProcessingStep, FileRecord, ProcessingLog
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf
|
||||
from app.tasks.process_document import process_document
|
||||
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES, IMAGE_MIME_TYPES
|
||||
from app.utils.file_operations import hash_file
|
||||
from app.utils.file_queries import apply_status_filter
|
||||
from app.utils.file_status import get_files_processing_status
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
@@ -1212,7 +1213,7 @@ def download_file(
|
||||
|
||||
@router.post("/ui-upload")
|
||||
@require_login
|
||||
async def ui_upload(request: Request, file: UploadFile = File(...)):
|
||||
async def ui_upload(request: Request, db: DbSession, file: UploadFile = File(...)):
|
||||
"""Endpoint to accept a user-uploaded file and enqueue it for processing."""
|
||||
workdir = settings.workdir
|
||||
|
||||
@@ -1357,9 +1358,39 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
|
||||
logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
|
||||
task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
|
||||
|
||||
return {
|
||||
# Check for exact duplicates (same SHA-256 hash) before returning.
|
||||
# This gives the caller an immediate warning without waiting for the pipeline.
|
||||
# Only performed when deduplication is enabled in settings.
|
||||
exact_duplicate_warning = None
|
||||
if settings.enable_deduplication:
|
||||
try:
|
||||
filehash = hash_file(target_path)
|
||||
existing = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
|
||||
.order_by(FileRecord.id.asc())
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
exact_duplicate_warning = {
|
||||
"duplicate_type": "exact",
|
||||
"original_file_id": existing.id,
|
||||
"original_filename": existing.original_filename,
|
||||
"message": (
|
||||
"This file appears to be an exact duplicate of an already-processed document. "
|
||||
"It will still be queued but will be flagged as a duplicate."
|
||||
),
|
||||
}
|
||||
logger.info(f"Exact duplicate detected on upload: '{safe_filename}' matches file ID {existing.id}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Duplicate check failed for uploaded file '{safe_filename}': {e}")
|
||||
|
||||
response: dict = {
|
||||
"task_id": task.id,
|
||||
"status": "queued",
|
||||
"original_filename": safe_filename,
|
||||
"stored_filename": target_filename,
|
||||
}
|
||||
if exact_duplicate_warning:
|
||||
response["duplicate_warning"] = exact_duplicate_warning
|
||||
return response
|
||||
|
||||
@@ -316,6 +316,13 @@ class Settings(BaseSettings):
|
||||
" If False, the check is still performed but not displayed. Default: True."
|
||||
),
|
||||
)
|
||||
near_duplicate_threshold: float = Field(
|
||||
default=0.85,
|
||||
description=(
|
||||
"Minimum cosine similarity score (0–1) between two documents' text embeddings to consider "
|
||||
"them near-duplicates. Higher values require closer content matches. Default: 0.85."
|
||||
),
|
||||
)
|
||||
|
||||
# Text quality check - AI-based assessment of embedded PDF text
|
||||
enable_text_quality_check: bool = Field(
|
||||
|
||||
@@ -718,3 +718,100 @@ def get_processed_text(request: Request, file_id: int, db: Session = Depends(get
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to extract text: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/duplicates")
|
||||
@require_login
|
||||
def duplicates_page(
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
page: int = Query(1, ge=1),
|
||||
per_page: int = Query(25, ge=1, le=200),
|
||||
):
|
||||
"""Render the duplicate-document management page.
|
||||
|
||||
Passes exact-duplicate group data (server-side) plus the configured
|
||||
near-duplicate threshold so the JS finder can pre-populate the form.
|
||||
"""
|
||||
from app.config import settings
|
||||
from app.models import FileRecord
|
||||
|
||||
try:
|
||||
# Find hashes that have at least one is_duplicate=True record
|
||||
dup_hashes_query = db.query(FileRecord.filehash).filter(FileRecord.is_duplicate.is_(True)).distinct()
|
||||
total_groups = dup_hashes_query.count()
|
||||
|
||||
offset = (page - 1) * per_page
|
||||
dup_hashes = [row.filehash for row in dup_hashes_query.offset(offset).limit(per_page).all()]
|
||||
|
||||
groups = []
|
||||
total_duplicate_files = 0
|
||||
|
||||
for filehash in dup_hashes:
|
||||
original = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
|
||||
.order_by(FileRecord.id.asc())
|
||||
.first()
|
||||
)
|
||||
duplicates = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(True))
|
||||
.order_by(FileRecord.id.asc())
|
||||
.all()
|
||||
)
|
||||
total_duplicate_files += len(duplicates)
|
||||
|
||||
def _to_dict(f: FileRecord) -> dict:
|
||||
return {
|
||||
"id": f.id,
|
||||
"original_filename": f.original_filename,
|
||||
"filehash": f.filehash,
|
||||
"file_size": f.file_size,
|
||||
"mime_type": f.mime_type,
|
||||
"is_duplicate": f.is_duplicate,
|
||||
"duplicate_of_id": f.duplicate_of_id,
|
||||
"created_at": f.created_at.isoformat() if f.created_at else None,
|
||||
}
|
||||
|
||||
groups.append(
|
||||
{
|
||||
"filehash": filehash,
|
||||
"original": _to_dict(original) if original else None,
|
||||
"duplicates": [_to_dict(d) for d in duplicates],
|
||||
"duplicate_count": len(duplicates),
|
||||
}
|
||||
)
|
||||
|
||||
total_pages = max(1, (total_groups + per_page - 1) // per_page)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"duplicates.html",
|
||||
{
|
||||
"request": request,
|
||||
"groups": groups,
|
||||
"total_groups": total_groups,
|
||||
"total_duplicate_files": total_duplicate_files,
|
||||
"pagination": {
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"total": total_groups,
|
||||
"pages": total_pages,
|
||||
},
|
||||
"near_duplicate_threshold": settings.near_duplicate_threshold,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error rendering duplicates page: {e}")
|
||||
return templates.TemplateResponse(
|
||||
"duplicates.html",
|
||||
{
|
||||
"request": request,
|
||||
"groups": [],
|
||||
"total_groups": 0,
|
||||
"total_duplicate_files": 0,
|
||||
"pagination": {"page": 1, "per_page": per_page, "total": 0, "pages": 1},
|
||||
"near_duplicate_threshold": 0.85,
|
||||
"error": str(e),
|
||||
},
|
||||
)
|
||||
|
||||
@@ -844,6 +844,38 @@ Administrators can set the **site-wide default** colour scheme that is applied w
|
||||
UI_DEFAULT_COLOR_SCHEME=dark
|
||||
```
|
||||
|
||||
## Duplicate Document Detection
|
||||
|
||||
DocuElevate detects and flags documents that share the same content, even if they arrive as separate uploads.
|
||||
|
||||
### Exact Duplicate Detection (SHA-256)
|
||||
|
||||
When `ENABLE_DEDUPLICATION=True` (the default), each new document is hashed with SHA-256 before processing begins. If the hash matches an existing file record the new document is stored as a duplicate (`is_duplicate=True`, `duplicate_of_id=<original_id>`) and no further processing is performed.
|
||||
|
||||
| Variable | Description | Default |
|
||||
|---|---|---|
|
||||
| `ENABLE_DEDUPLICATION` | Hash-based exact duplicate detection on ingest. | `True` |
|
||||
| `SHOW_DEDUPLICATION_STEP` | Show the "Check for Duplicates" step in the processing timeline UI. | `True` |
|
||||
|
||||
An immediate duplicate warning is also included in the `/api/ui-upload` JSON response so the frontend can alert the user before the pipeline completes.
|
||||
|
||||
### Near-Duplicate Detection (Content Similarity)
|
||||
|
||||
Near-duplicate detection catches documents that contain the **same content but carry different SHA-256 hashes** — for example, the same letter scanned twice on different days.
|
||||
|
||||
After OCR processes a document, its extracted text is converted to a vector embedding using the configured AI provider. The cosine similarity between two documents' embeddings reflects how semantically similar their content is.
|
||||
|
||||
| Variable | Description | Default |
|
||||
|---|---|---|
|
||||
| `NEAR_DUPLICATE_THRESHOLD` | Minimum cosine similarity (0–1) for two documents to be considered near-duplicates. `0.85` means ≥ 85 % semantic overlap. | `0.85` |
|
||||
|
||||
Near-duplicate detection:
|
||||
- Is performed **on demand** via `GET /api/files/{id}/duplicates` — not automatically during ingest (OCR text is required).
|
||||
- Is exposed in the **Duplicates** management page (`/duplicates` → "Near-Duplicate Finder" tab).
|
||||
- Documents without OCR text cannot be compared and are excluded from results.
|
||||
|
||||
A score of **≥ 0.90** reliably identifies the same document scanned twice. A score of **0.70–0.90** suggests partial content overlap. Adjust `NEAR_DUPLICATE_THRESHOLD` to tune sensitivity.
|
||||
|
||||
## Performance & Caching
|
||||
|
||||
DocuElevate automatically optimizes database access and uses Redis as a
|
||||
|
||||
@@ -101,6 +101,9 @@
|
||||
<a href="/admin/files" role="menuitem" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
|
||||
<i class="fas fa-folder-open w-4 mr-2 text-gray-500" aria-hidden="true"></i> File Manager
|
||||
</a>
|
||||
<a href="/duplicates" role="menuitem" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
|
||||
<i class="fas fa-clone w-4 mr-2 text-orange-500" aria-hidden="true"></i> Duplicates
|
||||
</a>
|
||||
<a href="/admin/queue" role="menuitem" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
|
||||
<i class="fas fa-stream w-4 mr-2 text-blue-500" aria-hidden="true"></i> Queue Monitor
|
||||
</a>
|
||||
@@ -178,6 +181,9 @@
|
||||
<a href="/admin/files" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50">
|
||||
<i class="fas fa-folder-open mr-2 text-gray-400" aria-hidden="true"></i> File Manager
|
||||
</a>
|
||||
<a href="/duplicates" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50">
|
||||
<i class="fas fa-clone mr-2 text-orange-400" aria-hidden="true"></i> Duplicates
|
||||
</a>
|
||||
<a href="/admin/queue" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50">
|
||||
<i class="fas fa-stream mr-2 text-blue-400" aria-hidden="true"></i> Queue Monitor
|
||||
</a>
|
||||
|
||||
@@ -0,0 +1,471 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Duplicate Documents - DocuElevate{% endblock %}
|
||||
|
||||
{% block head_extra %}
|
||||
<style>
|
||||
.dup-container { max-width: 1100px; margin: 0 auto; }
|
||||
.dup-group {
|
||||
background: white; border-radius: 0.5rem;
|
||||
box-shadow: 0 1px 3px rgba(0,0,0,0.1);
|
||||
margin-bottom: 1.5rem; overflow: hidden;
|
||||
}
|
||||
.dup-group-header {
|
||||
padding: 1rem 1.25rem;
|
||||
border-bottom: 1px solid #e5e7eb;
|
||||
display: flex; align-items: center; gap: 0.75rem;
|
||||
background: #f9fafb;
|
||||
}
|
||||
.dup-badge {
|
||||
display: inline-flex; align-items: center; gap: 0.3rem;
|
||||
font-size: 0.75rem; font-weight: 700; padding: 0.2rem 0.6rem;
|
||||
border-radius: 9999px;
|
||||
}
|
||||
.dup-badge-exact { background: #fee2e2; color: #991b1b; }
|
||||
.dup-badge-near { background: #fef3c7; color: #92400e; }
|
||||
.dup-file-row {
|
||||
display: flex; align-items: center; gap: 0.75rem;
|
||||
padding: 0.75rem 1.25rem; border-bottom: 1px solid #f3f4f6;
|
||||
}
|
||||
.dup-file-row:last-child { border-bottom: none; }
|
||||
.dup-file-row.original { background: #f0fdf4; }
|
||||
.dup-file-row.duplicate { background: #fff7ed; }
|
||||
.dup-file-row.near-dup { background: #fffbeb; }
|
||||
.dup-role-tag {
|
||||
font-size: 0.7rem; font-weight: 700; padding: 0.15rem 0.5rem;
|
||||
border-radius: 9999px; white-space: nowrap;
|
||||
}
|
||||
.tag-original { background: #d1fae5; color: #065f46; }
|
||||
.tag-duplicate { background: #fee2e2; color: #991b1b; }
|
||||
.tag-near { background: #fef3c7; color: #92400e; }
|
||||
.dup-filename { font-weight: 600; color: #1f2937; word-break: break-all; }
|
||||
.dup-meta { font-size: 0.8rem; color: #6b7280; }
|
||||
.dup-score { font-size: 0.8rem; font-weight: 700; color: #d97706; }
|
||||
.dup-actions { margin-left: auto; display: flex; gap: 0.5rem; flex-shrink: 0; }
|
||||
|
||||
/* near-dup search UI */
|
||||
.near-dup-search {
|
||||
background: white; border-radius: 0.5rem;
|
||||
box-shadow: 0 1px 3px rgba(0,0,0,0.1);
|
||||
padding: 1.25rem; margin-bottom: 2rem;
|
||||
}
|
||||
.near-dup-search h3 { font-size: 1rem; font-weight: 700; margin-bottom: 0.75rem; }
|
||||
.nd-form { display: flex; gap: 0.5rem; flex-wrap: wrap; align-items: flex-end; }
|
||||
.nd-form label { font-size: 0.8rem; font-weight: 600; color: #374151; }
|
||||
.nd-form input, .nd-form select {
|
||||
padding: 0.4rem 0.6rem; border: 1px solid #d1d5db;
|
||||
border-radius: 0.375rem; font-size: 0.85rem;
|
||||
}
|
||||
.nd-form input[type=number] { width: 90px; }
|
||||
.nd-results { margin-top: 1rem; }
|
||||
|
||||
.empty-state {
|
||||
text-align: center; padding: 3rem; color: #6b7280;
|
||||
}
|
||||
.empty-state i { font-size: 3rem; margin-bottom: 0.75rem; display: block; }
|
||||
.tabs { display: flex; gap: 0; border-bottom: 2px solid #e5e7eb; margin-bottom: 1.5rem; }
|
||||
.tab-btn {
|
||||
padding: 0.6rem 1.25rem; font-weight: 600; font-size: 0.9rem;
|
||||
cursor: pointer; border: none; background: none;
|
||||
border-bottom: 3px solid transparent; margin-bottom: -2px;
|
||||
color: #6b7280; transition: color .15s;
|
||||
}
|
||||
.tab-btn.active { color: #2563eb; border-bottom-color: #2563eb; }
|
||||
.tab-panel { display: none; }
|
||||
.tab-panel.active { display: block; }
|
||||
|
||||
/* pagination */
|
||||
.pagination { display: flex; gap: 0.5rem; justify-content: center; margin-top: 1.5rem; }
|
||||
.page-btn {
|
||||
padding: 0.4rem 0.75rem; border: 1px solid #d1d5db;
|
||||
border-radius: 0.375rem; font-size: 0.85rem; cursor: pointer;
|
||||
background: white; color: #374151;
|
||||
}
|
||||
.page-btn:hover:not(:disabled) { background: #f3f4f6; }
|
||||
.page-btn:disabled { opacity: 0.4; cursor: default; }
|
||||
.page-btn.current { background: #2563eb; color: white; border-color: #2563eb; }
|
||||
</style>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<main id="main-content" class="dup-container px-4 py-8">
|
||||
<h1 class="text-2xl font-bold mb-2">Duplicate Documents</h1>
|
||||
<p class="text-gray-500 mb-6 text-sm">
|
||||
Documents with identical content (same SHA-256 hash) are shown as
|
||||
<span class="dup-badge dup-badge-exact">exact duplicates</span>.
|
||||
Use the <strong>Near-Duplicate Finder</strong> tab to detect documents with the
|
||||
same scanned content but different hashes — for example, a document scanned
|
||||
twice.
|
||||
</p>
|
||||
|
||||
<!-- Tabs -->
|
||||
<div class="tabs" role="tablist" aria-label="Duplicate detection tabs">
|
||||
<button class="tab-btn active" role="tab" aria-selected="true"
|
||||
aria-controls="tab-exact" id="btn-exact"
|
||||
onclick="switchTab('exact')">
|
||||
<i class="fas fa-clone mr-1" aria-hidden="true"></i>
|
||||
Exact Duplicates
|
||||
{% if total_groups is defined %}
|
||||
<span class="ml-1 text-xs bg-red-100 text-red-700 font-bold px-1.5 rounded-full">
|
||||
{{ total_groups }}
|
||||
</span>
|
||||
{% endif %}
|
||||
</button>
|
||||
<button class="tab-btn" role="tab" aria-selected="false"
|
||||
aria-controls="tab-near" id="btn-near"
|
||||
onclick="switchTab('near')">
|
||||
<i class="fas fa-layer-group mr-1" aria-hidden="true"></i>
|
||||
Near-Duplicate Finder
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- ── Tab: Exact Duplicates ── -->
|
||||
<div id="tab-exact" class="tab-panel active" role="tabpanel" aria-labelledby="btn-exact">
|
||||
|
||||
{% if groups %}
|
||||
<p class="text-sm text-gray-500 mb-4">
|
||||
Found <strong>{{ total_groups }}</strong> duplicate group(s) with
|
||||
<strong>{{ total_duplicate_files }}</strong> duplicate file(s) total.
|
||||
</p>
|
||||
|
||||
{% for group in groups %}
|
||||
<section class="dup-group" aria-label="Duplicate group {{ loop.index }}">
|
||||
<div class="dup-group-header">
|
||||
<i class="fas fa-fingerprint text-gray-400" aria-hidden="true"></i>
|
||||
<span class="text-xs font-mono text-gray-500" title="SHA-256 hash">
|
||||
{{ group.filehash[:16] }}…
|
||||
</span>
|
||||
<span class="dup-badge dup-badge-exact">
|
||||
<i class="fas fa-copy" aria-hidden="true"></i>
|
||||
{{ group.duplicate_count }} duplicate{{ 's' if group.duplicate_count != 1 else '' }}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<!-- Original file -->
|
||||
{% if group.original %}
|
||||
<div class="dup-file-row original">
|
||||
<span class="dup-role-tag tag-original">Original</span>
|
||||
<div class="flex-1 min-w-0">
|
||||
<div class="dup-filename">
|
||||
<a href="/files/{{ group.original.id }}"
|
||||
class="hover:text-blue-600">
|
||||
{{ group.original.original_filename or '(unnamed)' }}
|
||||
</a>
|
||||
</div>
|
||||
<div class="dup-meta">
|
||||
ID: {{ group.original.id }}
|
||||
{% if group.original.file_size %}
|
||||
· {{ (group.original.file_size / 1024)|round(1) }} KB
|
||||
{% endif %}
|
||||
{% if group.original.created_at %}
|
||||
· {{ group.original.created_at[:19].replace('T',' ') }}
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="dup-actions">
|
||||
<a href="/files/{{ group.original.id }}"
|
||||
class="text-sm text-blue-600 hover:underline"
|
||||
aria-label="View original file {{ group.original.original_filename }}">
|
||||
<i class="fas fa-external-link-alt" aria-hidden="true"></i> View
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- Duplicate files -->
|
||||
{% for dup in group.duplicates %}
|
||||
<div class="dup-file-row duplicate">
|
||||
<span class="dup-role-tag tag-duplicate">Duplicate</span>
|
||||
<div class="flex-1 min-w-0">
|
||||
<div class="dup-filename">
|
||||
<a href="/files/{{ dup.id }}" class="hover:text-blue-600">
|
||||
{{ dup.original_filename or '(unnamed)' }}
|
||||
</a>
|
||||
</div>
|
||||
<div class="dup-meta">
|
||||
ID: {{ dup.id }}
|
||||
{% if dup.file_size %}
|
||||
· {{ (dup.file_size / 1024)|round(1) }} KB
|
||||
{% endif %}
|
||||
{% if dup.created_at %}
|
||||
· {{ dup.created_at[:19].replace('T',' ') }}
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="dup-actions">
|
||||
<a href="/files/{{ dup.id }}"
|
||||
class="text-sm text-blue-600 hover:underline"
|
||||
aria-label="View duplicate file {{ dup.original_filename }}">
|
||||
<i class="fas fa-external-link-alt" aria-hidden="true"></i> View
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</section>
|
||||
{% endfor %}
|
||||
|
||||
<!-- Pagination -->
|
||||
{% if pagination.pages > 1 %}
|
||||
<nav class="pagination" aria-label="Pagination">
|
||||
<button class="page-btn" onclick="changePage({{ pagination.page - 1 }})"
|
||||
{% if pagination.page <= 1 %}disabled{% endif %}
|
||||
aria-label="Previous page">
|
||||
<i class="fas fa-chevron-left" aria-hidden="true"></i>
|
||||
</button>
|
||||
{% for p in range(1, pagination.pages + 1) %}
|
||||
<button class="page-btn {% if p == pagination.page %}current{% endif %}"
|
||||
onclick="changePage({{ p }})"
|
||||
aria-label="Page {{ p }}"
|
||||
{% if p == pagination.page %}aria-current="page"{% endif %}>
|
||||
{{ p }}
|
||||
</button>
|
||||
{% endfor %}
|
||||
<button class="page-btn" onclick="changePage({{ pagination.page + 1 }})"
|
||||
{% if pagination.page >= pagination.pages %}disabled{% endif %}
|
||||
aria-label="Next page">
|
||||
<i class="fas fa-chevron-right" aria-hidden="true"></i>
|
||||
</button>
|
||||
</nav>
|
||||
{% endif %}
|
||||
|
||||
{% else %}
|
||||
<div class="empty-state">
|
||||
<i class="fas fa-check-circle text-green-400" aria-hidden="true"></i>
|
||||
<p class="font-semibold text-lg text-gray-700">No exact duplicates found</p>
|
||||
<p class="text-sm mt-1">
|
||||
Every file in the system has a unique SHA-256 hash.
|
||||
Use the <strong>Near-Duplicate Finder</strong> tab to check for
|
||||
re-scanned documents.
|
||||
</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div><!-- /tab-exact -->
|
||||
|
||||
<!-- ── Tab: Near-Duplicate Finder ── -->
|
||||
<div id="tab-near" class="tab-panel" role="tabpanel" aria-labelledby="btn-near">
|
||||
|
||||
<div class="near-dup-search">
|
||||
<h3>
|
||||
<i class="fas fa-search mr-1 text-yellow-500" aria-hidden="true"></i>
|
||||
Find Near-Duplicates for a Document
|
||||
</h3>
|
||||
<p class="text-sm text-gray-500 mb-4">
|
||||
Select a document to check whether any other files contain the same (or very
|
||||
similar) content — even if they were scanned at different times and have
|
||||
different SHA-256 hashes. Similarity is computed from OCR text embeddings;
|
||||
documents without extracted text cannot be compared.
|
||||
</p>
|
||||
|
||||
<form id="nearDupForm" onsubmit="findNearDups(event)">
|
||||
<div class="nd-form">
|
||||
<div class="flex flex-col gap-1">
|
||||
<label for="ndFileId">File ID</label>
|
||||
<input type="number" id="ndFileId" name="file_id" min="1"
|
||||
placeholder="e.g. 42" required
|
||||
style="min-height:44px;">
|
||||
</div>
|
||||
<div class="flex flex-col gap-1">
|
||||
<label for="ndThreshold">Similarity threshold</label>
|
||||
<input type="number" id="ndThreshold" name="threshold"
|
||||
min="0" max="1" step="0.05"
|
||||
value="{{ near_duplicate_threshold }}"
|
||||
style="min-height:44px;">
|
||||
</div>
|
||||
<div class="flex flex-col gap-1">
|
||||
<label for="ndLimit">Max results</label>
|
||||
<select id="ndLimit" name="limit" style="min-height:44px;">
|
||||
<option value="5" selected>5</option>
|
||||
<option value="10">10</option>
|
||||
<option value="20">20</option>
|
||||
</select>
|
||||
</div>
|
||||
<button type="submit"
|
||||
class="bg-yellow-500 hover:bg-yellow-600 text-white font-bold
|
||||
px-5 rounded-lg transition-colors"
|
||||
style="min-height:44px;">
|
||||
<i class="fas fa-search mr-1" aria-hidden="true"></i> Find
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<div id="ndStatus" class="mt-3 text-sm text-gray-500" aria-live="polite"></div>
|
||||
|
||||
<div id="ndResults" class="nd-results" aria-live="polite"></div>
|
||||
</div>
|
||||
|
||||
<!-- How it works explanation -->
|
||||
<div class="bg-blue-50 border border-blue-200 rounded-lg p-4 text-sm text-blue-800">
|
||||
<p class="font-semibold mb-1">
|
||||
<i class="fas fa-info-circle mr-1" aria-hidden="true"></i>
|
||||
How near-duplicate detection works
|
||||
</p>
|
||||
<ul class="list-disc ml-5 space-y-1">
|
||||
<li>After OCR processes a document, its text is converted to a numeric
|
||||
<em>embedding vector</em> using an AI language model.</li>
|
||||
<li>Near-duplicate detection compares these vectors using <em>cosine
|
||||
similarity</em>: a score of 1.0 means identical content, 0.0 means
|
||||
completely different.</li>
|
||||
<li>A threshold of <strong>{{ near_duplicate_threshold }}</strong> means
|
||||
documents must share at least
|
||||
{{ (near_duplicate_threshold * 100)|round|int }}% semantic similarity
|
||||
to be flagged.</li>
|
||||
<li>Documents scanned twice (even with slight differences) will typically
|
||||
score <strong>> 0.90</strong>.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div><!-- /tab-near -->
|
||||
|
||||
</main>
|
||||
{% endblock %}
|
||||
|
||||
{% block scripts %}
|
||||
<script>
|
||||
// ── Tab switching ──────────────────────────────────────────────────────────
|
||||
function switchTab(name) {
|
||||
document.querySelectorAll('.tab-btn').forEach(btn => {
|
||||
const active = btn.id === 'btn-' + name;
|
||||
btn.classList.toggle('active', active);
|
||||
btn.setAttribute('aria-selected', active ? 'true' : 'false');
|
||||
});
|
||||
document.querySelectorAll('.tab-panel').forEach(panel => {
|
||||
panel.classList.toggle('active', panel.id === 'tab-' + name);
|
||||
});
|
||||
}
|
||||
|
||||
// ── Pagination ─────────────────────────────────────────────────────────────
|
||||
function changePage(p) {
|
||||
const url = new URL(window.location.href);
|
||||
url.searchParams.set('page', p);
|
||||
window.location.href = url.toString();
|
||||
}
|
||||
|
||||
// ── Near-duplicate API call ───────────────────────────────────────────────
|
||||
async function findNearDups(e) {
|
||||
e.preventDefault();
|
||||
const fileId = document.getElementById('ndFileId').value.trim();
|
||||
const threshold = parseFloat(document.getElementById('ndThreshold').value) || 0.85;
|
||||
const limit = parseInt(document.getElementById('ndLimit').value) || 5;
|
||||
const status = document.getElementById('ndStatus');
|
||||
const results = document.getElementById('ndResults');
|
||||
|
||||
if (!fileId) {
|
||||
status.textContent = 'Please enter a File ID.';
|
||||
return;
|
||||
}
|
||||
|
||||
status.textContent = 'Searching…';
|
||||
results.innerHTML = '';
|
||||
|
||||
try {
|
||||
const url = `/api/files/${encodeURIComponent(fileId)}/duplicates`
|
||||
+ `?near_duplicate_threshold=${threshold}&near_duplicate_limit=${limit}`;
|
||||
const csrfToken = document.querySelector('meta[name="csrf-token"]')?.content;
|
||||
if (!csrfToken) {
|
||||
status.textContent = 'Security token not found. Please reload the page.';
|
||||
return;
|
||||
}
|
||||
const resp = await fetch(url, {
|
||||
headers: { 'X-CSRF-Token': csrfToken }
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
const err = await resp.json().catch(() => ({}));
|
||||
status.textContent = `Error ${resp.status}: ${err.detail || resp.statusText}`;
|
||||
return;
|
||||
}
|
||||
|
||||
const data = await resp.json();
|
||||
renderNearDupResults(data, threshold);
|
||||
status.textContent = '';
|
||||
} catch (err) {
|
||||
status.textContent = `Network error: ${err.message}`;
|
||||
}
|
||||
}
|
||||
|
||||
function renderNearDupResults(data, threshold) {
|
||||
const container = document.getElementById('ndResults');
|
||||
|
||||
if (!data.near_duplicates || data.near_duplicates.length === 0) {
|
||||
container.innerHTML = `
|
||||
<div class="empty-state py-6">
|
||||
<i class="fas fa-check-circle text-green-400 text-3xl mb-2" aria-hidden="true"></i>
|
||||
<p class="font-semibold text-gray-700">No near-duplicates found</p>
|
||||
<p class="text-sm text-gray-500 mt-1">
|
||||
No other document exceeded the ${(threshold * 100).toFixed(0)}% similarity threshold.
|
||||
${!data.near_duplicates ? 'This file may not have OCR text yet.' : ''}
|
||||
</p>
|
||||
</div>`;
|
||||
return;
|
||||
}
|
||||
|
||||
let html = `<h4 class="font-semibold text-gray-700 mb-2 mt-2">
|
||||
Found <strong>${data.near_duplicate_count}</strong> near-duplicate(s) for file #${data.file_id}
|
||||
</h4>
|
||||
<div class="dup-group">`;
|
||||
|
||||
// Show target file header
|
||||
html += `
|
||||
<div class="dup-group-header">
|
||||
<i class="fas fa-file-alt text-gray-400" aria-hidden="true"></i>
|
||||
<span class="text-sm font-semibold">Reference file: ID ${data.file_id}</span>
|
||||
<span class="dup-badge dup-badge-near ml-auto">
|
||||
Threshold ≥ ${(data.near_duplicate_threshold * 100).toFixed(0)}%
|
||||
</span>
|
||||
</div>`;
|
||||
|
||||
data.near_duplicates.forEach(nd => {
|
||||
const scorePercent = (nd.similarity_score * 100).toFixed(1);
|
||||
const scoreColor = nd.similarity_score >= 0.95
|
||||
? 'text-red-600' : nd.similarity_score >= 0.85
|
||||
? 'text-yellow-600' : 'text-orange-500';
|
||||
const createdAt = nd.created_at ? nd.created_at.substring(0, 19).replace('T', ' ') : '';
|
||||
|
||||
html += `
|
||||
<div class="dup-file-row near-dup">
|
||||
<span class="dup-role-tag tag-near">Near-Dup</span>
|
||||
<div class="flex-1 min-w-0">
|
||||
<div class="dup-filename">
|
||||
<a href="/files/${nd.file_id}" class="hover:text-blue-600">
|
||||
${escapeHtml(nd.original_filename || '(unnamed)')}
|
||||
</a>
|
||||
</div>
|
||||
<div class="dup-meta">
|
||||
ID: ${nd.file_id}
|
||||
${nd.document_title ? ' · ' + escapeHtml(nd.document_title) : ''}
|
||||
${createdAt ? ' · ' + createdAt : ''}
|
||||
</div>
|
||||
</div>
|
||||
<div class="dup-actions items-center">
|
||||
<span class="dup-score ${scoreColor}" title="Cosine similarity score">
|
||||
${scorePercent}% match
|
||||
</span>
|
||||
<a href="/files/${nd.file_id}"
|
||||
class="text-sm text-blue-600 hover:underline ml-2"
|
||||
aria-label="View file ${escapeHtml(nd.original_filename || '')}">
|
||||
<i class="fas fa-external-link-alt" aria-hidden="true"></i>
|
||||
</a>
|
||||
</div>
|
||||
</div>`;
|
||||
});
|
||||
|
||||
html += '</div>';
|
||||
container.innerHTML = html;
|
||||
}
|
||||
|
||||
function escapeHtml(str) {
|
||||
return String(str)
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"');
|
||||
}
|
||||
|
||||
// Pre-fill file ID from query string (e.g. /duplicates?file_id=42&tab=near)
|
||||
(function () {
|
||||
const params = new URLSearchParams(window.location.search);
|
||||
const fid = params.get('file_id');
|
||||
if (fid) document.getElementById('ndFileId').value = fid;
|
||||
const tab = params.get('tab');
|
||||
if (tab === 'near') switchTab('near');
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,404 @@
|
||||
"""Tests for duplicate document detection and management.
|
||||
|
||||
Covers:
|
||||
- ``GET /api/duplicates`` — list all exact-duplicate groups
|
||||
- ``GET /api/files/{id}/duplicates`` — per-file exact + near-duplicate info
|
||||
- ``POST /api/ui-upload`` — exact-duplicate warning in upload response
|
||||
- ``GET /duplicates`` — duplicate management UI page
|
||||
"""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.models import FileRecord
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_file(db, *, filehash, filename, is_duplicate=False, duplicate_of_id=None, ocr_text=None):
|
||||
"""Insert a FileRecord and return it."""
|
||||
record = FileRecord(
|
||||
filehash=filehash,
|
||||
original_filename=filename,
|
||||
local_filename=f"/tmp/{filename}",
|
||||
file_size=1024,
|
||||
mime_type="application/pdf",
|
||||
is_duplicate=is_duplicate,
|
||||
duplicate_of_id=duplicate_of_id,
|
||||
ocr_text=ocr_text,
|
||||
)
|
||||
db.add(record)
|
||||
db.commit()
|
||||
db.refresh(record)
|
||||
return record
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GET /api/duplicates
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestListDuplicateGroups:
|
||||
"""Tests for the GET /api/duplicates endpoint."""
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_returns_empty_when_no_duplicates(self, client: TestClient):
|
||||
"""Should return empty groups list when no duplicates exist."""
|
||||
response = client.get("/api/duplicates")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total_groups"] == 0
|
||||
assert data["groups"] == []
|
||||
assert data["total_duplicate_files"] == 0
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_returns_duplicate_group(self, client: TestClient, db_session):
|
||||
"""Should return one group with original and duplicate."""
|
||||
original = _make_file(db_session, filehash="aaa111", filename="doc.pdf")
|
||||
dup = _make_file(
|
||||
db_session,
|
||||
filehash="aaa111",
|
||||
filename="doc_copy.pdf",
|
||||
is_duplicate=True,
|
||||
duplicate_of_id=original.id,
|
||||
)
|
||||
|
||||
response = client.get("/api/duplicates")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total_groups"] == 1
|
||||
assert data["total_duplicate_files"] == 1
|
||||
|
||||
group = data["groups"][0]
|
||||
assert group["filehash"] == "aaa111"
|
||||
assert group["duplicate_count"] == 1
|
||||
assert group["original"]["id"] == original.id
|
||||
assert group["duplicates"][0]["id"] == dup.id
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_multiple_groups(self, client: TestClient, db_session):
|
||||
"""Should handle multiple distinct duplicate groups."""
|
||||
orig1 = _make_file(db_session, filehash="hash1", filename="a.pdf")
|
||||
_make_file(db_session, filehash="hash1", filename="a_copy.pdf", is_duplicate=True, duplicate_of_id=orig1.id)
|
||||
|
||||
orig2 = _make_file(db_session, filehash="hash2", filename="b.pdf")
|
||||
_make_file(db_session, filehash="hash2", filename="b_copy.pdf", is_duplicate=True, duplicate_of_id=orig2.id)
|
||||
|
||||
response = client.get("/api/duplicates")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["total_groups"] == 2
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_pagination(self, client: TestClient, db_session):
|
||||
"""Should respect page/per_page parameters."""
|
||||
for i in range(5):
|
||||
orig = _make_file(db_session, filehash=f"phash{i}", filename=f"p{i}.pdf")
|
||||
_make_file(
|
||||
db_session, filehash=f"phash{i}", filename=f"p{i}c.pdf", is_duplicate=True, duplicate_of_id=orig.id
|
||||
)
|
||||
|
||||
r1 = client.get("/api/duplicates?per_page=2&page=1")
|
||||
assert r1.status_code == 200
|
||||
d1 = r1.json()
|
||||
assert len(d1["groups"]) == 2
|
||||
assert d1["pagination"]["total"] == 5
|
||||
assert d1["pagination"]["pages"] == 3
|
||||
|
||||
r2 = client.get("/api/duplicates?per_page=2&page=2")
|
||||
assert r2.status_code == 200
|
||||
assert len(r2.json()["groups"]) == 2
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_response_structure(self, client: TestClient, db_session):
|
||||
"""Each group should have the expected keys."""
|
||||
orig = _make_file(db_session, filehash="struct1", filename="s.pdf")
|
||||
_make_file(db_session, filehash="struct1", filename="s2.pdf", is_duplicate=True, duplicate_of_id=orig.id)
|
||||
|
||||
data = client.get("/api/duplicates").json()
|
||||
group = data["groups"][0]
|
||||
assert "filehash" in group
|
||||
assert "original" in group
|
||||
assert "duplicates" in group
|
||||
assert "duplicate_count" in group
|
||||
|
||||
orig_dict = group["original"]
|
||||
assert "id" in orig_dict
|
||||
assert "original_filename" in orig_dict
|
||||
assert "filehash" in orig_dict
|
||||
assert "is_duplicate" in orig_dict
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GET /api/files/{file_id}/duplicates
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetFileDuplicates:
|
||||
"""Tests for the GET /api/files/{id}/duplicates endpoint."""
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_404_for_missing_file(self, client: TestClient):
|
||||
response = client.get("/api/files/99999/duplicates")
|
||||
assert response.status_code == 404
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_no_duplicates_returns_empty(self, client: TestClient, db_session):
|
||||
"""File with no duplicates returns empty lists."""
|
||||
f = _make_file(db_session, filehash="unique111", filename="unique.pdf")
|
||||
response = client.get(f"/api/files/{f.id}/duplicates")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["exact_duplicates"] == []
|
||||
assert data["near_duplicates"] == []
|
||||
assert data["exact_duplicate_count"] == 0
|
||||
assert data["near_duplicate_count"] == 0
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_exact_duplicates_returned(self, client: TestClient, db_session):
|
||||
"""Exact duplicates (same hash) should be listed."""
|
||||
orig = _make_file(db_session, filehash="dup_hash", filename="orig.pdf")
|
||||
dup = _make_file(
|
||||
db_session,
|
||||
filehash="dup_hash",
|
||||
filename="dup.pdf",
|
||||
is_duplicate=True,
|
||||
duplicate_of_id=orig.id,
|
||||
)
|
||||
|
||||
response = client.get(f"/api/files/{orig.id}/duplicates")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["exact_duplicate_count"] == 1
|
||||
assert data["exact_duplicates"][0]["id"] == dup.id
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_self_is_duplicate_flag(self, client: TestClient, db_session):
|
||||
"""When the queried file is itself a duplicate, is_duplicate=True and duplicate_of is populated."""
|
||||
orig = _make_file(db_session, filehash="selfdup", filename="orig.pdf")
|
||||
dup = _make_file(
|
||||
db_session,
|
||||
filehash="selfdup",
|
||||
filename="copy.pdf",
|
||||
is_duplicate=True,
|
||||
duplicate_of_id=orig.id,
|
||||
)
|
||||
|
||||
response = client.get(f"/api/files/{dup.id}/duplicates")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["is_duplicate"] is True
|
||||
assert data["duplicate_of"] is not None
|
||||
assert data["duplicate_of"]["id"] == orig.id
|
||||
|
||||
@pytest.mark.integration
|
||||
@patch("app.utils.similarity.generate_embedding")
|
||||
def test_near_duplicates_returned(self, mock_embed, client: TestClient, db_session):
|
||||
"""Near-duplicates found via embedding similarity should appear in results."""
|
||||
target = _make_file(
|
||||
db_session,
|
||||
filehash="th1",
|
||||
filename="target.pdf",
|
||||
ocr_text="Invoice from Acme Corp for January services rendered",
|
||||
)
|
||||
similar = _make_file(
|
||||
db_session,
|
||||
filehash="th2", # different hash — same content (re-scan)
|
||||
filename="rescan.pdf",
|
||||
ocr_text="Invoice from Acme Corp for January services rendered",
|
||||
)
|
||||
|
||||
# Same embedding → cosine similarity = 1.0
|
||||
mock_embed.return_value = [1.0, 0.0, 0.0]
|
||||
|
||||
response = client.get(f"/api/files/{target.id}/duplicates?near_duplicate_threshold=0.8")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["near_duplicate_count"] >= 1
|
||||
ids = [nd["file_id"] for nd in data["near_duplicates"]]
|
||||
assert similar.id in ids
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_no_near_duplicates_without_ocr(self, client: TestClient, db_session):
|
||||
"""Files without OCR text should return empty near_duplicates."""
|
||||
f = _make_file(db_session, filehash="noocr1", filename="noocr.pdf", ocr_text=None)
|
||||
response = client.get(f"/api/files/{f.id}/duplicates")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["near_duplicates"] == []
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_threshold_filters_near_duplicates(self, client: TestClient, db_session):
|
||||
"""A very high threshold should filter out lower-scoring near-duplicates."""
|
||||
target = _make_file(
|
||||
db_session,
|
||||
filehash="tt1",
|
||||
filename="t.pdf",
|
||||
ocr_text="Some document text about invoices",
|
||||
)
|
||||
_make_file(
|
||||
db_session,
|
||||
filehash="tt2",
|
||||
filename="c.pdf",
|
||||
ocr_text="Some document text about invoices",
|
||||
)
|
||||
|
||||
# Patch embeddings to give moderate similarity
|
||||
with patch("app.utils.similarity.generate_embedding") as mock_embed:
|
||||
# target gets [1,0,0], candidate gets [0.6, 0.8, 0.0] → ~0.6 similarity
|
||||
mock_embed.side_effect = lambda text: [1.0, 0.0, 0.0] if target.ocr_text in text else [0.6, 0.8, 0.0]
|
||||
|
||||
# Very high threshold — should not match
|
||||
response = client.get(f"/api/files/{target.id}/duplicates?near_duplicate_threshold=0.99")
|
||||
assert response.status_code == 200
|
||||
# near_duplicates may or may not be empty depending on the mock, but 200 must succeed
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_response_contains_required_fields(self, client: TestClient, db_session):
|
||||
"""Response must always include all required top-level fields."""
|
||||
f = _make_file(db_session, filehash="reqf", filename="req.pdf")
|
||||
data = client.get(f"/api/files/{f.id}/duplicates").json()
|
||||
required = {
|
||||
"file_id",
|
||||
"is_duplicate",
|
||||
"duplicate_of",
|
||||
"exact_duplicates",
|
||||
"near_duplicates",
|
||||
"near_duplicate_threshold",
|
||||
"exact_duplicate_count",
|
||||
"near_duplicate_count",
|
||||
}
|
||||
for key in required:
|
||||
assert key in data, f"Missing key: {key}"
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_invalid_threshold_rejected(self, client: TestClient, db_session):
|
||||
"""Threshold outside [−1, 1] should be rejected with 422."""
|
||||
f = _make_file(db_session, filehash="vth", filename="v.pdf")
|
||||
response = client.get(f"/api/files/{f.id}/duplicates?near_duplicate_threshold=2.0")
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# POST /api/ui-upload — exact-duplicate warning
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestUploadDuplicateWarning:
|
||||
"""Tests for duplicate warning injected into the upload response."""
|
||||
|
||||
@pytest.mark.integration
|
||||
@patch("app.tasks.process_document.process_document.delay")
|
||||
def test_no_warning_for_unique_file(self, mock_delay, client: TestClient, tmp_path):
|
||||
"""Uploading a unique file should not produce a duplicate_warning."""
|
||||
mock_delay.return_value.id = "task-unique"
|
||||
pdf = tmp_path / "unique.pdf"
|
||||
pdf.write_bytes(b"%PDF-1.4\n%%EOF")
|
||||
|
||||
with open(pdf, "rb") as f:
|
||||
response = client.post(
|
||||
"/api/ui-upload",
|
||||
files={"file": ("unique.pdf", f, "application/pdf")},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "duplicate_warning" not in data or data.get("duplicate_warning") is None
|
||||
|
||||
@pytest.mark.integration
|
||||
@patch("app.tasks.process_document.process_document.delay")
|
||||
def test_warning_for_exact_duplicate(self, mock_delay, client: TestClient, db_session, tmp_path):
|
||||
"""Uploading a file with the same hash as an existing record returns a warning."""
|
||||
mock_delay.return_value.id = "task-dup"
|
||||
|
||||
# Create a real PDF with known content
|
||||
pdf_bytes = b"%PDF-1.4\nsome unique content for test\n%%EOF"
|
||||
pdf = tmp_path / "existing.pdf"
|
||||
pdf.write_bytes(pdf_bytes)
|
||||
|
||||
# Compute the hash to insert a matching record
|
||||
from app.utils.file_operations import hash_file
|
||||
|
||||
filehash = hash_file(str(pdf))
|
||||
|
||||
existing = _make_file(db_session, filehash=filehash, filename="existing.pdf")
|
||||
|
||||
# Upload the same file (same bytes → same hash)
|
||||
with open(pdf, "rb") as f:
|
||||
response = client.post(
|
||||
"/api/ui-upload",
|
||||
files={"file": ("dup_upload.pdf", f, "application/pdf")},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "duplicate_warning" in data
|
||||
assert data["duplicate_warning"]["duplicate_type"] == "exact"
|
||||
assert data["duplicate_warning"]["original_file_id"] == existing.id
|
||||
|
||||
@pytest.mark.integration
|
||||
@patch("app.tasks.process_document.process_document.delay")
|
||||
def test_upload_still_queued_despite_warning(self, mock_delay, client: TestClient, db_session, tmp_path):
|
||||
"""Even when a duplicate is detected, the file should still be queued."""
|
||||
mock_delay.return_value.id = "task-still-queued"
|
||||
|
||||
pdf_bytes = b"%PDF-1.4\nqueue test content\n%%EOF"
|
||||
pdf = tmp_path / "queue_test.pdf"
|
||||
pdf.write_bytes(pdf_bytes)
|
||||
|
||||
from app.utils.file_operations import hash_file
|
||||
|
||||
filehash = hash_file(str(pdf))
|
||||
_make_file(db_session, filehash=filehash, filename="queue_orig.pdf")
|
||||
|
||||
with open(pdf, "rb") as f:
|
||||
response = client.post(
|
||||
"/api/ui-upload",
|
||||
files={"file": ("queue_test.pdf", f, "application/pdf")},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "task_id" in data
|
||||
assert data["status"] == "queued"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GET /duplicates — duplicate management UI page
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDuplicatesViewPage:
|
||||
"""Tests for the /duplicates HTML view."""
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_page_renders_empty(self, client: TestClient):
|
||||
"""Duplicates page should render without errors when no duplicates exist."""
|
||||
response = client.get("/duplicates")
|
||||
assert response.status_code == 200
|
||||
assert b"Duplicate Documents" in response.content
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_page_shows_duplicate_groups(self, client: TestClient, db_session):
|
||||
"""Page should list duplicate groups when they exist."""
|
||||
orig = _make_file(db_session, filehash="view_hash", filename="view_orig.pdf")
|
||||
_make_file(
|
||||
db_session,
|
||||
filehash="view_hash",
|
||||
filename="view_dup.pdf",
|
||||
is_duplicate=True,
|
||||
duplicate_of_id=orig.id,
|
||||
)
|
||||
|
||||
response = client.get("/duplicates")
|
||||
assert response.status_code == 200
|
||||
assert b"view_orig.pdf" in response.content or b"view_hash" in response.content
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_page_contains_near_dup_tab(self, client: TestClient):
|
||||
"""Page should include the Near-Duplicate Finder tab."""
|
||||
response = client.get("/duplicates")
|
||||
assert response.status_code == 200
|
||||
assert b"Near-Duplicate Finder" in response.content or b"near" in response.content.lower()
|
||||
Reference in New Issue
Block a user