diff --git a/.env.demo b/.env.demo index e06ab45b..3158ab8c 100644 --- a/.env.demo +++ b/.env.demo @@ -346,4 +346,10 @@ ENABLE_DEDUPLICATION=True SHOW_DEDUPLICATION_STEP=True # Minimum cosine similarity score (0–1) for two documents to be flagged as # near-duplicates. 0.85 means 85 % semantic overlap. Lower = more matches. -NEAR_DUPLICATE_THRESHOLD=0.85 +NEAR_DUPLICATE_THRESHOLD=0.85 +# Model used to generate text embeddings for document similarity. +# Must be supported by your OpenAI-compatible API endpoint. +EMBEDDING_MODEL=text-embedding-3-small +# Maximum tokens to send to the embedding model. Set below the model's +# context window (e.g. 8000 for an 8192-token model). +EMBEDDING_MAX_TOKENS=8000 diff --git a/app/config.py b/app/config.py index 62d87607..df5e60bc 100644 --- a/app/config.py +++ b/app/config.py @@ -330,6 +330,15 @@ class Settings(BaseSettings): "Embeddings drive the document similarity feature. Default: text-embedding-3-small." ), ) + embedding_max_tokens: int = Field( + default=8000, + description=( + "Maximum number of tokens to send to the embedding model. " + "Text is truncated to approximately this many tokens (using a " + "conservative 3-chars-per-token estimate) before calling the API. " + "Set this below the model's context window (e.g. 8000 for an 8192-token model)." + ), + ) # Text quality check - AI-based assessment of embedded PDF text enable_text_quality_check: bool = Field( diff --git a/app/utils/similarity.py b/app/utils/similarity.py index c1e69c15..b5948719 100644 --- a/app/utils/similarity.py +++ b/app/utils/similarity.py @@ -42,8 +42,9 @@ def generate_embedding(text: str, model: str | None = None) -> list[float]: """Generate a text embedding vector using the OpenAI-compatible API. Args: - text: The input text to embed. Truncated to ~8000 tokens worth of - characters to stay within model limits. + text: The input text to embed. Truncated to stay within the + model's context window based on ``settings.embedding_max_tokens`` + (default 8 000 tokens ≈ 24 000 characters). model: The embedding model to use. When ``None`` (the default), the value of ``settings.embedding_model`` is used. @@ -57,9 +58,18 @@ def generate_embedding(text: str, model: str | None = None) -> list[float]: if model is None: model = settings.embedding_model - # Truncate very long texts to stay within token limits (~4 chars per token) - max_chars = 30000 + # Truncate to stay within the model's context window. + # Use a conservative estimate of ~3 characters per token so that the + # resulting text fits comfortably within ``embedding_max_tokens``. + max_chars = settings.embedding_max_tokens * 3 if len(text) > max_chars: + logger.debug( + "Truncating text from %d to %d chars (~%d tokens) for model %s", + len(text), + max_chars, + settings.embedding_max_tokens, + model, + ) text = text[:max_chars] client = _get_embedding_client() diff --git a/app/utils/step_manager.py b/app/utils/step_manager.py index ca094847..ecbf2482 100644 --- a/app/utils/step_manager.py +++ b/app/utils/step_manager.py @@ -218,6 +218,7 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict: "embed_metadata_into_pdf", "finalize_document_storage", "send_to_all_destinations", + "compute_embedding", } # Add check_for_duplicates if deduplication is enabled @@ -326,6 +327,7 @@ def get_step_summary(db: Session, file_id: int) -> Dict: "embed_metadata_into_pdf", "finalize_document_storage", "send_to_all_destinations", + "compute_embedding", } # Add check_for_duplicates if deduplication is enabled diff --git a/docs/API.md b/docs/API.md index 78356dab..304345b0 100644 --- a/docs/API.md +++ b/docs/API.md @@ -670,7 +670,7 @@ curl -OJ "http:///api/files/123/download?version=original" **GET** `/api/files/{file_id}/similar` -Find documents similar to the specified file using text embeddings and cosine similarity. Similarity scores range from 0 (completely different) to 1 (identical content). Embeddings are generated from OCR-extracted text and cached for subsequent requests. +Find documents similar to the specified file using pre-computed text embeddings and cosine similarity. Similarity scores range from 0 (completely different) to 1 (identical content). Embeddings are computed automatically during document ingestion and cached in the database. **Parameters**: - `limit` (optional, default: `5`, max: `20`): Maximum number of similar documents to return @@ -706,9 +706,94 @@ curl "http:///api/files/42/similar?limit=10&threshold=0.5" **Error Responses**: - `404`: File not found - `422`: Invalid query parameters (limit or threshold out of range) -- `500`: Embedding generation failed +- `500`: Internal error -> **Note:** Documents without OCR text are excluded from similarity comparisons. The response includes a `message` field when the target file has no OCR text available. +> **Note:** Only pre-computed embeddings are used — no API calls are made during the query. If a file's embedding has not been computed yet, the response includes a `message` field explaining this. Documents without OCR text are excluded from similarity comparisons. + +### Similarity Pairs (Corpus-Wide) + +**GET** `/api/similarity/pairs` + +Scan the entire document corpus for pairs of highly similar documents, ranked by score. Unlike the per-file `/files/{id}/similar` endpoint, this discovers all matching pairs across all files. + +**Parameters**: +- `threshold` (optional, default: `0.7`, range: `0.0–1.0`): Minimum similarity score for a pair +- `limit` (optional, default: `50`, max: `200`): Maximum pairs per page +- `page` (optional, default: `1`): Page number + +**Response**: +```json +{ + "pairs": [ + { + "file_a": { + "file_id": 1, + "original_filename": "invoice_jan.pdf", + "document_title": "January Invoice", + "mime_type": "application/pdf", + "created_at": "2026-01-15T10:30:00+00:00" + }, + "file_b": { + "file_id": 5, + "original_filename": "invoice_feb.pdf", + "document_title": "February Invoice", + "mime_type": "application/pdf", + "created_at": "2026-02-15T10:30:00+00:00" + }, + "similarity_score": 0.94 + } + ], + "total_pairs": 12, + "threshold": 0.7, + "page": 1, + "pages": 1, + "per_page": 50, + "embedding_coverage": { + "total_files": 120, + "files_with_embedding": 95 + } +} +``` + +**Example**: +```bash +# Find all document pairs above 90% similarity +curl "http:///api/similarity/pairs?threshold=0.9" +``` + +### Embedding Diagnostics + +**GET** `/api/files/{file_id}/embedding-status` + +Check the embedding status for a specific file: whether OCR text is available, whether an embedding has been computed, and how many dimensions it has. + +```bash +curl "http:///api/files/42/embedding-status" +``` + +**POST** `/api/files/{file_id}/compute-embedding` + +Manually trigger embedding computation for a single file. Useful for debugging or re-computing after configuration changes. Requires OCR text to be available. + +```bash +curl -X POST "http:///api/files/42/compute-embedding" +``` + +**GET** `/api/diagnostic/embeddings` + +Get an overview of embedding coverage across all files: total files, how many have OCR text, how many have embeddings, and per-file status. + +```bash +curl "http:///api/diagnostic/embeddings" +``` + +**POST** `/api/diagnostic/compute-all-embeddings` + +Queue embedding computation for all files that have OCR text but no embedding yet. Each file is processed as a separate background task. + +```bash +curl -X POST "http:///api/diagnostic/compute-all-embeddings" +``` ### Batch Processing diff --git a/docs/ConfigurationGuide.md b/docs/ConfigurationGuide.md index 1d60ee4a..e7fd4120 100644 --- a/docs/ConfigurationGuide.md +++ b/docs/ConfigurationGuide.md @@ -868,10 +868,15 @@ After OCR processes a document, its extracted text is converted to a vector embe | Variable | Description | Default | |---|---|---| | `NEAR_DUPLICATE_THRESHOLD` | Minimum cosine similarity (0–1) for two documents to be considered near-duplicates. `0.85` means ≥ 85 % semantic overlap. | `0.85` | +| `EMBEDDING_MODEL` | Model name for generating text embeddings via the OpenAI-compatible API. Must be supported by the endpoint configured with `OPENAI_BASE_URL`. | `text-embedding-3-small` | +| `EMBEDDING_MAX_TOKENS` | Maximum tokens to send to the embedding model. Text is truncated to approximately this many tokens before calling the API. Set below the model's context window (e.g. 8 000 for an 8 192-token model). | `8000` | Near-duplicate detection: -- Is performed **on demand** via `GET /api/files/{id}/duplicates` — not automatically during ingest (OCR text is required). -- Is exposed in the **Duplicates** management page (`/duplicates` → "Near-Duplicate Finder" tab). +- Embeddings are computed **automatically during document ingestion** as a processing step ("Compute Embedding"). +- A periodic **backfill task** (every 5 minutes) picks up any files that were processed before the embedding pipeline was enabled. +- The **Similarity dashboard** (`/similarity`) shows all pairs of documents above the threshold, ranked by score. +- The **Duplicates** management page (`/duplicates` → "Near-Duplicate Finder" tab) allows per-file lookup. +- Debug endpoints are available to inspect embedding status and trigger recomputation (see API docs). - Documents without OCR text cannot be compared and are excluded from results. A score of **≥ 0.90** reliably identifies the same document scanned twice. A score of **0.70–0.90** suggests partial content overlap. Adjust `NEAR_DUPLICATE_THRESHOLD` to tune sensitivity. diff --git a/tests/test_duplicates.py b/tests/test_duplicates.py index 6dd84b2d..45486168 100644 --- a/tests/test_duplicates.py +++ b/tests/test_duplicates.py @@ -7,6 +7,7 @@ Covers: - ``GET /duplicates`` — duplicate management UI page """ +import json from unittest.mock import patch import pytest @@ -19,7 +20,7 @@ from app.models import FileRecord # --------------------------------------------------------------------------- -def _make_file(db, *, filehash, filename, is_duplicate=False, duplicate_of_id=None, ocr_text=None): +def _make_file(db, *, filehash, filename, is_duplicate=False, duplicate_of_id=None, ocr_text=None, embedding=None): """Insert a FileRecord and return it.""" record = FileRecord( filehash=filehash, @@ -30,6 +31,7 @@ def _make_file(db, *, filehash, filename, is_duplicate=False, duplicate_of_id=No is_duplicate=is_duplicate, duplicate_of_id=duplicate_of_id, ocr_text=ocr_text, + embedding=embedding, ) db.add(record) db.commit() @@ -195,25 +197,24 @@ class TestGetFileDuplicates: assert data["duplicate_of"]["id"] == orig.id @pytest.mark.integration - @patch("app.utils.similarity.generate_embedding") - def test_near_duplicates_returned(self, mock_embed, client: TestClient, db_session): + def test_near_duplicates_returned(self, client: TestClient, db_session): """Near-duplicates found via embedding similarity should appear in results.""" + embedding = json.dumps([1.0, 0.0, 0.0]) target = _make_file( db_session, filehash="th1", filename="target.pdf", ocr_text="Invoice from Acme Corp for January services rendered", + embedding=embedding, ) similar = _make_file( db_session, filehash="th2", # different hash — same content (re-scan) filename="rescan.pdf", ocr_text="Invoice from Acme Corp for January services rendered", + embedding=embedding, ) - # Same embedding → cosine similarity = 1.0 - mock_embed.return_value = [1.0, 0.0, 0.0] - response = client.get(f"/api/files/{target.id}/duplicates?near_duplicate_threshold=0.8") assert response.status_code == 200 data = response.json()