fix(similarity): truncate text to fit embedding model context window, fix step tracking

- Add EMBEDDING_MAX_TOKENS config (default 8000) for safe text truncation
- Use conservative 3 chars/token estimate (was 4) to prevent ContextWindowExceededError
- Add compute_embedding to REAL_MAIN_STEPS in both get_file_overall_status and get_step_summary
- Fix test_near_duplicates_returned to use pre-computed embeddings
- Update .env.demo and docs with EMBEDDING_MAX_TOKENS setting

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

.env.demo

@@ -346,4 +346,10 @@ ENABLE_DEDUPLICATION=True
 SHOW_DEDUPLICATION_STEP=True
 # Minimum cosine similarity score (0–1) for two documents to be flagged as
 # near-duplicates.  0.85 means 85 % semantic overlap.  Lower = more matches.
-NEAR_DUPLICATE_THRESHOLD=0.85
+NEAR_DUPLICATE_THRESHOLD=0.85
+# Model used to generate text embeddings for document similarity.
+# Must be supported by your OpenAI-compatible API endpoint.
+EMBEDDING_MODEL=text-embedding-3-small
+# Maximum tokens to send to the embedding model.  Set below the model's
+# context window (e.g. 8000 for an 8192-token model).
+EMBEDDING_MAX_TOKENS=8000

app/config.py

@@ -330,6 +330,15 @@ class Settings(BaseSettings):
             "Embeddings drive the document similarity feature.  Default: text-embedding-3-small."
         ),
     )
+    embedding_max_tokens: int = Field(
+        default=8000,
+        description=(
+            "Maximum number of tokens to send to the embedding model.  "
+            "Text is truncated to approximately this many tokens (using a "
+            "conservative 3-chars-per-token estimate) before calling the API.  "
+            "Set this below the model's context window (e.g. 8000 for an 8192-token model)."
+        ),
+    )
 
     # Text quality check - AI-based assessment of embedded PDF text
     enable_text_quality_check: bool = Field(

app/utils/similarity.py

@@ -42,8 +42,9 @@ def generate_embedding(text: str, model: str | None = None) -> list[float]:
     """Generate a text embedding vector using the OpenAI-compatible API.
 
     Args:
-        text: The input text to embed.  Truncated to ~8000 tokens worth of
-            characters to stay within model limits.
+        text: The input text to embed.  Truncated to stay within the
+            model's context window based on ``settings.embedding_max_tokens``
+            (default 8 000 tokens ≈ 24 000 characters).
         model: The embedding model to use.  When ``None`` (the default), the
             value of ``settings.embedding_model`` is used.
 
@@ -57,9 +58,18 @@ def generate_embedding(text: str, model: str | None = None) -> list[float]:
     if model is None:
         model = settings.embedding_model
 
-    # Truncate very long texts to stay within token limits (~4 chars per token)
-    max_chars = 30000
+    # Truncate to stay within the model's context window.
+    # Use a conservative estimate of ~3 characters per token so that the
+    # resulting text fits comfortably within ``embedding_max_tokens``.
+    max_chars = settings.embedding_max_tokens * 3
     if len(text) > max_chars:
+        logger.debug(
+            "Truncating text from %d to %d chars (~%d tokens) for model %s",
+            len(text),
+            max_chars,
+            settings.embedding_max_tokens,
+            model,
+        )
         text = text[:max_chars]
 
     client = _get_embedding_client()

app/utils/step_manager.py

@@ -218,6 +218,7 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict:
         "embed_metadata_into_pdf",
         "finalize_document_storage",
         "send_to_all_destinations",
+        "compute_embedding",
     }
 
     # Add check_for_duplicates if deduplication is enabled
@@ -326,6 +327,7 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
         "embed_metadata_into_pdf",
         "finalize_document_storage",
         "send_to_all_destinations",
+        "compute_embedding",
     }
 
     # Add check_for_duplicates if deduplication is enabled

docs/API.md

@@ -670,7 +670,7 @@ curl -OJ "http://<your-instance>/api/files/123/download?version=original"
 
 **GET** `/api/files/{file_id}/similar`
 
-Find documents similar to the specified file using text embeddings and cosine similarity. Similarity scores range from 0 (completely different) to 1 (identical content). Embeddings are generated from OCR-extracted text and cached for subsequent requests.
+Find documents similar to the specified file using pre-computed text embeddings and cosine similarity. Similarity scores range from 0 (completely different) to 1 (identical content). Embeddings are computed automatically during document ingestion and cached in the database.
 
 **Parameters**:
 - `limit` (optional, default: `5`, max: `20`): Maximum number of similar documents to return
@@ -706,9 +706,94 @@ curl "http://<your-instance>/api/files/42/similar?limit=10&threshold=0.5"
 **Error Responses**:
 - `404`: File not found
 - `422`: Invalid query parameters (limit or threshold out of range)
-- `500`: Embedding generation failed
+- `500`: Internal error
 
-> **Note:** Documents without OCR text are excluded from similarity comparisons. The response includes a `message` field when the target file has no OCR text available.
+> **Note:** Only pre-computed embeddings are used — no API calls are made during the query. If a file's embedding has not been computed yet, the response includes a `message` field explaining this. Documents without OCR text are excluded from similarity comparisons.
+
+### Similarity Pairs (Corpus-Wide)
+
+**GET** `/api/similarity/pairs`
+
+Scan the entire document corpus for pairs of highly similar documents, ranked by score. Unlike the per-file `/files/{id}/similar` endpoint, this discovers all matching pairs across all files.
+
+**Parameters**:
+- `threshold` (optional, default: `0.7`, range: `0.0–1.0`): Minimum similarity score for a pair
+- `limit` (optional, default: `50`, max: `200`): Maximum pairs per page
+- `page` (optional, default: `1`): Page number
+
+**Response**:
+```json
+{
+  "pairs": [
+    {
+      "file_a": {
+        "file_id": 1,
+        "original_filename": "invoice_jan.pdf",
+        "document_title": "January Invoice",
+        "mime_type": "application/pdf",
+        "created_at": "2026-01-15T10:30:00+00:00"
+      },
+      "file_b": {
+        "file_id": 5,
+        "original_filename": "invoice_feb.pdf",
+        "document_title": "February Invoice",
+        "mime_type": "application/pdf",
+        "created_at": "2026-02-15T10:30:00+00:00"
+      },
+      "similarity_score": 0.94
+    }
+  ],
+  "total_pairs": 12,
+  "threshold": 0.7,
+  "page": 1,
+  "pages": 1,
+  "per_page": 50,
+  "embedding_coverage": {
+    "total_files": 120,
+    "files_with_embedding": 95
+  }
+}
+```
+
+**Example**:
+```bash
+# Find all document pairs above 90% similarity
+curl "http://<your-instance>/api/similarity/pairs?threshold=0.9"
+```
+
+### Embedding Diagnostics
+
+**GET** `/api/files/{file_id}/embedding-status`
+
+Check the embedding status for a specific file: whether OCR text is available, whether an embedding has been computed, and how many dimensions it has.
+
+```bash
+curl "http://<your-instance>/api/files/42/embedding-status"
+```
+
+**POST** `/api/files/{file_id}/compute-embedding`
+
+Manually trigger embedding computation for a single file. Useful for debugging or re-computing after configuration changes. Requires OCR text to be available.
+
+```bash
+curl -X POST "http://<your-instance>/api/files/42/compute-embedding"
+```
+
+**GET** `/api/diagnostic/embeddings`
+
+Get an overview of embedding coverage across all files: total files, how many have OCR text, how many have embeddings, and per-file status.
+
+```bash
+curl "http://<your-instance>/api/diagnostic/embeddings"
+```
+
+**POST** `/api/diagnostic/compute-all-embeddings`
+
+Queue embedding computation for all files that have OCR text but no embedding yet. Each file is processed as a separate background task.
+
+```bash
+curl -X POST "http://<your-instance>/api/diagnostic/compute-all-embeddings"
+```
 
 ### Batch Processing
 

docs/ConfigurationGuide.md

@@ -868,10 +868,15 @@ After OCR processes a document, its extracted text is converted to a vector embe
 | Variable | Description | Default |
 |---|---|---|
 | `NEAR_DUPLICATE_THRESHOLD` | Minimum cosine similarity (0–1) for two documents to be considered near-duplicates. `0.85` means ≥ 85 % semantic overlap. | `0.85` |
+| `EMBEDDING_MODEL` | Model name for generating text embeddings via the OpenAI-compatible API.  Must be supported by the endpoint configured with `OPENAI_BASE_URL`. | `text-embedding-3-small` |
+| `EMBEDDING_MAX_TOKENS` | Maximum tokens to send to the embedding model.  Text is truncated to approximately this many tokens before calling the API.  Set below the model's context window (e.g. 8 000 for an 8 192-token model). | `8000` |
 
 Near-duplicate detection:
-- Is performed **on demand** via `GET /api/files/{id}/duplicates` — not automatically during ingest (OCR text is required).
-- Is exposed in the **Duplicates** management page (`/duplicates` → "Near-Duplicate Finder" tab).
+- Embeddings are computed **automatically during document ingestion** as a processing step ("Compute Embedding").
+- A periodic **backfill task** (every 5 minutes) picks up any files that were processed before the embedding pipeline was enabled.
+- The **Similarity dashboard** (`/similarity`) shows all pairs of documents above the threshold, ranked by score.
+- The **Duplicates** management page (`/duplicates` → "Near-Duplicate Finder" tab) allows per-file lookup.
+- Debug endpoints are available to inspect embedding status and trigger recomputation (see API docs).
 - Documents without OCR text cannot be compared and are excluded from results.
 
 A score of **≥ 0.90** reliably identifies the same document scanned twice. A score of **0.70–0.90** suggests partial content overlap. Adjust `NEAR_DUPLICATE_THRESHOLD` to tune sensitivity.

tests/test_duplicates.py

@@ -7,6 +7,7 @@ Covers:
 - ``GET /duplicates``                — duplicate management UI page
 """
 
+import json
 from unittest.mock import patch
 
 import pytest
@@ -19,7 +20,7 @@ from app.models import FileRecord
 # ---------------------------------------------------------------------------
 
 
-def _make_file(db, *, filehash, filename, is_duplicate=False, duplicate_of_id=None, ocr_text=None):
+def _make_file(db, *, filehash, filename, is_duplicate=False, duplicate_of_id=None, ocr_text=None, embedding=None):
     """Insert a FileRecord and return it."""
     record = FileRecord(
         filehash=filehash,
@@ -30,6 +31,7 @@ def _make_file(db, *, filehash, filename, is_duplicate=False, duplicate_of_id=No
         is_duplicate=is_duplicate,
         duplicate_of_id=duplicate_of_id,
         ocr_text=ocr_text,
+        embedding=embedding,
     )
     db.add(record)
     db.commit()
@@ -195,25 +197,24 @@ class TestGetFileDuplicates:
         assert data["duplicate_of"]["id"] == orig.id
 
     @pytest.mark.integration
-    @patch("app.utils.similarity.generate_embedding")
-    def test_near_duplicates_returned(self, mock_embed, client: TestClient, db_session):
+    def test_near_duplicates_returned(self, client: TestClient, db_session):
         """Near-duplicates found via embedding similarity should appear in results."""
+        embedding = json.dumps([1.0, 0.0, 0.0])
         target = _make_file(
             db_session,
             filehash="th1",
             filename="target.pdf",
             ocr_text="Invoice from Acme Corp for January services rendered",
+            embedding=embedding,
         )
         similar = _make_file(
             db_session,
             filehash="th2",  # different hash — same content (re-scan)
             filename="rescan.pdf",
             ocr_text="Invoice from Acme Corp for January services rendered",
+            embedding=embedding,
         )
 
-        # Same embedding → cosine similarity = 1.0
-        mock_embed.return_value = [1.0, 0.0, 0.0]
-
         response = client.get(f"/api/files/{target.id}/duplicates?near_duplicate_threshold=0.8")
         assert response.status_code == 200
         data = response.json()