feat(similarity): add similarity pairs dashboard, step tracking, and fix tests for pre-computed embeddings

- Add GET /api/similarity/pairs endpoint for corpus-wide pair discovery
- Add /similarity view route and similarity_dashboard.html template
- Add Similarity link to desktop and mobile nav menus
- Register compute_embedding as a tracked FileProcessingStep
- Update compute_embedding task with update_step_status calls
- Add compute_embedding to flow visualization in _compute_processing_flow
- Add backfill_missing_embeddings periodic beat task (every 5 min)
- Return clear message when embedding not yet computed in similar docs API
- Fix all tests to use pre-computed embeddings (no lazy API calls)
- Add tests for similarity pairs, backfill task, and embedding-not-computed

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/api/similarity.py

@@ -82,6 +82,19 @@ def get_similar_documents(
             "message": "No OCR text available for similarity comparison",
         }
 
+    # Check whether an embedding has been computed yet
+    if not file_record.embedding:
+        return {
+            "file_id": file_id,
+            "similar_documents": [],
+            "count": 0,
+            "message": (
+                "Embedding not yet computed for this file. "
+                "It will be generated automatically during processing or via the backfill task. "
+                "You can also trigger it manually with POST /api/files/{file_id}/compute-embedding."
+            ),
+        }
+
     try:
         from app.utils.similarity import find_similar_documents
 
@@ -331,3 +344,120 @@ def trigger_compute_all_embeddings(
         "status": "queued",
         "files_queued": queued,
     }
+
+
+@router.get("/similarity/pairs")
+@require_login
+def get_similarity_pairs(
+    request: Request,
+    db: DbSession,
+    threshold: float = Query(0.7, ge=0.0, le=1.0, description="Minimum similarity score for a pair"),
+    limit: int = Query(50, ge=1, le=200, description="Maximum number of pairs to return"),
+    page: int = Query(1, ge=1, description="Page number"),
+):
+    """Return pairs of documents with high similarity across the entire corpus.
+
+    Unlike the per-file ``/files/{id}/similar`` endpoint, this scans every
+    document that has a pre-computed embedding and returns **all** pairs
+    whose cosine similarity exceeds ``threshold``, sorted by descending
+    score.
+
+    To keep memory bounded the query loads only the columns needed for
+    scoring and streams results in chunks.
+
+    Response:
+    ```json
+    {
+      "pairs": [
+        {
+          "file_a": {"file_id": 1, "original_filename": "invoice_jan.pdf", ...},
+          "file_b": {"file_id": 5, "original_filename": "invoice_feb.pdf", ...},
+          "similarity_score": 0.94
+        }
+      ],
+      "total_pairs": 12,
+      "threshold": 0.7,
+      "page": 1,
+      "pages": 1,
+      "embedding_coverage": {"total_files": 120, "files_with_embedding": 95}
+    }
+    ```
+    """
+    from app.utils.similarity import cosine_similarity
+
+    # Load all files that have embeddings (columns only for efficiency)
+    rows = (
+        db.query(
+            FileRecord.id,
+            FileRecord.original_filename,
+            FileRecord.document_title,
+            FileRecord.mime_type,
+            FileRecord.created_at,
+            FileRecord.embedding,
+        )
+        .filter(
+            FileRecord.embedding.isnot(None),
+            FileRecord.embedding != "",
+        )
+        .order_by(FileRecord.id)
+        .all()
+    )
+
+    # Parse embeddings upfront
+    parsed: list[tuple] = []
+    for row in rows:
+        try:
+            vec = json.loads(row.embedding)
+            parsed.append((row, vec))
+        except (json.JSONDecodeError, TypeError):
+            continue
+
+    # Pairwise comparison (triangle: i < j avoids duplicating A↔B / B↔A)
+    all_pairs: list[dict] = []
+    for i in range(len(parsed)):
+        row_a, vec_a = parsed[i]
+        for j in range(i + 1, len(parsed)):
+            row_b, vec_b = parsed[j]
+            score = cosine_similarity(vec_a, vec_b)
+            if score >= threshold:
+                all_pairs.append(
+                    {
+                        "file_a": _row_to_dict(row_a),
+                        "file_b": _row_to_dict(row_b),
+                        "similarity_score": round(score, 4),
+                    }
+                )
+
+    # Sort by score descending
+    all_pairs.sort(key=lambda p: p["similarity_score"], reverse=True)
+
+    total_pairs = len(all_pairs)
+    total_pages = max(1, (total_pairs + limit - 1) // limit)
+    offset = (page - 1) * limit
+    page_pairs = all_pairs[offset : offset + limit]
+
+    total_files = db.query(FileRecord).count()
+
+    return {
+        "pairs": page_pairs,
+        "total_pairs": total_pairs,
+        "threshold": threshold,
+        "page": page,
+        "pages": total_pages,
+        "per_page": limit,
+        "embedding_coverage": {
+            "total_files": total_files,
+            "files_with_embedding": len(parsed),
+        },
+    }
+
+
+def _row_to_dict(row) -> dict:
+    """Serialise a column-only query row to a dict for JSON responses."""
+    return {
+        "file_id": row.id,
+        "original_filename": row.original_filename,
+        "document_title": row.document_title,
+        "mime_type": row.mime_type,
+        "created_at": row.created_at.isoformat() if row.created_at else None,
+    }

app/tasks/compute_embedding.py

@@ -6,12 +6,14 @@ access.
 """
 
 import logging
+from datetime import datetime, timezone
 
 from app.celery_app import celery
 from app.database import SessionLocal
 from app.models import FileRecord
 from app.tasks.retry_config import BaseTaskWithRetry
 from app.utils import log_task_progress
+from app.utils.step_manager import update_step_status
 
 logger = logging.getLogger(__name__)
 
@@ -47,6 +49,9 @@ def compute_document_embedding(self, file_id: int) -> dict:
             logger.warning("[%s] File %s not found, skipping embedding", task_id, file_id)
             return {"status": "skipped", "detail": "File not found"}
 
+        now = datetime.now(timezone.utc)
+        update_step_status(db, file_id, "compute_embedding", "in_progress", started_at=now)
+
         # Already has a cached embedding – nothing to do
         if file_record.embedding:
             logger.info("[%s] File %s already has a cached embedding", task_id, file_id)
@@ -57,6 +62,7 @@ def compute_document_embedding(self, file_id: int) -> dict:
                 "Embedding already cached",
                 file_id=file_id,
             )
+            update_step_status(db, file_id, "compute_embedding", "success", completed_at=now)
             return {"status": "skipped", "detail": "Embedding already cached"}
 
         if not file_record.ocr_text or not file_record.ocr_text.strip():
@@ -68,12 +74,14 @@ def compute_document_embedding(self, file_id: int) -> dict:
                 "No OCR text available",
                 file_id=file_id,
             )
+            update_step_status(db, file_id, "compute_embedding", "skipped", completed_at=now)
             return {"status": "skipped", "detail": "No OCR text available"}
 
         try:
             from app.utils.similarity import compute_and_store_embedding
 
             embedding = compute_and_store_embedding(db, file_record)
+            completed = datetime.now(timezone.utc)
             if embedding:
                 log_task_progress(
                     task_id,
@@ -82,6 +90,7 @@ def compute_document_embedding(self, file_id: int) -> dict:
                     f"Embedding computed ({len(embedding)} dimensions)",
                     file_id=file_id,
                 )
+                update_step_status(db, file_id, "compute_embedding", "success", completed_at=completed)
                 return {
                     "status": "success",
                     "detail": f"Embedding computed ({len(embedding)} dimensions)",
@@ -94,6 +103,14 @@ def compute_document_embedding(self, file_id: int) -> dict:
                     "Embedding computation returned None",
                     file_id=file_id,
                 )
+                update_step_status(
+                    db,
+                    file_id,
+                    "compute_embedding",
+                    "failure",
+                    error_message="Embedding computation returned None",
+                    completed_at=completed,
+                )
                 return {"status": "error", "detail": "Embedding computation returned None"}
         except Exception as exc:
             logger.exception("[%s] Embedding computation failed for file %s: %s", task_id, file_id, exc)
@@ -104,6 +121,14 @@ def compute_document_embedding(self, file_id: int) -> dict:
                 f"Exception: {exc}",
                 file_id=file_id,
             )
+            update_step_status(
+                db,
+                file_id,
+                "compute_embedding",
+                "failure",
+                error_message=str(exc),
+                completed_at=datetime.now(timezone.utc),
+            )
             return {"status": "error", "detail": str(exc)}
 
 

app/utils/step_manager.py

@@ -23,6 +23,7 @@ BASE_MAIN_PROCESSING_STEPS = [
     "embed_metadata_into_pdf",
     "finalize_document_storage",
     "send_to_all_destinations",
+    "compute_embedding",
 ]
 
 OPTIONAL_PROCESSING_STEPS = {

app/views/files.py

@@ -389,8 +389,12 @@ def _compute_processing_flow(logs):
         },
         "extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]},
         "embed_metadata_into_pdf": {"label": "Embed Metadata into PDF", "next": ["finalize_document_storage"]},
-        "finalize_document_storage": {"label": "Finalize & Queue Distribution", "next": ["send_to_all_destinations"]},
+        "finalize_document_storage": {
+            "label": "Finalize & Queue Distribution",
+            "next": ["send_to_all_destinations", "compute_embedding"],
+        },
         "send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True},
+        "compute_embedding": {"label": "Compute Embedding", "next": []},
     }
 
     # Filter out deduplication step if not enabled or if not showing it
@@ -815,3 +819,54 @@ def duplicates_page(
                 "error": str(e),
             },
         )
+
+
+@router.get("/similarity")
+@require_login
+def similarity_dashboard_page(
+    request: Request,
+    db: Session = Depends(get_db),
+):
+    """Render the corpus-wide similarity dashboard.
+
+    Passes the configured threshold and embedding coverage stats so the
+    template can display them immediately while the JS fetches the actual
+    pairs from the API asynchronously.
+    """
+    from app.config import settings
+    from app.models import FileRecord
+
+    try:
+        total_files = db.query(FileRecord).count()
+        files_with_embedding = (
+            db.query(FileRecord).filter(FileRecord.embedding.isnot(None), FileRecord.embedding != "").count()
+        )
+        files_with_ocr = db.query(FileRecord).filter(FileRecord.ocr_text.isnot(None), FileRecord.ocr_text != "").count()
+
+        return templates.TemplateResponse(
+            "similarity_dashboard.html",
+            {
+                "request": request,
+                "default_threshold": settings.near_duplicate_threshold,
+                "embedding_model": settings.embedding_model,
+                "total_files": total_files,
+                "files_with_embedding": files_with_embedding,
+                "files_with_ocr": files_with_ocr,
+                "files_missing_embedding": files_with_ocr - files_with_embedding,
+            },
+        )
+    except Exception as e:
+        logger.error(f"Error rendering similarity dashboard: {e}")
+        return templates.TemplateResponse(
+            "similarity_dashboard.html",
+            {
+                "request": request,
+                "default_threshold": 0.85,
+                "embedding_model": "text-embedding-3-small",
+                "total_files": 0,
+                "files_with_embedding": 0,
+                "files_with_ocr": 0,
+                "files_missing_embedding": 0,
+                "error": str(e),
+            },
+        )