feat(similarity): add similarity pairs dashboard, step tracking, and fix tests for pre-computed embeddings

- Add GET /api/similarity/pairs endpoint for corpus-wide pair discovery
- Add /similarity view route and similarity_dashboard.html template
- Add Similarity link to desktop and mobile nav menus
- Register compute_embedding as a tracked FileProcessingStep
- Update compute_embedding task with update_step_status calls
- Add compute_embedding to flow visualization in _compute_processing_flow
- Add backfill_missing_embeddings periodic beat task (every 5 min)
- Return clear message when embedding not yet computed in similar docs API
- Fix all tests to use pre-computed embeddings (no lazy API calls)
- Add tests for similarity pairs, backfill task, and embedding-not-computed

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/api/similarity.py

@@ -82,6 +82,19 @@ def get_similar_documents(
             "message": "No OCR text available for similarity comparison",
         }
 
+    # Check whether an embedding has been computed yet
+    if not file_record.embedding:
+        return {
+            "file_id": file_id,
+            "similar_documents": [],
+            "count": 0,
+            "message": (
+                "Embedding not yet computed for this file. "
+                "It will be generated automatically during processing or via the backfill task. "
+                "You can also trigger it manually with POST /api/files/{file_id}/compute-embedding."
+            ),
+        }
+
     try:
         from app.utils.similarity import find_similar_documents
 
@@ -331,3 +344,120 @@ def trigger_compute_all_embeddings(
         "status": "queued",
         "files_queued": queued,
     }
+
+
+@router.get("/similarity/pairs")
+@require_login
+def get_similarity_pairs(
+    request: Request,
+    db: DbSession,
+    threshold: float = Query(0.7, ge=0.0, le=1.0, description="Minimum similarity score for a pair"),
+    limit: int = Query(50, ge=1, le=200, description="Maximum number of pairs to return"),
+    page: int = Query(1, ge=1, description="Page number"),
+):
+    """Return pairs of documents with high similarity across the entire corpus.
+
+    Unlike the per-file ``/files/{id}/similar`` endpoint, this scans every
+    document that has a pre-computed embedding and returns **all** pairs
+    whose cosine similarity exceeds ``threshold``, sorted by descending
+    score.
+
+    To keep memory bounded the query loads only the columns needed for
+    scoring and streams results in chunks.
+
+    Response:
+    ```json
+    {
+      "pairs": [
+        {
+          "file_a": {"file_id": 1, "original_filename": "invoice_jan.pdf", ...},
+          "file_b": {"file_id": 5, "original_filename": "invoice_feb.pdf", ...},
+          "similarity_score": 0.94
+        }
+      ],
+      "total_pairs": 12,
+      "threshold": 0.7,
+      "page": 1,
+      "pages": 1,
+      "embedding_coverage": {"total_files": 120, "files_with_embedding": 95}
+    }
+    ```
+    """
+    from app.utils.similarity import cosine_similarity
+
+    # Load all files that have embeddings (columns only for efficiency)
+    rows = (
+        db.query(
+            FileRecord.id,
+            FileRecord.original_filename,
+            FileRecord.document_title,
+            FileRecord.mime_type,
+            FileRecord.created_at,
+            FileRecord.embedding,
+        )
+        .filter(
+            FileRecord.embedding.isnot(None),
+            FileRecord.embedding != "",
+        )
+        .order_by(FileRecord.id)
+        .all()
+    )
+
+    # Parse embeddings upfront
+    parsed: list[tuple] = []
+    for row in rows:
+        try:
+            vec = json.loads(row.embedding)
+            parsed.append((row, vec))
+        except (json.JSONDecodeError, TypeError):
+            continue
+
+    # Pairwise comparison (triangle: i < j avoids duplicating A↔B / B↔A)
+    all_pairs: list[dict] = []
+    for i in range(len(parsed)):
+        row_a, vec_a = parsed[i]
+        for j in range(i + 1, len(parsed)):
+            row_b, vec_b = parsed[j]
+            score = cosine_similarity(vec_a, vec_b)
+            if score >= threshold:
+                all_pairs.append(
+                    {
+                        "file_a": _row_to_dict(row_a),
+                        "file_b": _row_to_dict(row_b),
+                        "similarity_score": round(score, 4),
+                    }
+                )
+
+    # Sort by score descending
+    all_pairs.sort(key=lambda p: p["similarity_score"], reverse=True)
+
+    total_pairs = len(all_pairs)
+    total_pages = max(1, (total_pairs + limit - 1) // limit)
+    offset = (page - 1) * limit
+    page_pairs = all_pairs[offset : offset + limit]
+
+    total_files = db.query(FileRecord).count()
+
+    return {
+        "pairs": page_pairs,
+        "total_pairs": total_pairs,
+        "threshold": threshold,
+        "page": page,
+        "pages": total_pages,
+        "per_page": limit,
+        "embedding_coverage": {
+            "total_files": total_files,
+            "files_with_embedding": len(parsed),
+        },
+    }
+
+
+def _row_to_dict(row) -> dict:
+    """Serialise a column-only query row to a dict for JSON responses."""
+    return {
+        "file_id": row.id,
+        "original_filename": row.original_filename,
+        "document_title": row.document_title,
+        "mime_type": row.mime_type,
+        "created_at": row.created_at.isoformat() if row.created_at else None,
+    }

app/tasks/compute_embedding.py

@@ -6,12 +6,14 @@ access.
 """
 
 import logging
+from datetime import datetime, timezone
 
 from app.celery_app import celery
 from app.database import SessionLocal
 from app.models import FileRecord
 from app.tasks.retry_config import BaseTaskWithRetry
 from app.utils import log_task_progress
+from app.utils.step_manager import update_step_status
 
 logger = logging.getLogger(__name__)
 
@@ -47,6 +49,9 @@ def compute_document_embedding(self, file_id: int) -> dict:
             logger.warning("[%s] File %s not found, skipping embedding", task_id, file_id)
             return {"status": "skipped", "detail": "File not found"}
 
+        now = datetime.now(timezone.utc)
+        update_step_status(db, file_id, "compute_embedding", "in_progress", started_at=now)
+
         # Already has a cached embedding – nothing to do
         if file_record.embedding:
             logger.info("[%s] File %s already has a cached embedding", task_id, file_id)
@@ -57,6 +62,7 @@ def compute_document_embedding(self, file_id: int) -> dict:
                 "Embedding already cached",
                 file_id=file_id,
             )
+            update_step_status(db, file_id, "compute_embedding", "success", completed_at=now)
             return {"status": "skipped", "detail": "Embedding already cached"}
 
         if not file_record.ocr_text or not file_record.ocr_text.strip():
@@ -68,12 +74,14 @@ def compute_document_embedding(self, file_id: int) -> dict:
                 "No OCR text available",
                 file_id=file_id,
             )
+            update_step_status(db, file_id, "compute_embedding", "skipped", completed_at=now)
             return {"status": "skipped", "detail": "No OCR text available"}
 
         try:
             from app.utils.similarity import compute_and_store_embedding
 
             embedding = compute_and_store_embedding(db, file_record)
+            completed = datetime.now(timezone.utc)
             if embedding:
                 log_task_progress(
                     task_id,
@@ -82,6 +90,7 @@ def compute_document_embedding(self, file_id: int) -> dict:
                     f"Embedding computed ({len(embedding)} dimensions)",
                     file_id=file_id,
                 )
+                update_step_status(db, file_id, "compute_embedding", "success", completed_at=completed)
                 return {
                     "status": "success",
                     "detail": f"Embedding computed ({len(embedding)} dimensions)",
@@ -94,6 +103,14 @@ def compute_document_embedding(self, file_id: int) -> dict:
                     "Embedding computation returned None",
                     file_id=file_id,
                 )
+                update_step_status(
+                    db,
+                    file_id,
+                    "compute_embedding",
+                    "failure",
+                    error_message="Embedding computation returned None",
+                    completed_at=completed,
+                )
                 return {"status": "error", "detail": "Embedding computation returned None"}
         except Exception as exc:
             logger.exception("[%s] Embedding computation failed for file %s: %s", task_id, file_id, exc)
@@ -104,6 +121,14 @@ def compute_document_embedding(self, file_id: int) -> dict:
                 f"Exception: {exc}",
                 file_id=file_id,
             )
+            update_step_status(
+                db,
+                file_id,
+                "compute_embedding",
+                "failure",
+                error_message=str(exc),
+                completed_at=datetime.now(timezone.utc),
+            )
             return {"status": "error", "detail": str(exc)}
 
 

app/utils/step_manager.py

@@ -23,6 +23,7 @@ BASE_MAIN_PROCESSING_STEPS = [
     "embed_metadata_into_pdf",
     "finalize_document_storage",
     "send_to_all_destinations",
+    "compute_embedding",
 ]
 
 OPTIONAL_PROCESSING_STEPS = {

app/views/files.py

@@ -389,8 +389,12 @@ def _compute_processing_flow(logs):
         },
         "extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]},
         "embed_metadata_into_pdf": {"label": "Embed Metadata into PDF", "next": ["finalize_document_storage"]},
-        "finalize_document_storage": {"label": "Finalize & Queue Distribution", "next": ["send_to_all_destinations"]},
+        "finalize_document_storage": {
+            "label": "Finalize & Queue Distribution",
+            "next": ["send_to_all_destinations", "compute_embedding"],
+        },
         "send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True},
+        "compute_embedding": {"label": "Compute Embedding", "next": []},
     }
 
     # Filter out deduplication step if not enabled or if not showing it
@@ -815,3 +819,54 @@ def duplicates_page(
                 "error": str(e),
             },
         )
+
+
+@router.get("/similarity")
+@require_login
+def similarity_dashboard_page(
+    request: Request,
+    db: Session = Depends(get_db),
+):
+    """Render the corpus-wide similarity dashboard.
+
+    Passes the configured threshold and embedding coverage stats so the
+    template can display them immediately while the JS fetches the actual
+    pairs from the API asynchronously.
+    """
+    from app.config import settings
+    from app.models import FileRecord
+
+    try:
+        total_files = db.query(FileRecord).count()
+        files_with_embedding = (
+            db.query(FileRecord).filter(FileRecord.embedding.isnot(None), FileRecord.embedding != "").count()
+        )
+        files_with_ocr = db.query(FileRecord).filter(FileRecord.ocr_text.isnot(None), FileRecord.ocr_text != "").count()
+
+        return templates.TemplateResponse(
+            "similarity_dashboard.html",
+            {
+                "request": request,
+                "default_threshold": settings.near_duplicate_threshold,
+                "embedding_model": settings.embedding_model,
+                "total_files": total_files,
+                "files_with_embedding": files_with_embedding,
+                "files_with_ocr": files_with_ocr,
+                "files_missing_embedding": files_with_ocr - files_with_embedding,
+            },
+        )
+    except Exception as e:
+        logger.error(f"Error rendering similarity dashboard: {e}")
+        return templates.TemplateResponse(
+            "similarity_dashboard.html",
+            {
+                "request": request,
+                "default_threshold": 0.85,
+                "embedding_model": "text-embedding-3-small",
+                "total_files": 0,
+                "files_with_embedding": 0,
+                "files_with_ocr": 0,
+                "files_missing_embedding": 0,
+                "error": str(e),
+            },
+        )

frontend/templates/base.html

@@ -104,6 +104,9 @@
                   <a href="/duplicates" role="menuitem" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
                     <i class="fas fa-clone w-4 mr-2 text-orange-500" aria-hidden="true"></i> Duplicates
                   </a>
+                  <a href="/similarity" role="menuitem" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
+                    <i class="fas fa-project-diagram w-4 mr-2 text-blue-500" aria-hidden="true"></i> Similarity
+                  </a>
                   <a href="/admin/queue" role="menuitem" class="flex items-center px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
                     <i class="fas fa-stream w-4 mr-2 text-blue-500" aria-hidden="true"></i> Queue Monitor
                   </a>
@@ -184,6 +187,9 @@
                   <a href="/duplicates" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50">
                     <i class="fas fa-clone mr-2 text-orange-400" aria-hidden="true"></i> Duplicates
                   </a>
+                  <a href="/similarity" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50">
+                    <i class="fas fa-project-diagram mr-2 text-blue-400" aria-hidden="true"></i> Similarity
+                  </a>
                   <a href="/admin/queue" class="block px-3 py-3 rounded-md text-base font-medium text-gray-700 hover:text-gray-900 hover:bg-gray-50">
                     <i class="fas fa-stream mr-2 text-blue-400" aria-hidden="true"></i> Queue Monitor
                   </a>

frontend/templates/similarity_dashboard.html

@@ -0,0 +1,368 @@
+{% extends "base.html" %}
+{% block title %}Document Similarity - DocuElevate{% endblock %}
+
+{% block head_extra %}
+<style>
+  .sim-container { max-width: 1100px; margin: 0 auto; }
+
+  /* Stats bar */
+  .stats-bar {
+    display: flex; flex-wrap: wrap; gap: 1rem; margin-bottom: 1.5rem;
+  }
+  .stat-card {
+    flex: 1; min-width: 140px; background: white; border-radius: 0.5rem;
+    box-shadow: 0 1px 3px rgba(0,0,0,0.1); padding: 1rem 1.25rem;
+    text-align: center;
+  }
+  .stat-value { font-size: 1.5rem; font-weight: 700; }
+  .stat-label { font-size: 0.8rem; color: #6b7280; margin-top: 0.25rem; }
+
+  /* Pair cards */
+  .pair-card {
+    background: white; border-radius: 0.5rem;
+    box-shadow: 0 1px 3px rgba(0,0,0,0.1);
+    margin-bottom: 1rem; overflow: hidden;
+  }
+  .pair-header {
+    padding: 0.75rem 1.25rem; border-bottom: 1px solid #e5e7eb;
+    display: flex; align-items: center; gap: 0.75rem;
+    background: #f9fafb;
+  }
+  .pair-body {
+    display: grid; grid-template-columns: 1fr auto 1fr; gap: 0;
+  }
+  .pair-file {
+    padding: 0.75rem 1.25rem;
+  }
+  .pair-file:first-child { border-right: 1px solid #f3f4f6; }
+  .pair-connector {
+    display: flex; align-items: center; justify-content: center;
+    padding: 0 0.5rem; color: #9ca3af; font-size: 1.25rem;
+  }
+  .pair-filename {
+    font-weight: 600; color: #1f2937; word-break: break-all;
+    white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
+  }
+  .pair-meta { font-size: 0.8rem; color: #6b7280; }
+  .score-badge {
+    display: inline-flex; align-items: center; gap: 0.3rem;
+    font-size: 0.85rem; font-weight: 700; padding: 0.25rem 0.75rem;
+    border-radius: 9999px;
+  }
+  .score-high   { background: #fee2e2; color: #991b1b; }
+  .score-medium { background: #fef3c7; color: #92400e; }
+  .score-low    { background: #e5e7eb; color: #374151; }
+
+  /* Controls */
+  .controls {
+    background: white; border-radius: 0.5rem;
+    box-shadow: 0 1px 3px rgba(0,0,0,0.1);
+    padding: 1rem 1.25rem; margin-bottom: 1.5rem;
+  }
+  .controls-form {
+    display: flex; gap: 0.75rem; flex-wrap: wrap; align-items: flex-end;
+  }
+  .controls-form label { font-size: 0.8rem; font-weight: 600; color: #374151; }
+  .controls-form input, .controls-form select {
+    padding: 0.4rem 0.6rem; border: 1px solid #d1d5db;
+    border-radius: 0.375rem; font-size: 0.85rem;
+  }
+  .controls-form input[type=number] { width: 90px; }
+
+  /* Pagination */
+  .pagination { display: flex; gap: 0.5rem; justify-content: center; margin-top: 1.5rem; }
+  .page-btn {
+    padding: 0.4rem 0.75rem; border: 1px solid #d1d5db;
+    border-radius: 0.375rem; font-size: 0.85rem; cursor: pointer;
+    background: white; color: #374151;
+  }
+  .page-btn:hover:not(:disabled) { background: #f3f4f6; }
+  .page-btn:disabled { opacity: 0.4; cursor: default; }
+  .page-btn.current { background: #2563eb; color: white; border-color: #2563eb; }
+
+  .empty-state {
+    text-align: center; padding: 3rem; color: #6b7280;
+  }
+  .empty-state i { font-size: 3rem; margin-bottom: 0.75rem; display: block; }
+
+  @media (max-width: 640px) {
+    .pair-body { grid-template-columns: 1fr; }
+    .pair-file:first-child { border-right: none; border-bottom: 1px solid #f3f4f6; }
+    .pair-connector { padding: 0.25rem 0; }
+  }
+</style>
+{% endblock %}
+
+{% block content %}
+<main id="main-content" class="sim-container px-4 py-8">
+  <h1 class="text-2xl font-bold mb-2">
+    <i class="fas fa-project-diagram text-blue-500" aria-hidden="true"></i>
+    Document Similarity
+  </h1>
+  <p class="text-gray-500 mb-6 text-sm">
+    Pairs of documents with high semantic similarity, ranked by score.
+    Embeddings are computed during document ingestion; a background task
+    also backfills any files that were processed before this feature was enabled.
+  </p>
+
+  <!-- Embedding coverage stats -->
+  <div class="stats-bar">
+    <div class="stat-card">
+      <div class="stat-value">{{ total_files }}</div>
+      <div class="stat-label">Total Files</div>
+    </div>
+    <div class="stat-card">
+      <div class="stat-value" style="color: #2563eb;">{{ files_with_embedding }}</div>
+      <div class="stat-label">With Embedding</div>
+    </div>
+    <div class="stat-card">
+      <div class="stat-value" style="color: {% if files_missing_embedding > 0 %}#d97706{% else %}#059669{% endif %};">
+        {{ files_missing_embedding }}
+      </div>
+      <div class="stat-label">Missing Embedding</div>
+    </div>
+    <div class="stat-card">
+      <div class="stat-value text-sm" style="word-break: break-all;">{{ embedding_model }}</div>
+      <div class="stat-label">Embedding Model</div>
+    </div>
+  </div>
+
+  {% if files_missing_embedding > 0 %}
+  <div style="background: #fffff0; border: 1px solid #fefcbf; border-radius: 0.5rem; padding: 0.75rem 1rem; margin-bottom: 1.5rem; font-size: 0.85rem; color: #975a16;">
+    <i class="fas fa-exclamation-triangle" aria-hidden="true"></i>
+    <strong>{{ files_missing_embedding }}</strong> file(s) have OCR text but no embedding yet.
+    The background task will compute them automatically every 5 minutes, or you can
+    <button onclick="triggerBackfill()" id="backfill-btn"
+            style="background: #d97706; color: white; border: none; padding: 0.2rem 0.6rem; border-radius: 0.25rem; cursor: pointer; font-size: 0.8rem;"
+            aria-label="Trigger embedding computation for all files missing embeddings">
+      trigger it now
+    </button>.
+  </div>
+  {% endif %}
+
+  <!-- Controls -->
+  <div class="controls">
+    <form id="pairsForm" onsubmit="loadPairs(event)" class="controls-form">
+      <div class="flex flex-col gap-1">
+        <label for="pairThreshold">Min. similarity</label>
+        <input type="number" id="pairThreshold" min="0" max="1" step="0.05"
+               value="{{ default_threshold }}" style="min-height:44px;">
+      </div>
+      <div class="flex flex-col gap-1">
+        <label for="pairLimit">Per page</label>
+        <select id="pairLimit" style="min-height:44px;">
+          <option value="25">25</option>
+          <option value="50" selected>50</option>
+          <option value="100">100</option>
+        </select>
+      </div>
+      <button type="submit"
+              style="min-height:44px; background-color: #2563eb; color: white; font-weight: 700;
+                     padding: 0 1.25rem; border: none; border-radius: 0.5rem; cursor: pointer;">
+        <i class="fas fa-search" aria-hidden="true"></i> Find Pairs
+      </button>
+    </form>
+  </div>
+
+  <!-- Results area -->
+  <div id="pairs-loading" style="text-align: center; padding: 2rem; color: #718096;">
+    <i class="fas fa-spinner fa-spin" aria-hidden="true" style="font-size: 1.5rem; margin-bottom: 0.5rem;"></i>
+    <p>Scanning for similar document pairs…</p>
+  </div>
+  <div id="pairs-content" style="display: none;" aria-live="polite"></div>
+  <div id="pairs-empty" style="display: none;" class="empty-state">
+    <i class="fas fa-check-circle text-green-400" aria-hidden="true"></i>
+    <p class="font-semibold text-lg text-gray-700">No similar pairs found</p>
+    <p class="text-sm mt-1" id="pairs-empty-detail">
+      No document pairs exceed the similarity threshold.
+    </p>
+  </div>
+  <div id="pairs-error" style="display: none;" class="empty-state">
+    <i class="fas fa-exclamation-triangle text-red-400" aria-hidden="true"></i>
+    <p class="font-semibold text-lg text-gray-700" id="pairs-error-msg">Failed to load pairs.</p>
+  </div>
+
+  <!-- Pagination -->
+  <nav id="pairs-pagination" class="pagination" style="display: none;" aria-label="Similarity pairs pagination"></nav>
+</main>
+{% endblock %}
+
+{% block scripts %}
+<script>
+  let currentPage = 1;
+
+  function getThreshold() {
+    return parseFloat(document.getElementById('pairThreshold').value) || 0.7;
+  }
+  function getLimit() {
+    return parseInt(document.getElementById('pairLimit').value) || 50;
+  }
+
+  function loadPairs(e) {
+    if (e) e.preventDefault();
+    fetchPairs(1);
+  }
+
+  async function fetchPairs(page) {
+    currentPage = page;
+    const threshold = getThreshold();
+    const limit = getLimit();
+
+    const loadingDiv = document.getElementById('pairs-loading');
+    const contentDiv = document.getElementById('pairs-content');
+    const emptyDiv = document.getElementById('pairs-empty');
+    const errorDiv = document.getElementById('pairs-error');
+    const pagDiv = document.getElementById('pairs-pagination');
+
+    loadingDiv.style.display = 'block';
+    contentDiv.style.display = 'none';
+    emptyDiv.style.display = 'none';
+    errorDiv.style.display = 'none';
+    pagDiv.style.display = 'none';
+
+    try {
+      const url = `/api/similarity/pairs?threshold=${threshold}&limit=${limit}&page=${page}`;
+      const resp = await fetch(url);
+      if (!resp.ok) {
+        const err = await resp.json().catch(() => ({}));
+        throw new Error(err.detail || resp.statusText);
+      }
+
+      const data = await resp.json();
+      loadingDiv.style.display = 'none';
+
+      if (!data.pairs || data.pairs.length === 0) {
+        emptyDiv.style.display = 'block';
+        const detail = document.getElementById('pairs-empty-detail');
+        if (data.embedding_coverage && data.embedding_coverage.files_with_embedding === 0) {
+          detail.textContent = 'No files have embeddings yet. Wait for the background task or trigger it manually.';
+        } else {
+          detail.textContent = `No document pairs exceed the ${(threshold * 100).toFixed(0)}% similarity threshold.`;
+        }
+        return;
+      }
+
+      renderPairs(data);
+      contentDiv.style.display = 'block';
+      renderPagination(data);
+    } catch (err) {
+      loadingDiv.style.display = 'none';
+      errorDiv.style.display = 'block';
+      document.getElementById('pairs-error-msg').textContent = 'Failed to load pairs: ' + err.message;
+    }
+  }
+
+  function renderPairs(data) {
+    const container = document.getElementById('pairs-content');
+    let html = `<p class="text-sm text-gray-500 mb-3">
+      Found <strong>${data.total_pairs}</strong> pair(s) above
+      ${(data.threshold * 100).toFixed(0)}% similarity
+      (${data.embedding_coverage.files_with_embedding} of ${data.embedding_coverage.total_files} files have embeddings).
+    </p>`;
+
+    for (const pair of data.pairs) {
+      const pct = Math.round(pair.similarity_score * 100);
+      const scoreClass = pct >= 90 ? 'score-high' : pct >= 75 ? 'score-medium' : 'score-low';
+      const titleA = pair.file_a.document_title || pair.file_a.original_filename || 'Untitled';
+      const titleB = pair.file_b.document_title || pair.file_b.original_filename || 'Untitled';
+      const fnA = pair.file_a.original_filename || '(unnamed)';
+      const fnB = pair.file_b.original_filename || '(unnamed)';
+      const dateA = pair.file_a.created_at ? pair.file_a.created_at.substring(0, 10) : '';
+      const dateB = pair.file_b.created_at ? pair.file_b.created_at.substring(0, 10) : '';
+
+      html += `
+      <div class="pair-card">
+        <div class="pair-header">
+          <i class="fas fa-link text-blue-400" aria-hidden="true"></i>
+          <span class="score-badge ${scoreClass}">${pct}% match</span>
+        </div>
+        <div class="pair-body">
+          <div class="pair-file">
+            <div class="pair-filename" title="${escapeAttr(titleA)}">
+              <a href="/files/${pair.file_a.file_id}/detail" class="hover:text-blue-600">
+                ${escapeHtml(titleA)}
+              </a>
+            </div>
+            <div class="pair-meta">
+              #${pair.file_a.file_id} · ${escapeHtml(fnA)}${dateA ? ' · ' + dateA : ''}
+            </div>
+          </div>
+          <div class="pair-connector" aria-hidden="true">
+            <i class="fas fa-arrows-alt-h"></i>
+          </div>
+          <div class="pair-file">
+            <div class="pair-filename" title="${escapeAttr(titleB)}">
+              <a href="/files/${pair.file_b.file_id}/detail" class="hover:text-blue-600">
+                ${escapeHtml(titleB)}
+              </a>
+            </div>
+            <div class="pair-meta">
+              #${pair.file_b.file_id} · ${escapeHtml(fnB)}${dateB ? ' · ' + dateB : ''}
+            </div>
+          </div>
+        </div>
+      </div>`;
+    }
+
+    container.innerHTML = html;
+  }
+
+  function renderPagination(data) {
+    const nav = document.getElementById('pairs-pagination');
+    if (data.pages <= 1) { nav.style.display = 'none'; return; }
+    nav.style.display = 'flex';
+
+    let html = `<button class="page-btn" onclick="fetchPairs(${data.page - 1})"
+                  ${data.page <= 1 ? 'disabled' : ''} aria-label="Previous page">
+                  <i class="fas fa-chevron-left" aria-hidden="true"></i>
+                </button>`;
+
+    for (let p = 1; p <= data.pages; p++) {
+      html += `<button class="page-btn ${p === data.page ? 'current' : ''}"
+                onclick="fetchPairs(${p})" aria-label="Page ${p}"
+                ${p === data.page ? 'aria-current="page"' : ''}>${p}</button>`;
+    }
+
+    html += `<button class="page-btn" onclick="fetchPairs(${data.page + 1})"
+               ${data.page >= data.pages ? 'disabled' : ''} aria-label="Next page">
+               <i class="fas fa-chevron-right" aria-hidden="true"></i>
+             </button>`;
+
+    nav.innerHTML = html;
+  }
+
+  async function triggerBackfill() {
+    const btn = document.getElementById('backfill-btn');
+    btn.disabled = true;
+    btn.textContent = 'Queuing…';
+    try {
+      const resp = await fetch('/api/diagnostic/compute-all-embeddings', { method: 'POST' });
+      const data = await resp.json();
+      btn.textContent = `Queued ${data.files_queued} file(s)`;
+      btn.style.backgroundColor = '#059669';
+    } catch (err) {
+      btn.textContent = 'Failed';
+      btn.style.backgroundColor = '#dc2626';
+    }
+    setTimeout(() => {
+      btn.disabled = false;
+      btn.textContent = 'trigger it now';
+      btn.style.backgroundColor = '#d97706';
+    }, 5000);
+  }
+
+  function escapeHtml(str) {
+    return String(str)
+      .replace(/&/g, '&amp;').replace(/</g, '&lt;')
+      .replace(/>/g, '&gt;').replace(/"/g, '&quot;');
+  }
+  function escapeAttr(str) {
+    return String(str).replace(/"/g, '&quot;').replace(/'/g, '&#39;');
+  }
+
+  // Load pairs on page load
+  document.addEventListener('DOMContentLoaded', function() {
+    fetchPairs(1);
+  });
+</script>
+{% endblock %}

tests/test_similarity.py

@@ -108,18 +108,23 @@ class TestFindSimilarDocuments:
         assert result == []
 
     @pytest.mark.unit
-    @patch("app.utils.similarity.generate_embedding")
+    def test_finds_similar_documents(self, db_session):
-    def test_finds_similar_documents(self, mock_embed, db_session):
+        """Should find similar documents based on pre-computed embedding similarity."""
-        """Should find similar documents based on embedding similarity."""
+        # Pre-computed embeddings that reflect similarity
-        # Create a target file with OCR text
+        target_embedding = [1.0, 0.0, 0.0]
+        similar_embedding = [0.95, 0.05, 0.0]
+        different_embedding = [0.0, 0.0, 1.0]
+
+        # Create a target file with pre-computed embedding
         target = FileRecord(
             filehash="hash1",
             local_filename="/tmp/target.pdf",
             file_size=1024,
             original_filename="target.pdf",
             ocr_text="This is an invoice from Amazon for January 2026",
+            embedding=json.dumps(target_embedding),
         )
-        # Create a similar file
+        # Create a similar file with pre-computed embedding
         similar = FileRecord(
             filehash="hash2",
             local_filename="/tmp/similar.pdf",
@@ -128,8 +133,9 @@ class TestFindSimilarDocuments:
             ocr_text="This is an invoice from Amazon for February 2026",
             document_title="Amazon Invoice Feb",
             mime_type="application/pdf",
+            embedding=json.dumps(similar_embedding),
         )
-        # Create a different file
+        # Create a different file with pre-computed embedding
         different = FileRecord(
             filehash="hash3",
             local_filename="/tmp/different.pdf",
@@ -138,26 +144,12 @@ class TestFindSimilarDocuments:
             ocr_text="Recipe for chocolate cake with detailed instructions",
             document_title="Chocolate Cake Recipe",
             mime_type="application/pdf",
+            embedding=json.dumps(different_embedding),
         )
 
         db_session.add_all([target, similar, different])
         db_session.commit()
 
-        # Mock embeddings that reflect similarity
-        target_embedding = [1.0, 0.0, 0.0]
-        similar_embedding = [0.95, 0.05, 0.0]
-        different_embedding = [0.0, 0.0, 1.0]
-
-        def mock_embed_side_effect(text):
-            if "January" in text or "invoice" in text.lower()[:30]:
-                return target_embedding
-            elif "February" in text:
-                return similar_embedding
-            else:
-                return different_embedding
-
-        mock_embed.side_effect = mock_embed_side_effect
-
         result = find_similar_documents(db_session, file_id=target.id, threshold=0.3)
 
         assert len(result) == 1
@@ -166,8 +158,7 @@ class TestFindSimilarDocuments:
         assert result[0]["original_filename"] == "similar.pdf"
 
     @pytest.mark.unit
-    @patch("app.utils.similarity.generate_embedding")
+    def test_respects_threshold(self, db_session):
-    def test_respects_threshold(self, mock_embed, db_session):
         """Should filter out documents below the threshold."""
         target = FileRecord(
             filehash="hash1",
@@ -175,6 +166,7 @@ class TestFindSimilarDocuments:
             file_size=100,
             original_filename="target.pdf",
             ocr_text="target text",
+            embedding=json.dumps([1.0, 0.0]),
         )
         candidate = FileRecord(
             filehash="hash2",
@@ -182,26 +174,25 @@ class TestFindSimilarDocuments:
             file_size=100,
             original_filename="candidate.pdf",
             ocr_text="different text",
+            embedding=json.dumps([0.1, 0.99]),
         )
         db_session.add_all([target, candidate])
         db_session.commit()
 
-        # Return nearly orthogonal vectors -> low similarity
-        mock_embed.side_effect = lambda text: [1.0, 0.0] if "target" in text else [0.1, 0.99]
-
         result = find_similar_documents(db_session, file_id=target.id, threshold=0.9)
         assert len(result) == 0
 
     @pytest.mark.unit
-    @patch("app.utils.similarity.generate_embedding")
+    def test_respects_limit(self, db_session):
-    def test_respects_limit(self, mock_embed, db_session):
         """Should respect the limit parameter."""
+        embedding = [1.0, 0.0, 0.0]
         target = FileRecord(
             filehash="hash0",
             local_filename="/tmp/t.pdf",
             file_size=100,
             original_filename="target.pdf",
             ocr_text="target text",
+            embedding=json.dumps(embedding),
         )
         db_session.add(target)
 
@@ -212,12 +203,11 @@ class TestFindSimilarDocuments:
                 file_size=100,
                 original_filename=f"candidate_{i}.pdf",
                 ocr_text=f"similar text {i}",
+                embedding=json.dumps(embedding),
             )
             db_session.add(f)
         db_session.commit()
 
-        mock_embed.return_value = [1.0, 0.0, 0.0]
-
         result = find_similar_documents(db_session, file_id=target.id, limit=2, threshold=0.0)
         assert len(result) <= 2
 
@@ -286,15 +276,16 @@ class TestSimilarDocumentsAPI:
         assert "message" in data
 
     @pytest.mark.integration
-    @patch("app.utils.similarity.generate_embedding")
+    def test_returns_similar_documents(self, client: TestClient, db_session):
-    def test_returns_similar_documents(self, mock_embed, client: TestClient, db_session):
         """Should return similar documents with scores."""
+        embedding = [1.0, 0.0, 0.0]
         target = FileRecord(
             filehash="hash1",
             local_filename="/tmp/target.pdf",
             file_size=1024,
             original_filename="target.pdf",
             ocr_text="Invoice from Amazon January 2026",
+            embedding=json.dumps(embedding),
         )
         similar = FileRecord(
             filehash="hash2",
@@ -304,12 +295,11 @@ class TestSimilarDocumentsAPI:
             ocr_text="Invoice from Amazon February 2026",
             document_title="Amazon Invoice",
             mime_type="application/pdf",
+            embedding=json.dumps(embedding),
         )
         db_session.add_all([target, similar])
         db_session.commit()
 
-        mock_embed.return_value = [1.0, 0.0, 0.0]
-
         response = client.get(f"/api/files/{target.id}/similar")
         assert response.status_code == 200
         data = response.json()
@@ -324,15 +314,16 @@ class TestSimilarDocumentsAPI:
         assert "original_filename" in doc
 
     @pytest.mark.integration
-    @patch("app.utils.similarity.generate_embedding")
+    def test_query_parameters(self, client: TestClient, db_session):
-    def test_query_parameters(self, mock_embed, client: TestClient, db_session):
         """Should respect limit and threshold query parameters."""
+        embedding = [1.0, 0.0]
         target = FileRecord(
             filehash="hash1",
             local_filename="/tmp/t.pdf",
             file_size=100,
             original_filename="t.pdf",
             ocr_text="test",
+            embedding=json.dumps(embedding),
         )
         db_session.add(target)
 
@@ -343,12 +334,11 @@ class TestSimilarDocumentsAPI:
                 file_size=100,
                 original_filename=f"c{i}.pdf",
                 ocr_text=f"text {i}",
+                embedding=json.dumps(embedding),
             )
             db_session.add(f)
         db_session.commit()
 
-        mock_embed.return_value = [1.0, 0.0]
-
         response = client.get(f"/api/files/{target.id}/similar?limit=2&threshold=0.0")
         assert response.status_code == 200
         data = response.json()
@@ -403,15 +393,36 @@ class TestSimilarDocumentsAPI:
         assert data["count"] == 0
 
     @pytest.mark.integration
-    @patch("app.utils.similarity.generate_embedding")
+    def test_embedding_not_computed_message(self, client: TestClient, db_session):
-    def test_response_structure(self, mock_embed, client: TestClient, db_session):
+        """Should return a message when OCR text exists but no embedding yet."""
+        file_record = FileRecord(
+            filehash="noembhash",
+            local_filename="/tmp/noemb.pdf",
+            file_size=100,
+            original_filename="noemb.pdf",
+            ocr_text="Some OCR text content",
+            embedding=None,
+        )
+        db_session.add(file_record)
+        db_session.commit()
+
+        response = client.get(f"/api/files/{file_record.id}/similar")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["count"] == 0
+        assert "message" in data
+        assert "not yet computed" in data["message"].lower()
+
+    def test_response_structure(self, client: TestClient, db_session):
         """Should return proper response structure for each similar document."""
+        embedding = [1.0, 0.0]
         target = FileRecord(
             filehash="h1",
             local_filename="/tmp/t.pdf",
             file_size=100,
             original_filename="target.pdf",
             ocr_text="Some text content here",
+            embedding=json.dumps(embedding),
         )
         other = FileRecord(
             filehash="h2",
@@ -421,12 +432,11 @@ class TestSimilarDocumentsAPI:
             ocr_text="Some similar text content",
             document_title="Other Doc",
             mime_type="application/pdf",
+            embedding=json.dumps(embedding),
         )
         db_session.add_all([target, other])
         db_session.commit()
 
-        mock_embed.return_value = [1.0, 0.0]
-
         response = client.get(f"/api/files/{target.id}/similar")
         assert response.status_code == 200
         data = response.json()
@@ -829,3 +839,176 @@ class TestComputeDocumentEmbeddingTask:
 
         assert result["status"] == "skipped"
         assert "already cached" in result["detail"]
+
+
+# ---------------------------------------------------------------------------
+# Tests for similarity pairs endpoint
+# ---------------------------------------------------------------------------
+
+
+class TestSimilarityPairsAPI:
+    """Integration tests for GET /api/similarity/pairs."""
+
+    @pytest.mark.integration
+    def test_empty_database(self, client: TestClient):
+        """Should return zero pairs on empty database."""
+        response = client.get("/api/similarity/pairs")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["total_pairs"] == 0
+        assert data["pairs"] == []
+        assert "embedding_coverage" in data
+
+    @pytest.mark.integration
+    def test_finds_similar_pairs(self, client: TestClient, db_session):
+        """Should find and return pairs of similar files."""
+        emb_a = [1.0, 0.0, 0.0]
+        emb_b = [0.98, 0.02, 0.0]  # Very similar to A
+        emb_c = [0.0, 0.0, 1.0]  # Different from A and B
+
+        f1 = FileRecord(
+            filehash="pairA",
+            local_filename="/tmp/pA.pdf",
+            file_size=100,
+            original_filename="fileA.pdf",
+            ocr_text="text A",
+            embedding=json.dumps(emb_a),
+        )
+        f2 = FileRecord(
+            filehash="pairB",
+            local_filename="/tmp/pB.pdf",
+            file_size=100,
+            original_filename="fileB.pdf",
+            ocr_text="text B",
+            embedding=json.dumps(emb_b),
+        )
+        f3 = FileRecord(
+            filehash="pairC",
+            local_filename="/tmp/pC.pdf",
+            file_size=100,
+            original_filename="fileC.pdf",
+            ocr_text="text C",
+            embedding=json.dumps(emb_c),
+        )
+        db_session.add_all([f1, f2, f3])
+        db_session.commit()
+
+        response = client.get("/api/similarity/pairs?threshold=0.9")
+        assert response.status_code == 200
+        data = response.json()
+
+        # Only A-B pair should be above 0.9
+        assert data["total_pairs"] == 1
+        pair = data["pairs"][0]
+        assert pair["similarity_score"] > 0.9
+        pair_ids = {pair["file_a"]["file_id"], pair["file_b"]["file_id"]}
+        assert pair_ids == {f1.id, f2.id}
+
+    @pytest.mark.integration
+    def test_respects_threshold(self, client: TestClient, db_session):
+        """Should filter pairs below threshold."""
+        emb = [1.0, 0.0]
+        different_emb = [0.0, 1.0]
+
+        f1 = FileRecord(
+            filehash="thA",
+            local_filename="/tmp/thA.pdf",
+            file_size=100,
+            original_filename="thA.pdf",
+            ocr_text="a",
+            embedding=json.dumps(emb),
+        )
+        f2 = FileRecord(
+            filehash="thB",
+            local_filename="/tmp/thB.pdf",
+            file_size=100,
+            original_filename="thB.pdf",
+            ocr_text="b",
+            embedding=json.dumps(different_emb),
+        )
+        db_session.add_all([f1, f2])
+        db_session.commit()
+
+        response = client.get("/api/similarity/pairs?threshold=0.9")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["total_pairs"] == 0
+
+    @pytest.mark.integration
+    def test_pagination(self, client: TestClient, db_session):
+        """Should respect pagination parameters."""
+        emb = [1.0, 0.0, 0.0]
+        for i in range(5):
+            f = FileRecord(
+                filehash=f"pg{i}",
+                local_filename=f"/tmp/pg{i}.pdf",
+                file_size=100,
+                original_filename=f"pg{i}.pdf",
+                ocr_text=f"text {i}",
+                embedding=json.dumps(emb),
+            )
+            db_session.add(f)
+        db_session.commit()
+
+        response = client.get("/api/similarity/pairs?threshold=0.0&limit=2&page=1")
+        assert response.status_code == 200
+        data = response.json()
+        assert len(data["pairs"]) <= 2
+        assert data["per_page"] == 2
+
+
+# ---------------------------------------------------------------------------
+# Tests for backfill_missing_embeddings task
+# ---------------------------------------------------------------------------
+
+
+class TestBackfillMissingEmbeddingsTask:
+    """Unit tests for the backfill_missing_embeddings Celery task."""
+
+    @pytest.mark.unit
+    @patch("app.tasks.compute_embedding.compute_document_embedding.delay")
+    def test_queues_files_without_embeddings(self, mock_delay, db_session):
+        """Should queue tasks for files with OCR text but no embedding."""
+        f1 = FileRecord(
+            filehash="bf1",
+            local_filename="/tmp/bf1.pdf",
+            file_size=100,
+            original_filename="bf1.pdf",
+            ocr_text="Some text",
+        )
+        f2 = FileRecord(
+            filehash="bf2",
+            local_filename="/tmp/bf2.pdf",
+            file_size=100,
+            original_filename="bf2.pdf",
+            ocr_text="More text",
+            embedding=json.dumps([0.1]),
+        )
+        db_session.add_all([f1, f2])
+        db_session.commit()
+
+        from app.tasks.compute_embedding import backfill_missing_embeddings
+
+        with patch("app.tasks.compute_embedding.SessionLocal") as mock_session_local:
+            mock_session_local.return_value.__enter__ = lambda self: db_session
+            mock_session_local.return_value.__exit__ = lambda self, *args: None
+
+            result = backfill_missing_embeddings()
+
+        assert result["queued"] == 1
+        mock_delay.assert_called_once_with(f1.id)
+
+    @pytest.mark.unit
+    @patch("app.tasks.compute_embedding.compute_document_embedding.delay")
+    def test_empty_database(self, mock_delay, db_session):
+        """Should queue nothing when no files need embeddings."""
+        from app.tasks.compute_embedding import backfill_missing_embeddings
+
+        with patch("app.tasks.compute_embedding.SessionLocal") as mock_session_local:
+            mock_session_local.return_value.__enter__ = lambda self: db_session
+            mock_session_local.return_value.__exit__ = lambda self, *args: None
+
+            result = backfill_missing_embeddings()
+
+        assert result["queued"] == 0
+        mock_delay.assert_not_called()