feat(similarity): add similarity pairs dashboard, step tracking, and fix tests for pre-computed embeddings

- Add GET /api/similarity/pairs endpoint for corpus-wide pair discovery
- Add /similarity view route and similarity_dashboard.html template
- Add Similarity link to desktop and mobile nav menus
- Register compute_embedding as a tracked FileProcessingStep
- Update compute_embedding task with update_step_status calls
- Add compute_embedding to flow visualization in _compute_processing_flow
- Add backfill_missing_embeddings periodic beat task (every 5 min)
- Return clear message when embedding not yet computed in similar docs API
- Fix all tests to use pre-computed embeddings (no lazy API calls)
- Add tests for similarity pairs, backfill task, and embedding-not-computed

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/views/files.py

@@ -389,8 +389,12 @@ def _compute_processing_flow(logs):
         },
         "extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]},
         "embed_metadata_into_pdf": {"label": "Embed Metadata into PDF", "next": ["finalize_document_storage"]},
-        "finalize_document_storage": {"label": "Finalize & Queue Distribution", "next": ["send_to_all_destinations"]},
+        "finalize_document_storage": {
+            "label": "Finalize & Queue Distribution",
+            "next": ["send_to_all_destinations", "compute_embedding"],
+        },
         "send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True},
+        "compute_embedding": {"label": "Compute Embedding", "next": []},
     }
 
     # Filter out deduplication step if not enabled or if not showing it
@@ -815,3 +819,54 @@ def duplicates_page(
                 "error": str(e),
             },
         )
+
+
+@router.get("/similarity")
+@require_login
+def similarity_dashboard_page(
+    request: Request,
+    db: Session = Depends(get_db),
+):
+    """Render the corpus-wide similarity dashboard.
+
+    Passes the configured threshold and embedding coverage stats so the
+    template can display them immediately while the JS fetches the actual
+    pairs from the API asynchronously.
+    """
+    from app.config import settings
+    from app.models import FileRecord
+
+    try:
+        total_files = db.query(FileRecord).count()
+        files_with_embedding = (
+            db.query(FileRecord).filter(FileRecord.embedding.isnot(None), FileRecord.embedding != "").count()
+        )
+        files_with_ocr = db.query(FileRecord).filter(FileRecord.ocr_text.isnot(None), FileRecord.ocr_text != "").count()
+
+        return templates.TemplateResponse(
+            "similarity_dashboard.html",
+            {
+                "request": request,
+                "default_threshold": settings.near_duplicate_threshold,
+                "embedding_model": settings.embedding_model,
+                "total_files": total_files,
+                "files_with_embedding": files_with_embedding,
+                "files_with_ocr": files_with_ocr,
+                "files_missing_embedding": files_with_ocr - files_with_embedding,
+            },
+        )
+    except Exception as e:
+        logger.error(f"Error rendering similarity dashboard: {e}")
+        return templates.TemplateResponse(
+            "similarity_dashboard.html",
+            {
+                "request": request,
+                "default_threshold": 0.85,
+                "embedding_model": "text-embedding-3-small",
+                "total_files": 0,
+                "files_with_embedding": 0,
+                "files_with_ocr": 0,
+                "files_missing_embedding": 0,
+                "error": str(e),
+            },
+        )