feat(similarity): add embedding pipeline, debug endpoints, backfill task, and scalable similarity search

- Add embedding_model config setting (replaces hardcoded text-embedding-3-small)
- Add compute_document_embedding Celery task for ingestion-time embedding
- Chain embedding task into finalize_document_storage pipeline
- Add backfill_missing_embeddings periodic task (every 5 min) for legacy files
- Add debug API endpoints: embedding-status, compute-embedding, diagnostic/embeddings, diagnostic/compute-all-embeddings
- Refactor find_similar_documents to only use pre-computed embeddings (no lazy API calls)
- Use yield_per(500) and column-only queries for 100K+ scale
- Add embedding status indicator and recompute button in file detail UI

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/api/similarity.py

@@ -1,9 +1,11 @@
 """Document similarity API endpoints.
 
-Provides an endpoint to find documents similar to a given file based on
-text embeddings and cosine similarity scoring.
+Provides endpoints to find documents similar to a given file based on
+text embeddings and cosine similarity scoring, plus debug/diagnostic
+endpoints for inspecting and triggering embedding computation.
 """
 
+import json
 import logging
 from typing import Annotated
 
@@ -11,6 +13,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
 from sqlalchemy.orm import Session
 
 from app.auth import require_login
+from app.config import settings
 from app.database import get_db
 from app.models import FileRecord
 
@@ -95,3 +98,236 @@ def get_similar_documents(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
             detail="Failed to compute document similarity",
         )
+
+
+# ---------------------------------------------------------------------------
+# Debug / diagnostic endpoints
+# ---------------------------------------------------------------------------
+
+
+@router.get("/files/{file_id}/embedding-status")
+@require_login
+def get_embedding_status(
+    request: Request,
+    file_id: int,
+    db: DbSession,
+):
+    """Return the embedding status for a single file.
+
+    Useful for debugging whether the embedding has been computed
+    and cached for a given document.
+
+    Response:
+    ```json
+    {
+      "file_id": 42,
+      "has_embedding": true,
+      "embedding_dimensions": 1536,
+      "has_ocr_text": true,
+      "ocr_text_length": 4200,
+      "embedding_model": "text-embedding-3-small"
+    }
+    ```
+    """
+    file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
+    if not file_record:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
+
+    has_embedding = False
+    embedding_dimensions = None
+    if file_record.embedding:
+        try:
+            parsed = json.loads(file_record.embedding)
+            has_embedding = True
+            embedding_dimensions = len(parsed)
+        except (json.JSONDecodeError, TypeError):
+            pass
+
+    has_ocr_text = bool(file_record.ocr_text and file_record.ocr_text.strip())
+
+    return {
+        "file_id": file_id,
+        "has_embedding": has_embedding,
+        "embedding_dimensions": embedding_dimensions,
+        "has_ocr_text": has_ocr_text,
+        "ocr_text_length": len(file_record.ocr_text) if file_record.ocr_text else 0,
+        "embedding_model": settings.embedding_model,
+    }
+
+
+@router.post("/files/{file_id}/compute-embedding")
+@require_login
+def trigger_compute_embedding(
+    request: Request,
+    file_id: int,
+    db: DbSession,
+):
+    """Trigger embedding computation for a single file.
+
+    If the file already has a cached embedding it will be recomputed.
+    The computation happens synchronously so the caller receives the
+    result immediately.
+
+    Response:
+    ```json
+    {
+      "file_id": 42,
+      "status": "success",
+      "embedding_dimensions": 1536
+    }
+    ```
+    """
+    file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
+    if not file_record:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
+
+    if not file_record.ocr_text or not file_record.ocr_text.strip():
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="File has no OCR text — cannot generate embedding",
+        )
+
+    try:
+        from app.utils.similarity import generate_embedding
+
+        # Clear cached embedding to force recomputation
+        file_record.embedding = None
+        db.flush()
+
+        embedding = generate_embedding(file_record.ocr_text)
+        file_record.embedding = json.dumps(embedding)
+        db.commit()
+
+        return {
+            "file_id": file_id,
+            "status": "success",
+            "embedding_dimensions": len(embedding),
+        }
+    except Exception as e:
+        db.rollback()
+        logger.error(f"Failed to compute embedding for file {file_id}: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Embedding computation failed: {e}",
+        )
+
+
+@router.get("/diagnostic/embeddings")
+@require_login
+def get_embeddings_overview(
+    request: Request,
+    db: DbSession,
+):
+    """Return an overview of embedding status across all files.
+
+    Provides aggregate counts as well as a per-file breakdown so an
+    administrator can quickly identify documents that are missing
+    embeddings.
+
+    Response:
+    ```json
+    {
+      "total_files": 120,
+      "files_with_ocr_text": 95,
+      "files_with_embedding": 42,
+      "files_missing_embedding": 53,
+      "embedding_model": "text-embedding-3-small",
+      "files": [
+        {
+          "file_id": 1,
+          "original_filename": "invoice.pdf",
+          "has_ocr_text": true,
+          "has_embedding": true,
+          "embedding_dimensions": 1536
+        }
+      ]
+    }
+    ```
+    """
+    all_files = db.query(FileRecord).order_by(FileRecord.id.desc()).all()
+
+    files_info = []
+    total_with_ocr = 0
+    total_with_embedding = 0
+
+    for f in all_files:
+        has_ocr = bool(f.ocr_text and f.ocr_text.strip())
+        has_emb = False
+        emb_dims = None
+
+        if f.embedding:
+            try:
+                parsed = json.loads(f.embedding)
+                has_emb = True
+                emb_dims = len(parsed)
+            except (json.JSONDecodeError, TypeError):
+                pass
+
+        if has_ocr:
+            total_with_ocr += 1
+        if has_emb:
+            total_with_embedding += 1
+
+        files_info.append(
+            {
+                "file_id": f.id,
+                "original_filename": f.original_filename,
+                "has_ocr_text": has_ocr,
+                "has_embedding": has_emb,
+                "embedding_dimensions": emb_dims,
+            }
+        )
+
+    return {
+        "total_files": len(all_files),
+        "files_with_ocr_text": total_with_ocr,
+        "files_with_embedding": total_with_embedding,
+        "files_missing_embedding": total_with_ocr - total_with_embedding,
+        "embedding_model": settings.embedding_model,
+        "files": files_info,
+    }
+
+
+@router.post("/diagnostic/compute-all-embeddings")
+@require_login
+def trigger_compute_all_embeddings(
+    request: Request,
+    db: DbSession,
+):
+    """Queue embedding computation for all files that have OCR text but no embedding.
+
+    Each file is processed as a separate Celery task so the endpoint
+    returns immediately.
+
+    Response:
+    ```json
+    {
+      "status": "queued",
+      "files_queued": 53
+    }
+    ```
+    """
+    candidates = (
+        db.query(FileRecord)
+        .filter(
+            FileRecord.ocr_text.isnot(None),
+            FileRecord.ocr_text != "",
+            (FileRecord.embedding.is_(None)) | (FileRecord.embedding == ""),
+        )
+        .all()
+    )
+
+    queued = 0
+    for f in candidates:
+        try:
+            from app.tasks.compute_embedding import compute_document_embedding
+
+            compute_document_embedding.delay(f.id)
+            queued += 1
+        except Exception as e:
+            logger.warning(f"Could not queue embedding for file {f.id}: {e}")
+
+    return {
+        "status": "queued",
+        "files_queued": queued,
+    }

app/celery_worker.py

@@ -9,6 +9,7 @@ from app import tasks  # noqa: F401 - Imports app/tasks.py so Celery can registe
 from app.celery_app import celery
 from app.config import settings
 from app.tasks.check_credentials import check_credentials
+from app.tasks.compute_embedding import backfill_missing_embeddings, compute_document_embedding  # noqa: F401
 from app.tasks.convert_to_pdf import convert_to_pdf  # noqa: F401
 from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf  # noqa: F401
 from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt  # noqa: F401
@@ -91,6 +92,13 @@ celery.conf.beat_schedule = {
         "schedule": crontab(minute="*/1"),  # Every minute
         "options": {"expires": 55},  # Must complete within 55 seconds
     },
+    # Backfill embeddings for files that were processed before the
+    # embedding pipeline was enabled, or where the embedding task failed.
+    "backfill-missing-embeddings": {
+        "task": "backfill_missing_embeddings",
+        "schedule": crontab(minute="*/5"),  # Every 5 minutes
+        "options": {"expires": 240},  # 4 minutes expiry
+    },
 }
 
 # Remove None entries from beat_schedule

app/config.py

@@ -323,6 +323,13 @@ class Settings(BaseSettings):
             "them near-duplicates.  Higher values require closer content matches.  Default: 0.85."
         ),
     )
+    embedding_model: str = Field(
+        default="text-embedding-3-small",
+        description=(
+            "Model name used for generating text embeddings via the OpenAI-compatible API.  "
+            "Embeddings drive the document similarity feature.  Default: text-embedding-3-small."
+        ),
+    )
 
     # Text quality check - AI-based assessment of embedded PDF text
     enable_text_quality_check: bool = Field(

app/tasks/compute_embedding.py

@@ -0,0 +1,148 @@
+"""Celery task for pre-computing document text embeddings.
+
+Runs after document processing to ensure embeddings are available for
+the similarity feature without requiring a user to trigger them on first
+access.
+"""
+
+import logging
+
+from app.celery_app import celery
+from app.database import SessionLocal
+from app.models import FileRecord
+from app.tasks.retry_config import BaseTaskWithRetry
+from app.utils import log_task_progress
+
+logger = logging.getLogger(__name__)
+
+
+@celery.task(base=BaseTaskWithRetry, bind=True, name="compute_document_embedding")
+def compute_document_embedding(self, file_id: int) -> dict:
+    """Compute and cache the text embedding for a single document.
+
+    Skips silently when the file has no OCR text or already has a cached
+    embedding.  The result is stored in ``FileRecord.embedding`` for
+    subsequent similarity queries.
+
+    Args:
+        file_id: Primary key of the :class:`~app.models.FileRecord`.
+
+    Returns:
+        A dict with ``status`` (``"success"`` / ``"skipped"`` / ``"error"``)
+        and optional ``detail`` message.
+    """
+    task_id = self.request.id
+    logger.info("[%s] Computing embedding for file %s", task_id, file_id)
+    log_task_progress(
+        task_id,
+        "compute_embedding",
+        "in_progress",
+        f"Computing text embedding for file {file_id}",
+        file_id=file_id,
+    )
+
+    with SessionLocal() as db:
+        file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
+        if not file_record:
+            logger.warning("[%s] File %s not found, skipping embedding", task_id, file_id)
+            return {"status": "skipped", "detail": "File not found"}
+
+        # Already has a cached embedding – nothing to do
+        if file_record.embedding:
+            logger.info("[%s] File %s already has a cached embedding", task_id, file_id)
+            log_task_progress(
+                task_id,
+                "compute_embedding",
+                "success",
+                "Embedding already cached",
+                file_id=file_id,
+            )
+            return {"status": "skipped", "detail": "Embedding already cached"}
+
+        if not file_record.ocr_text or not file_record.ocr_text.strip():
+            logger.info("[%s] File %s has no OCR text, skipping embedding", task_id, file_id)
+            log_task_progress(
+                task_id,
+                "compute_embedding",
+                "skipped",
+                "No OCR text available",
+                file_id=file_id,
+            )
+            return {"status": "skipped", "detail": "No OCR text available"}
+
+        try:
+            from app.utils.similarity import compute_and_store_embedding
+
+            embedding = compute_and_store_embedding(db, file_record)
+            if embedding:
+                log_task_progress(
+                    task_id,
+                    "compute_embedding",
+                    "success",
+                    f"Embedding computed ({len(embedding)} dimensions)",
+                    file_id=file_id,
+                )
+                return {
+                    "status": "success",
+                    "detail": f"Embedding computed ({len(embedding)} dimensions)",
+                }
+            else:
+                log_task_progress(
+                    task_id,
+                    "compute_embedding",
+                    "failure",
+                    "Embedding computation returned None",
+                    file_id=file_id,
+                )
+                return {"status": "error", "detail": "Embedding computation returned None"}
+        except Exception as exc:
+            logger.exception("[%s] Embedding computation failed for file %s: %s", task_id, file_id, exc)
+            log_task_progress(
+                task_id,
+                "compute_embedding",
+                "failure",
+                f"Exception: {exc}",
+                file_id=file_id,
+            )
+            return {"status": "error", "detail": str(exc)}
+
+
+@celery.task(bind=True, name="backfill_missing_embeddings")
+def backfill_missing_embeddings(self) -> dict:
+    """Periodic task that computes embeddings for documents that lack them.
+
+    Iterates over all ``FileRecord`` rows that have OCR text but no
+    cached embedding and queues a :func:`compute_document_embedding`
+    task for each one.  A configurable ``batch_size`` caps the number
+    of tasks queued per run to avoid overwhelming the worker or the
+    embedding API.
+
+    Returns:
+        A dict with the number of tasks ``queued``.
+    """
+    batch_size = 50  # max files to queue per run
+    task_id = self.request.id
+    logger.info("[%s] Backfill: scanning for files missing embeddings (batch_size=%d)", task_id, batch_size)
+
+    with SessionLocal() as db:
+        candidates = (
+            db.query(FileRecord.id)
+            .filter(
+                FileRecord.ocr_text.isnot(None),
+                FileRecord.ocr_text != "",
+                (FileRecord.embedding.is_(None)) | (FileRecord.embedding == ""),
+            )
+            .limit(batch_size)
+            .all()
+        )
+
+    queued = 0
+    for (file_id,) in candidates:
+        try:
+            compute_document_embedding.delay(file_id)
+            queued += 1
+        except Exception as exc:
+            logger.warning("[%s] Could not queue embedding for file %s: %s", task_id, file_id, exc)
+
+    logger.info("[%s] Backfill: queued %d embedding tasks", task_id, queued)
+    return {"queued": queued}

app/tasks/finalize_document_storage.py

@@ -76,6 +76,16 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
     # We pass 'True' (delete_after) and 'file_id' as per Main branch requirements
     send_to_all_destinations.delay(processed_file, True, file_id)
 
+    # 3a. Queue embedding computation so similarity scores are ready for queries
+    if file_id is not None:
+        try:
+            from app.tasks.compute_embedding import compute_document_embedding
+
+            compute_document_embedding.delay(file_id)
+            logger.info(f"[{task_id}] Queued embedding computation for file {file_id}")
+        except Exception as e:
+            logger.warning(f"[{task_id}] Could not queue embedding task: {e}")
+
     # 4. Send Notification (From Copilot)
     # Note: This notification is sent after processing is complete but while uploads
     # are being queued.

app/utils/similarity.py

@@ -38,13 +38,14 @@ def _get_embedding_client() -> Any:
     )
 
 
-def generate_embedding(text: str, model: str = "text-embedding-3-small") -> list[float]:
+def generate_embedding(text: str, model: str | None = None) -> list[float]:
     """Generate a text embedding vector using the OpenAI-compatible API.
 
     Args:
         text: The input text to embed.  Truncated to ~8000 tokens worth of
             characters to stay within model limits.
-        model: The embedding model to use.  Defaults to ``text-embedding-3-small``.
+        model: The embedding model to use.  When ``None`` (the default), the
+            value of ``settings.embedding_model`` is used.
 
     Returns:
         A list of floats representing the embedding vector.
@@ -53,12 +54,16 @@ def generate_embedding(text: str, model: str = "text-embedding-3-small") -> list
         RuntimeError: If the OpenAI client cannot be created.
         Exception: If the API call fails.
     """
+    if model is None:
+        model = settings.embedding_model
+
     # Truncate very long texts to stay within token limits (~4 chars per token)
     max_chars = 30000
     if len(text) > max_chars:
         text = text[:max_chars]
 
     client = _get_embedding_client()
+    logger.debug("Generating embedding for %d chars using model=%s", len(text), model)
     response = client.embeddings.create(input=text, model=model)
     return response.data[0].embedding
 
@@ -89,8 +94,41 @@ def cosine_similarity(vec_a: list[float], vec_b: list[float]) -> float:
     return max(0.0, min(1.0, similarity))
 
 
-def _get_or_compute_embedding(db: Session, file_record: Any) -> list[float] | None:
-    """Retrieve a cached embedding or compute and store a new one.
+def _get_cached_embedding(file_record: Any) -> list[float] | None:
+    """Return the cached embedding for a file record, or ``None``.
+
+    This is a **read-only** helper — it never triggers an API call.  Use
+    :func:`compute_and_store_embedding` when you need to generate a new
+    embedding.
+
+    Args:
+        file_record: A ``FileRecord`` instance (or any object with ``id``
+            and ``embedding`` attributes).
+
+    Returns:
+        The parsed embedding vector, or ``None`` if no valid cached
+        embedding exists.
+    """
+    raw = file_record.embedding if hasattr(file_record, "embedding") else None
+    if not raw:
+        return None
+    try:
+        cached = json.loads(raw)
+        logger.debug("Using cached embedding for file %s (%d dimensions)", file_record.id, len(cached))
+        return cached
+    except (json.JSONDecodeError, TypeError):
+        logger.warning("Invalid cached embedding for file %s", file_record.id)
+        return None
+
+
+def compute_and_store_embedding(db: Session, file_record: Any) -> list[float] | None:
+    """Generate an embedding for a file and persist it in the database.
+
+    Called during document ingestion (Celery task) or via the manual
+    ``POST /api/files/{id}/compute-embedding`` debug endpoint.  The
+    similarity query path (:func:`find_similar_documents`) intentionally
+    does **not** call this — it only reads pre-computed embeddings so
+    that it returns instantly without blocking on external API calls.
 
     Args:
         db: Active database session.
@@ -100,40 +138,54 @@ def _get_or_compute_embedding(db: Session, file_record: Any) -> list[float] | No
         The embedding vector, or ``None`` if the document has no OCR text
         or embedding generation fails.
     """
-    # Return cached embedding if available
+    # Return cached embedding if already present
     if file_record.embedding:
         try:
-            return json.loads(file_record.embedding)
+            cached = json.loads(file_record.embedding)
+            logger.debug("Embedding already cached for file %s (%d dims)", file_record.id, len(cached))
+            return cached
         except (json.JSONDecodeError, TypeError):
-            logger.warning(f"Invalid cached embedding for file {file_record.id}, recomputing")
+            logger.warning("Invalid cached embedding for file %s, recomputing", file_record.id)
 
     # Need OCR text to generate an embedding
     if not file_record.ocr_text or not file_record.ocr_text.strip():
+        logger.debug("No OCR text for file %s, cannot generate embedding", file_record.id)
         return None
 
     try:
+        logger.info("Computing embedding for file %s (%d chars of OCR text)", file_record.id, len(file_record.ocr_text))
         embedding = generate_embedding(file_record.ocr_text)
-        # Cache the embedding in the database
+        # Persist in the database
         file_record.embedding = json.dumps(embedding)
         db.commit()
+        logger.info("Embedding computed and cached for file %s (%d dimensions)", file_record.id, len(embedding))
         return embedding
     except Exception as e:
         db.rollback()
-        logger.error(f"Failed to generate embedding for file {file_record.id}: {e}")
+        logger.error("Failed to generate embedding for file %s: %s", file_record.id, e)
         return None
 
 
+# Keep the legacy alias so that existing callers (e.g. tests) keep working.
+_get_or_compute_embedding = compute_and_store_embedding
+
+
 def find_similar_documents(
     db: Session,
     file_id: int,
     limit: int = 5,
     threshold: float = 0.3,
 ) -> list[dict[str, Any]]:
-    """Find documents similar to the given file.
+    """Find documents similar to the given file using **pre-computed** embeddings.
 
-    Computes cosine similarity between the target document's embedding and
-    all other documents that have OCR text.  Results are sorted by descending
-    similarity score.
+    Only documents whose embeddings were already generated (during
+    ingestion or via the debug endpoint) are considered.  No external API
+    calls are made — the function reads cached vectors from the database
+    and computes cosine similarity in-process.
+
+    To keep memory usage bounded for large corpora (100 k+ documents) the
+    candidate query fetches only the columns needed for scoring and
+    iterates in chunks via ``yield_per``.
 
     Args:
         db: Active database session.
@@ -152,43 +204,54 @@ def find_similar_documents(
     """
     from app.models import FileRecord
 
-    # Get the target document
+    # Get the target document's cached embedding (read-only, no API call)
     target = db.query(FileRecord).filter(FileRecord.id == file_id).first()
     if not target:
         return []
 
-    # Get the target embedding
-    target_embedding = _get_or_compute_embedding(db, target)
+    target_embedding = _get_cached_embedding(target)
     if not target_embedding:
+        logger.info("No cached embedding for target file %s — skipping similarity search", file_id)
         return []
 
-    # Get candidate documents (those with OCR text, excluding the target)
+    # Query only candidates that already have a pre-computed embedding.
+    # Fetch only the columns needed for scoring to minimise memory use.
+    # yield_per streams rows in chunks so we never materialise all 100k+
+    # records at once.
     candidates = (
-        db.query(FileRecord)
+        db.query(
+            FileRecord.id,
+            FileRecord.original_filename,
+            FileRecord.document_title,
+            FileRecord.mime_type,
+            FileRecord.created_at,
+            FileRecord.embedding,
+        )
         .filter(
             FileRecord.id != file_id,
-            FileRecord.ocr_text.isnot(None),
-            FileRecord.ocr_text != "",
+            FileRecord.embedding.isnot(None),
+            FileRecord.embedding != "",
         )
-        .all()
+        .yield_per(500)
     )
 
-    results = []
-    for candidate in candidates:
-        candidate_embedding = _get_or_compute_embedding(db, candidate)
-        if not candidate_embedding:
+    results: list[dict[str, Any]] = []
+    for row in candidates:
+        try:
+            candidate_embedding: list[float] = json.loads(row.embedding)
+        except (json.JSONDecodeError, TypeError):
             continue
 
         score = cosine_similarity(target_embedding, candidate_embedding)
         if score >= threshold:
             results.append(
                 {
-                    "file_id": candidate.id,
-                    "original_filename": candidate.original_filename,
-                    "document_title": candidate.document_title,
+                    "file_id": row.id,
+                    "original_filename": row.original_filename,
+                    "document_title": row.document_title,
                     "similarity_score": round(score, 4),
-                    "mime_type": candidate.mime_type,
-                    "created_at": candidate.created_at.isoformat() if candidate.created_at else None,
+                    "mime_type": row.mime_type,
+                    "created_at": row.created_at.isoformat() if row.created_at else None,
                 }
             )
 

frontend/templates/file_detail.html

@@ -952,10 +952,75 @@
       loadPDF('processed', fileId);
       {% endif %}
 
-      // Load similar documents
+      // Load similar documents and embedding status
       loadSimilarDocuments(fileId);
+      loadEmbeddingStatus(fileId);
     });
 
+    // Fetch and display embedding debug info
+    async function loadEmbeddingStatus(fileId) {
+      const statusDiv = document.getElementById('embedding-status');
+      const actionsDiv = document.getElementById('embedding-actions');
+      try {
+        const response = await fetch(`/api/files/${fileId}/embedding-status`);
+        if (!response.ok) return;
+        const data = await response.json();
+
+        let html = '';
+        if (data.has_embedding) {
+          html = `<i class="fas fa-check-circle" aria-hidden="true"></i> Embedding: <strong>${data.embedding_dimensions} dimensions</strong> (model: ${data.embedding_model})`;
+          statusDiv.style.backgroundColor = '#f0fff4';
+          statusDiv.style.color = '#276749';
+          statusDiv.style.border = '1px solid #c6f6d5';
+        } else if (data.has_ocr_text) {
+          html = `<i class="fas fa-exclamation-circle" aria-hidden="true"></i> Embedding: <strong>not yet computed</strong> — OCR text available (${data.ocr_text_length} chars). Click "Recompute Embedding" to generate.`;
+          statusDiv.style.backgroundColor = '#fffff0';
+          statusDiv.style.color = '#975a16';
+          statusDiv.style.border = '1px solid #fefcbf';
+        } else {
+          html = `<i class="fas fa-times-circle" aria-hidden="true"></i> Embedding: <strong>unavailable</strong> — no OCR text extracted for this file.`;
+          statusDiv.style.backgroundColor = '#fff5f5';
+          statusDiv.style.color = '#9b2c2c';
+          statusDiv.style.border = '1px solid #fed7d7';
+        }
+        statusDiv.innerHTML = html;
+        statusDiv.style.display = 'block';
+        actionsDiv.style.display = data.has_ocr_text ? 'block' : 'none';
+      } catch (error) {
+        console.error('Error loading embedding status:', error);
+      }
+    }
+
+    // Trigger embedding recomputation
+    async function recomputeEmbedding(fileId) {
+      const btn = document.getElementById('recompute-embedding-btn');
+      btn.disabled = true;
+      btn.innerHTML = '<i class="fas fa-spinner fa-spin" aria-hidden="true"></i> Computing…';
+
+      try {
+        const response = await fetch(`/api/files/${fileId}/compute-embedding`, { method: 'POST' });
+        const data = await response.json();
+        if (!response.ok) {
+          throw new Error(data.detail || 'Request failed');
+        }
+        btn.innerHTML = '<i class="fas fa-check" aria-hidden="true"></i> Done';
+        btn.style.backgroundColor = '#48bb78';
+        // Refresh both embedding status and similar documents
+        loadEmbeddingStatus(fileId);
+        loadSimilarDocuments(fileId);
+      } catch (error) {
+        btn.innerHTML = '<i class="fas fa-times" aria-hidden="true"></i> Failed';
+        btn.style.backgroundColor = '#e53e3e';
+        console.error('Recompute embedding failed:', error);
+      } finally {
+        setTimeout(() => {
+          btn.disabled = false;
+          btn.innerHTML = '<i class="fas fa-redo" aria-hidden="true"></i> Recompute Embedding';
+          btn.style.backgroundColor = '#4299e1';
+        }, 3000);
+      }
+    }
+
     // Similar documents loading
     async function loadSimilarDocuments(fileId) {
       const loadingDiv = document.getElementById('similar-documents-loading');
@@ -1602,6 +1667,8 @@
   <!-- Similar Documents Card -->
   <div class="detail-card" id="similar-documents-card">
     <h3><i class="fas fa-copy" aria-hidden="true"></i> Similar Documents</h3>
+    <!-- Embedding debug info -->
+    <div id="embedding-status" style="display: none; padding: 0.5rem 1rem; margin-bottom: 0.75rem; border-radius: 0.375rem; font-size: 0.8rem;"></div>
     <div id="similar-documents-loading" style="text-align: center; padding: 2rem; color: #718096;">
       <i class="fas fa-spinner fa-spin" aria-hidden="true" style="font-size: 1.5rem; margin-bottom: 0.5rem;"></i>
       <p>Searching for similar documents…</p>
@@ -1615,6 +1682,12 @@
       <i class="fas fa-exclamation-triangle" aria-hidden="true" style="font-size: 2rem; margin-bottom: 0.5rem;"></i>
       <p id="similar-documents-error-msg">Failed to load similar documents.</p>
     </div>
+    <!-- Recompute embedding button -->
+    <div id="embedding-actions" style="display: none; text-align: right; margin-top: 0.75rem;">
+      <button id="recompute-embedding-btn" onclick="recomputeEmbedding({{ file.id | tojson }})" style="background-color: #4299e1; color: white; border: none; padding: 0.4rem 0.75rem; border-radius: 0.375rem; font-size: 0.8rem; cursor: pointer;" aria-label="Recompute document embedding for similarity analysis">
+        <i class="fas fa-redo" aria-hidden="true"></i> Recompute Embedding
+      </button>
+    </div>
   </div>
 
   <!-- File Preview Card -->

tests/test_similarity.py

@@ -443,3 +443,389 @@ class TestSimilarDocumentsAPI:
             assert "similarity_score" in doc
             assert "mime_type" in doc
             assert "created_at" in doc
+
+
+# ---------------------------------------------------------------------------
+# Tests for embedding status endpoint
+# ---------------------------------------------------------------------------
+
+
+class TestEmbeddingStatusAPI:
+    """Integration tests for GET /api/files/{file_id}/embedding-status."""
+
+    @pytest.mark.integration
+    def test_file_not_found(self, client: TestClient):
+        """Should return 404 for non-existent file."""
+        response = client.get("/api/files/9999/embedding-status")
+        assert response.status_code == 404
+
+    @pytest.mark.integration
+    def test_file_without_embedding_or_ocr(self, client: TestClient, db_session):
+        """Should report no embedding and no OCR text."""
+        file_record = FileRecord(
+            filehash="abc1",
+            local_filename="/tmp/test.pdf",
+            file_size=100,
+            original_filename="test.pdf",
+        )
+        db_session.add(file_record)
+        db_session.commit()
+
+        response = client.get(f"/api/files/{file_record.id}/embedding-status")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["file_id"] == file_record.id
+        assert data["has_embedding"] is False
+        assert data["embedding_dimensions"] is None
+        assert data["has_ocr_text"] is False
+        assert data["ocr_text_length"] == 0
+        assert "embedding_model" in data
+
+    @pytest.mark.integration
+    def test_file_with_ocr_text_no_embedding(self, client: TestClient, db_session):
+        """Should report OCR text present but no embedding."""
+        file_record = FileRecord(
+            filehash="abc2",
+            local_filename="/tmp/test2.pdf",
+            file_size=100,
+            original_filename="test2.pdf",
+            ocr_text="Some OCR text content",
+        )
+        db_session.add(file_record)
+        db_session.commit()
+
+        response = client.get(f"/api/files/{file_record.id}/embedding-status")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["has_embedding"] is False
+        assert data["has_ocr_text"] is True
+        assert data["ocr_text_length"] == 21
+
+    @pytest.mark.integration
+    def test_file_with_cached_embedding(self, client: TestClient, db_session):
+        """Should report embedding present with correct dimensions."""
+        embedding = [0.1, 0.2, 0.3, 0.4, 0.5]
+        file_record = FileRecord(
+            filehash="abc3",
+            local_filename="/tmp/test3.pdf",
+            file_size=100,
+            original_filename="test3.pdf",
+            ocr_text="Some text",
+            embedding=json.dumps(embedding),
+        )
+        db_session.add(file_record)
+        db_session.commit()
+
+        response = client.get(f"/api/files/{file_record.id}/embedding-status")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["has_embedding"] is True
+        assert data["embedding_dimensions"] == 5
+        assert data["has_ocr_text"] is True
+
+
+# ---------------------------------------------------------------------------
+# Tests for compute-embedding endpoint
+# ---------------------------------------------------------------------------
+
+
+class TestComputeEmbeddingAPI:
+    """Integration tests for POST /api/files/{file_id}/compute-embedding."""
+
+    @pytest.mark.integration
+    def test_file_not_found(self, client: TestClient):
+        """Should return 404 for non-existent file."""
+        response = client.post("/api/files/9999/compute-embedding")
+        assert response.status_code == 404
+
+    @pytest.mark.integration
+    def test_no_ocr_text(self, client: TestClient, db_session):
+        """Should return 400 when file has no OCR text."""
+        file_record = FileRecord(
+            filehash="emb1",
+            local_filename="/tmp/emb1.pdf",
+            file_size=100,
+            original_filename="emb1.pdf",
+        )
+        db_session.add(file_record)
+        db_session.commit()
+
+        response = client.post(f"/api/files/{file_record.id}/compute-embedding")
+        assert response.status_code == 400
+
+    @pytest.mark.integration
+    @patch("app.utils.similarity.generate_embedding")
+    def test_computes_embedding(self, mock_embed, client: TestClient, db_session):
+        """Should compute and store an embedding."""
+        mock_embed.return_value = [0.1, 0.2, 0.3]
+
+        file_record = FileRecord(
+            filehash="emb2",
+            local_filename="/tmp/emb2.pdf",
+            file_size=100,
+            original_filename="emb2.pdf",
+            ocr_text="Some document text",
+        )
+        db_session.add(file_record)
+        db_session.commit()
+
+        response = client.post(f"/api/files/{file_record.id}/compute-embedding")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "success"
+        assert data["embedding_dimensions"] == 3
+
+        # Verify embedding is stored
+        db_session.refresh(file_record)
+        assert file_record.embedding is not None
+        stored = json.loads(file_record.embedding)
+        assert len(stored) == 3
+
+    @pytest.mark.integration
+    @patch("app.utils.similarity.generate_embedding")
+    def test_recomputes_existing_embedding(self, mock_embed, client: TestClient, db_session):
+        """Should overwrite existing embedding when recomputing."""
+        mock_embed.return_value = [0.9, 0.8, 0.7]
+
+        file_record = FileRecord(
+            filehash="emb3",
+            local_filename="/tmp/emb3.pdf",
+            file_size=100,
+            original_filename="emb3.pdf",
+            ocr_text="Some text",
+            embedding=json.dumps([0.1, 0.2, 0.3]),
+        )
+        db_session.add(file_record)
+        db_session.commit()
+
+        response = client.post(f"/api/files/{file_record.id}/compute-embedding")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "success"
+
+        db_session.refresh(file_record)
+        stored = json.loads(file_record.embedding)
+        assert stored == [0.9, 0.8, 0.7]
+
+
+# ---------------------------------------------------------------------------
+# Tests for diagnostic embeddings overview endpoint
+# ---------------------------------------------------------------------------
+
+
+class TestEmbeddingsOverviewAPI:
+    """Integration tests for GET /api/diagnostic/embeddings."""
+
+    @pytest.mark.integration
+    def test_empty_database(self, client: TestClient):
+        """Should return zero counts on empty database."""
+        response = client.get("/api/diagnostic/embeddings")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["total_files"] == 0
+        assert data["files_with_ocr_text"] == 0
+        assert data["files_with_embedding"] == 0
+        assert data["files_missing_embedding"] == 0
+        assert "embedding_model" in data
+        assert data["files"] == []
+
+    @pytest.mark.integration
+    def test_mixed_files(self, client: TestClient, db_session):
+        """Should report correct counts for mixed embedding states."""
+        # File with both OCR text and embedding
+        f1 = FileRecord(
+            filehash="diag1",
+            local_filename="/tmp/d1.pdf",
+            file_size=100,
+            original_filename="d1.pdf",
+            ocr_text="Some text",
+            embedding=json.dumps([0.1, 0.2]),
+        )
+        # File with OCR text but no embedding
+        f2 = FileRecord(
+            filehash="diag2",
+            local_filename="/tmp/d2.pdf",
+            file_size=100,
+            original_filename="d2.pdf",
+            ocr_text="More text",
+        )
+        # File with no OCR text
+        f3 = FileRecord(
+            filehash="diag3",
+            local_filename="/tmp/d3.pdf",
+            file_size=100,
+            original_filename="d3.pdf",
+        )
+        db_session.add_all([f1, f2, f3])
+        db_session.commit()
+
+        response = client.get("/api/diagnostic/embeddings")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["total_files"] == 3
+        assert data["files_with_ocr_text"] == 2
+        assert data["files_with_embedding"] == 1
+        assert data["files_missing_embedding"] == 1
+        assert len(data["files"]) == 3
+
+        # Check per-file info
+        files_by_id = {f["file_id"]: f for f in data["files"]}
+        assert files_by_id[f1.id]["has_embedding"] is True
+        assert files_by_id[f1.id]["embedding_dimensions"] == 2
+        assert files_by_id[f2.id]["has_embedding"] is False
+        assert files_by_id[f2.id]["has_ocr_text"] is True
+        assert files_by_id[f3.id]["has_ocr_text"] is False
+
+
+# ---------------------------------------------------------------------------
+# Tests for compute-all-embeddings endpoint
+# ---------------------------------------------------------------------------
+
+
+class TestComputeAllEmbeddingsAPI:
+    """Integration tests for POST /api/diagnostic/compute-all-embeddings."""
+
+    @pytest.mark.integration
+    @patch("app.tasks.compute_embedding.compute_document_embedding.delay")
+    def test_queues_tasks_for_files_missing_embeddings(self, mock_delay, client: TestClient, db_session):
+        """Should queue embedding tasks for files with OCR text but no embedding."""
+        # File with OCR text but no embedding -> should be queued
+        f1 = FileRecord(
+            filehash="all1",
+            local_filename="/tmp/a1.pdf",
+            file_size=100,
+            original_filename="a1.pdf",
+            ocr_text="Text for embedding",
+        )
+        # File already with embedding -> should NOT be queued
+        f2 = FileRecord(
+            filehash="all2",
+            local_filename="/tmp/a2.pdf",
+            file_size=100,
+            original_filename="a2.pdf",
+            ocr_text="More text",
+            embedding=json.dumps([0.1, 0.2]),
+        )
+        # File without OCR text -> should NOT be queued
+        f3 = FileRecord(
+            filehash="all3",
+            local_filename="/tmp/a3.pdf",
+            file_size=100,
+            original_filename="a3.pdf",
+        )
+        db_session.add_all([f1, f2, f3])
+        db_session.commit()
+
+        response = client.post("/api/diagnostic/compute-all-embeddings")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "queued"
+        assert data["files_queued"] == 1
+        mock_delay.assert_called_once_with(f1.id)
+
+    @pytest.mark.integration
+    @patch("app.tasks.compute_embedding.compute_document_embedding.delay")
+    def test_empty_database_queues_nothing(self, mock_delay, client: TestClient):
+        """Should queue nothing when database is empty."""
+        response = client.post("/api/diagnostic/compute-all-embeddings")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["files_queued"] == 0
+        mock_delay.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# Tests for compute_document_embedding Celery task
+# ---------------------------------------------------------------------------
+
+
+class TestComputeDocumentEmbeddingTask:
+    """Unit tests for the compute_document_embedding Celery task."""
+
+    @pytest.mark.unit
+    @patch("app.utils.similarity.generate_embedding")
+    def test_computes_embedding_for_file(self, mock_embed, db_session):
+        """Should compute and store embedding when file has OCR text."""
+        mock_embed.return_value = [0.1, 0.2, 0.3]
+
+        file_record = FileRecord(
+            filehash="task1",
+            local_filename="/tmp/task1.pdf",
+            file_size=100,
+            original_filename="task1.pdf",
+            ocr_text="Some document text",
+        )
+        db_session.add(file_record)
+        db_session.commit()
+
+        from app.tasks.compute_embedding import compute_document_embedding
+
+        # Patch SessionLocal to return our test session
+        with patch("app.tasks.compute_embedding.SessionLocal") as mock_session_local:
+            mock_session_local.return_value.__enter__ = lambda self: db_session
+            mock_session_local.return_value.__exit__ = lambda self, *args: None
+
+            result = compute_document_embedding(file_record.id)
+
+        assert result["status"] == "success"
+        assert "dimensions" in result["detail"]
+
+    @pytest.mark.unit
+    def test_skips_missing_file(self, db_session):
+        """Should skip when file ID does not exist."""
+        from app.tasks.compute_embedding import compute_document_embedding
+
+        with patch("app.tasks.compute_embedding.SessionLocal") as mock_session_local:
+            mock_session_local.return_value.__enter__ = lambda self: db_session
+            mock_session_local.return_value.__exit__ = lambda self, *args: None
+
+            result = compute_document_embedding(9999)
+
+        assert result["status"] == "skipped"
+
+    @pytest.mark.unit
+    def test_skips_file_without_ocr_text(self, db_session):
+        """Should skip when file has no OCR text."""
+        file_record = FileRecord(
+            filehash="task2",
+            local_filename="/tmp/task2.pdf",
+            file_size=100,
+            original_filename="task2.pdf",
+        )
+        db_session.add(file_record)
+        db_session.commit()
+
+        from app.tasks.compute_embedding import compute_document_embedding
+
+        with patch("app.tasks.compute_embedding.SessionLocal") as mock_session_local:
+            mock_session_local.return_value.__enter__ = lambda self: db_session
+            mock_session_local.return_value.__exit__ = lambda self, *args: None
+
+            result = compute_document_embedding(file_record.id)
+
+        assert result["status"] == "skipped"
+
+    @pytest.mark.unit
+    def test_skips_file_with_existing_embedding(self, db_session):
+        """Should skip when file already has a cached embedding."""
+        file_record = FileRecord(
+            filehash="task3",
+            local_filename="/tmp/task3.pdf",
+            file_size=100,
+            original_filename="task3.pdf",
+            ocr_text="Some text",
+            embedding=json.dumps([0.1, 0.2]),
+        )
+        db_session.add(file_record)
+        db_session.commit()
+
+        from app.tasks.compute_embedding import compute_document_embedding
+
+        with patch("app.tasks.compute_embedding.SessionLocal") as mock_session_local:
+            mock_session_local.return_value.__enter__ = lambda self: db_session
+            mock_session_local.return_value.__exit__ = lambda self, *args: None
+
+            result = compute_document_embedding(file_record.id)
+
+        assert result["status"] == "skipped"
+        assert "already cached" in result["detail"]