Merge branch 'main' into copilot/add-pdfa-export-option

app/api/similarity.py

@@ -1,9 +1,11 @@
 """Document similarity API endpoints.
 
-Provides an endpoint to find documents similar to a given file based on
-text embeddings and cosine similarity scoring.
+Provides endpoints to find documents similar to a given file based on
+text embeddings and cosine similarity scoring, plus debug/diagnostic
+endpoints for inspecting and triggering embedding computation.
 """
 
+import json
 import logging
 from typing import Annotated
 
@@ -11,6 +13,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
 from sqlalchemy.orm import Session
 
 from app.auth import require_login
+from app.config import settings
 from app.database import get_db
 from app.models import FileRecord
 
@@ -79,6 +82,19 @@ def get_similar_documents(
             "message": "No OCR text available for similarity comparison",
         }
 
+    # Check whether an embedding has been computed yet
+    if not file_record.embedding:
+        return {
+            "file_id": file_id,
+            "similar_documents": [],
+            "count": 0,
+            "message": (
+                "Embedding not yet computed for this file. "
+                "It will be generated automatically during processing or via the backfill task. "
+                "You can also trigger it manually with POST /api/files/{file_id}/compute-embedding."
+            ),
+        }
+
     try:
         from app.utils.similarity import find_similar_documents
 
@@ -95,3 +111,363 @@ def get_similar_documents(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
             detail="Failed to compute document similarity",
         )
+
+
+# ---------------------------------------------------------------------------
+# Debug / diagnostic endpoints
+# ---------------------------------------------------------------------------
+
+
+@router.get("/files/{file_id}/embedding-status")
+@require_login
+def get_embedding_status(
+    request: Request,
+    file_id: int,
+    db: DbSession,
+):
+    """Return the embedding status for a single file.
+
+    Useful for debugging whether the embedding has been computed
+    and cached for a given document.
+
+    Response:
+    ```json
+    {
+      "file_id": 42,
+      "has_embedding": true,
+      "embedding_dimensions": 1536,
+      "has_ocr_text": true,
+      "ocr_text_length": 4200,
+      "embedding_model": "text-embedding-3-small"
+    }
+    ```
+    """
+    file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
+    if not file_record:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
+
+    has_embedding = False
+    embedding_dimensions = None
+    if file_record.embedding:
+        try:
+            parsed = json.loads(file_record.embedding)
+            has_embedding = True
+            embedding_dimensions = len(parsed)
+        except (json.JSONDecodeError, TypeError):
+            pass
+
+    has_ocr_text = bool(file_record.ocr_text and file_record.ocr_text.strip())
+
+    return {
+        "file_id": file_id,
+        "has_embedding": has_embedding,
+        "embedding_dimensions": embedding_dimensions,
+        "has_ocr_text": has_ocr_text,
+        "ocr_text_length": len(file_record.ocr_text) if file_record.ocr_text else 0,
+        "embedding_model": settings.embedding_model,
+    }
+
+
+@router.post("/files/{file_id}/compute-embedding")
+@require_login
+def trigger_compute_embedding(
+    request: Request,
+    file_id: int,
+    db: DbSession,
+):
+    """Trigger embedding computation for a single file.
+
+    If the file already has a cached embedding it will be recomputed.
+    The computation happens synchronously so the caller receives the
+    result immediately.
+
+    Response:
+    ```json
+    {
+      "file_id": 42,
+      "status": "success",
+      "embedding_dimensions": 1536
+    }
+    ```
+    """
+    file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
+    if not file_record:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
+
+    if not file_record.ocr_text or not file_record.ocr_text.strip():
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="File has no OCR text — cannot generate embedding",
+        )
+
+    try:
+        from app.utils.similarity import generate_embedding
+
+        # Clear cached embedding to force recomputation
+        file_record.embedding = None
+        db.flush()
+
+        embedding = generate_embedding(file_record.ocr_text)
+        file_record.embedding = json.dumps(embedding)
+        db.commit()
+
+        return {
+            "file_id": file_id,
+            "status": "success",
+            "embedding_dimensions": len(embedding),
+        }
+    except Exception as e:
+        db.rollback()
+        logger.error(f"Failed to compute embedding for file {file_id}: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Embedding computation failed: {e}",
+        )
+
+
+@router.get("/diagnostic/embeddings")
+@require_login
+def get_embeddings_overview(
+    request: Request,
+    db: DbSession,
+):
+    """Return an overview of embedding status across all files.
+
+    Provides aggregate counts as well as a per-file breakdown so an
+    administrator can quickly identify documents that are missing
+    embeddings.
+
+    Response:
+    ```json
+    {
+      "total_files": 120,
+      "files_with_ocr_text": 95,
+      "files_with_embedding": 42,
+      "files_missing_embedding": 53,
+      "embedding_model": "text-embedding-3-small",
+      "files": [
+        {
+          "file_id": 1,
+          "original_filename": "invoice.pdf",
+          "has_ocr_text": true,
+          "has_embedding": true,
+          "embedding_dimensions": 1536
+        }
+      ]
+    }
+    ```
+    """
+    # Use column-only query to avoid loading full ORM objects into memory
+    all_files = (
+        db.query(
+            FileRecord.id,
+            FileRecord.original_filename,
+            FileRecord.ocr_text,
+            FileRecord.embedding,
+        )
+        .order_by(FileRecord.id.desc())
+        .all()
+    )
+
+    files_info = []
+    total_with_ocr = 0
+    total_with_embedding = 0
+
+    for f in all_files:
+        has_ocr = bool(f.ocr_text and f.ocr_text.strip())
+        has_emb = False
+        emb_dims = None
+
+        if f.embedding:
+            try:
+                parsed = json.loads(f.embedding)
+                has_emb = True
+                emb_dims = len(parsed)
+            except (json.JSONDecodeError, TypeError):
+                pass
+
+        if has_ocr:
+            total_with_ocr += 1
+        if has_emb:
+            total_with_embedding += 1
+
+        files_info.append(
+            {
+                "file_id": f.id,
+                "original_filename": f.original_filename,
+                "has_ocr_text": has_ocr,
+                "has_embedding": has_emb,
+                "embedding_dimensions": emb_dims,
+            }
+        )
+
+    return {
+        "total_files": len(all_files),
+        "files_with_ocr_text": total_with_ocr,
+        "files_with_embedding": total_with_embedding,
+        "files_missing_embedding": total_with_ocr - total_with_embedding,
+        "embedding_model": settings.embedding_model,
+        "files": files_info,
+    }
+
+
+@router.post("/diagnostic/compute-all-embeddings")
+@require_login
+def trigger_compute_all_embeddings(
+    request: Request,
+    db: DbSession,
+):
+    """Queue embedding computation for all files that have OCR text but no embedding.
+
+    Each file is processed as a separate Celery task so the endpoint
+    returns immediately.
+
+    Response:
+    ```json
+    {
+      "status": "queued",
+      "files_queued": 53
+    }
+    ```
+    """
+    candidates = (
+        db.query(FileRecord)
+        .filter(
+            FileRecord.ocr_text.isnot(None),
+            FileRecord.ocr_text != "",
+            (FileRecord.embedding.is_(None)) | (FileRecord.embedding == ""),
+        )
+        .all()
+    )
+
+    queued = 0
+    for f in candidates:
+        try:
+            from app.tasks.compute_embedding import compute_document_embedding
+
+            compute_document_embedding.delay(f.id)
+            queued += 1
+        except Exception as e:
+            logger.warning(f"Could not queue embedding for file {f.id}: {e}")
+
+    return {
+        "status": "queued",
+        "files_queued": queued,
+    }
+
+
+@router.get("/similarity/pairs")
+@require_login
+def get_similarity_pairs(
+    request: Request,
+    db: DbSession,
+    threshold: float = Query(0.7, ge=0.0, le=1.0, description="Minimum similarity score for a pair"),
+    limit: int = Query(50, ge=1, le=200, description="Maximum number of pairs to return"),
+    page: int = Query(1, ge=1, description="Page number"),
+):
+    """Return pairs of documents with high similarity across the entire corpus.
+
+    Unlike the per-file ``/files/{id}/similar`` endpoint, this scans every
+    document that has a pre-computed embedding and returns **all** pairs
+    whose cosine similarity exceeds ``threshold``, sorted by descending
+    score.
+
+    To keep memory bounded the query loads only the columns needed for
+    scoring and streams results in chunks.
+
+    Response:
+    ```json
+    {
+      "pairs": [
+        {
+          "file_a": {"file_id": 1, "original_filename": "invoice_jan.pdf", ...},
+          "file_b": {"file_id": 5, "original_filename": "invoice_feb.pdf", ...},
+          "similarity_score": 0.94
+        }
+      ],
+      "total_pairs": 12,
+      "threshold": 0.7,
+      "page": 1,
+      "pages": 1,
+      "embedding_coverage": {"total_files": 120, "files_with_embedding": 95}
+    }
+    ```
+    """
+    from app.utils.similarity import cosine_similarity
+
+    # Load all files that have embeddings (columns only for efficiency)
+    rows = (
+        db.query(
+            FileRecord.id,
+            FileRecord.original_filename,
+            FileRecord.document_title,
+            FileRecord.mime_type,
+            FileRecord.created_at,
+            FileRecord.embedding,
+        )
+        .filter(
+            FileRecord.embedding.isnot(None),
+            FileRecord.embedding != "",
+        )
+        .order_by(FileRecord.id)
+        .all()
+    )
+
+    # Parse embeddings upfront
+    parsed: list[tuple] = []
+    for row in rows:
+        try:
+            vec = json.loads(row.embedding)
+            parsed.append((row, vec))
+        except (json.JSONDecodeError, TypeError):
+            continue
+
+    # Pairwise comparison (triangle: i < j avoids duplicating A↔B / B↔A)
+    all_pairs: list[dict] = []
+    for i in range(len(parsed)):
+        row_a, vec_a = parsed[i]
+        for j in range(i + 1, len(parsed)):
+            row_b, vec_b = parsed[j]
+            score = cosine_similarity(vec_a, vec_b)
+            if score >= threshold:
+                all_pairs.append(
+                    {
+                        "file_a": _row_to_dict(row_a),
+                        "file_b": _row_to_dict(row_b),
+                        "similarity_score": round(score, 4),
+                    }
+                )
+
+    # Sort by score descending
+    all_pairs.sort(key=lambda p: p["similarity_score"], reverse=True)
+
+    total_pairs = len(all_pairs)
+    total_pages = max(1, (total_pairs + limit - 1) // limit)
+    offset = (page - 1) * limit
+    page_pairs = all_pairs[offset : offset + limit]
+
+    total_files = db.query(FileRecord).count()
+
+    return {
+        "pairs": page_pairs,
+        "total_pairs": total_pairs,
+        "threshold": threshold,
+        "page": page,
+        "pages": total_pages,
+        "per_page": limit,
+        "embedding_coverage": {
+            "total_files": total_files,
+            "files_with_embedding": len(parsed),
+        },
+    }
+
+
+def _row_to_dict(row) -> dict:
+    """Serialise a column-only query row to a dict for JSON responses."""
+    return {
+        "file_id": row.id,
+        "original_filename": row.original_filename,
+        "document_title": row.document_title,
+        "mime_type": row.mime_type,
+        "created_at": row.created_at.isoformat() if row.created_at else None,
+    }

app/celery_worker.py

@@ -9,6 +9,7 @@ from app import tasks  # noqa: F401 - Imports app/tasks.py so Celery can registe
 from app.celery_app import celery
 from app.config import settings
 from app.tasks.check_credentials import check_credentials
+from app.tasks.compute_embedding import backfill_missing_embeddings, compute_document_embedding  # noqa: F401
 from app.tasks.convert_to_pdf import convert_to_pdf  # noqa: F401
 from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf  # noqa: F401
 from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt  # noqa: F401
@@ -91,6 +92,13 @@ celery.conf.beat_schedule = {
         "schedule": crontab(minute="*/1"),  # Every minute
         "options": {"expires": 55},  # Must complete within 55 seconds
     },
+    # Backfill embeddings for files that were processed before the
+    # embedding pipeline was enabled, or where the embedding task failed.
+    "backfill-missing-embeddings": {
+        "task": "backfill_missing_embeddings",
+        "schedule": crontab(minute="*/5"),  # Every 5 minutes
+        "options": {"expires": 240},  # 4 minutes expiry
+    },
 }
 
 # Remove None entries from beat_schedule

app/config.py

@@ -386,6 +386,29 @@ class Settings(BaseSettings):
             "them near-duplicates.  Higher values require closer content matches.  Default: 0.85."
         ),
     )
+    embedding_model: str = Field(
+        default="text-embedding-3-small",
+        description=(
+            "Model name used for generating text embeddings via the OpenAI-compatible API.  "
+            "Embeddings drive the document similarity feature.  Default: text-embedding-3-small."
+        ),
+    )
+    embedding_max_tokens: int = Field(
+        default=8000,
+        description=(
+            "Maximum number of tokens to send to the embedding model.  "
+            "Text is truncated to approximately this many tokens (using a "
+            "conservative 3-chars-per-token estimate) before calling the API.  "
+            "Set this below the model's context window (e.g. 8000 for an 8192-token model)."
+        ),
+    )
+    embedding_backfill_batch_size: int = Field(
+        default=50,
+        description=(
+            "Maximum number of files to queue for embedding computation per "
+            "backfill run.  Keeps the worker and embedding API load bounded."
+        ),
+    )
 
     # Text quality check - AI-based assessment of embedded PDF text
     enable_text_quality_check: bool = Field(

app/tasks/compute_embedding.py

@@ -0,0 +1,174 @@
+"""Celery task for pre-computing document text embeddings.
+
+Runs after document processing to ensure embeddings are available for
+the similarity feature without requiring a user to trigger them on first
+access.
+"""
+
+import logging
+from datetime import datetime, timezone
+
+from app.celery_app import celery
+from app.config import settings
+from app.database import SessionLocal
+from app.models import FileRecord
+from app.tasks.retry_config import BaseTaskWithRetry
+from app.utils import log_task_progress
+from app.utils.step_manager import update_step_status
+
+logger = logging.getLogger(__name__)
+
+
+@celery.task(base=BaseTaskWithRetry, bind=True, name="compute_document_embedding")
+def compute_document_embedding(self, file_id: int) -> dict:
+    """Compute and cache the text embedding for a single document.
+
+    Skips silently when the file has no OCR text or already has a cached
+    embedding.  The result is stored in ``FileRecord.embedding`` for
+    subsequent similarity queries.
+
+    Args:
+        file_id: Primary key of the :class:`~app.models.FileRecord`.
+
+    Returns:
+        A dict with ``status`` (``"success"`` / ``"skipped"`` / ``"error"``)
+        and optional ``detail`` message.
+    """
+    task_id = self.request.id
+    logger.info("[%s] Computing embedding for file %s", task_id, file_id)
+    log_task_progress(
+        task_id,
+        "compute_embedding",
+        "in_progress",
+        f"Computing text embedding for file {file_id}",
+        file_id=file_id,
+    )
+
+    with SessionLocal() as db:
+        file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
+        if not file_record:
+            logger.warning("[%s] File %s not found, skipping embedding", task_id, file_id)
+            return {"status": "skipped", "detail": "File not found"}
+
+        now = datetime.now(timezone.utc)
+        update_step_status(db, file_id, "compute_embedding", "in_progress", started_at=now)
+
+        # Already has a cached embedding – nothing to do
+        if file_record.embedding:
+            logger.info("[%s] File %s already has a cached embedding", task_id, file_id)
+            log_task_progress(
+                task_id,
+                "compute_embedding",
+                "success",
+                "Embedding already cached",
+                file_id=file_id,
+            )
+            update_step_status(db, file_id, "compute_embedding", "success", completed_at=now)
+            return {"status": "skipped", "detail": "Embedding already cached"}
+
+        if not file_record.ocr_text or not file_record.ocr_text.strip():
+            logger.info("[%s] File %s has no OCR text, skipping embedding", task_id, file_id)
+            log_task_progress(
+                task_id,
+                "compute_embedding",
+                "skipped",
+                "No OCR text available",
+                file_id=file_id,
+            )
+            update_step_status(db, file_id, "compute_embedding", "skipped", completed_at=now)
+            return {"status": "skipped", "detail": "No OCR text available"}
+
+        try:
+            from app.utils.similarity import compute_and_store_embedding
+
+            embedding = compute_and_store_embedding(db, file_record)
+            completed = datetime.now(timezone.utc)
+            if embedding:
+                log_task_progress(
+                    task_id,
+                    "compute_embedding",
+                    "success",
+                    f"Embedding computed ({len(embedding)} dimensions)",
+                    file_id=file_id,
+                )
+                update_step_status(db, file_id, "compute_embedding", "success", completed_at=completed)
+                return {
+                    "status": "success",
+                    "detail": f"Embedding computed ({len(embedding)} dimensions)",
+                }
+            else:
+                log_task_progress(
+                    task_id,
+                    "compute_embedding",
+                    "failure",
+                    "Embedding computation returned None",
+                    file_id=file_id,
+                )
+                update_step_status(
+                    db,
+                    file_id,
+                    "compute_embedding",
+                    "failure",
+                    error_message="Embedding computation returned None",
+                    completed_at=completed,
+                )
+                return {"status": "error", "detail": "Embedding computation returned None"}
+        except Exception as exc:
+            logger.exception("[%s] Embedding computation failed for file %s: %s", task_id, file_id, exc)
+            log_task_progress(
+                task_id,
+                "compute_embedding",
+                "failure",
+                f"Exception: {exc}",
+                file_id=file_id,
+            )
+            update_step_status(
+                db,
+                file_id,
+                "compute_embedding",
+                "failure",
+                error_message=str(exc),
+                completed_at=datetime.now(timezone.utc),
+            )
+            return {"status": "error", "detail": str(exc)}
+
+
+@celery.task(bind=True, name="backfill_missing_embeddings")
+def backfill_missing_embeddings(self) -> dict:
+    """Periodic task that computes embeddings for documents that lack them.
+
+    Iterates over all ``FileRecord`` rows that have OCR text but no
+    cached embedding and queues a :func:`compute_document_embedding`
+    task for each one.  A configurable ``batch_size`` caps the number
+    of tasks queued per run to avoid overwhelming the worker or the
+    embedding API.
+
+    Returns:
+        A dict with the number of tasks ``queued``.
+    """
+    batch_size = settings.embedding_backfill_batch_size
+    task_id = self.request.id
+    logger.info("[%s] Backfill: scanning for files missing embeddings (batch_size=%d)", task_id, batch_size)
+
+    with SessionLocal() as db:
+        candidates = (
+            db.query(FileRecord.id)
+            .filter(
+                FileRecord.ocr_text.isnot(None),
+                FileRecord.ocr_text != "",
+                (FileRecord.embedding.is_(None)) | (FileRecord.embedding == ""),
+            )
+            .limit(batch_size)
+            .all()
+        )
+
+    queued = 0
+    for (file_id,) in candidates:
+        try:
+            compute_document_embedding.delay(file_id)
+            queued += 1
+        except Exception as exc:
+            logger.warning("[%s] Could not queue embedding for file %s: %s", task_id, file_id, exc)
+
+    logger.info("[%s] Backfill: queued %d embedding tasks", task_id, queued)
+    return {"queued": queued}

app/tasks/finalize_document_storage.py

@@ -32,7 +32,7 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
     task_id = self.request.id
     logger.info(f"[{task_id}] Finalizing document storage for {processed_file}")
 
-    # 1. Update Database Status (From Main)
+    # 1. Update Database Status
     log_task_progress(
         task_id,
         "finalize_document_storage",
@@ -41,7 +41,7 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
         file_id=file_id,
     )
 
-    # Get file_id from database if not provided (fallback logic from Main)
+    # Get file_id from database if not provided (fallback logic)
     if file_id is None:
         with SessionLocal() as db:
             # Only as a last resort, try to find by exact match on local_filename
@@ -50,7 +50,7 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
             if file_record:
                 file_id = file_record.id
 
-    # 2. Determine Configured Destinations (From Copilot)
+    # 2. Determine Configured Destinations
     # This is needed for the notification message later
     configured_destinations = []
     try:
@@ -65,43 +65,53 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
         logger.warning(f"[WARNING] Could not determine configured destinations: {e}")
         configured_destinations = ["configured destinations"]
 
-    # 3. Queue Uploads (Merged)
-    # Uses Main branch signature to ensure file_id is passed, but keeps logic structure
+    # 3. Queue Uploads
     logger.info(f"[{task_id}] Queueing uploads to all destinations")
     log_task_progress(
         task_id, "finalize_document_storage", "success", "Queuing uploads to destinations", file_id=file_id
     )
 
     # Note: send_to_all_destinations is asynchronous and queues upload tasks
-    # We pass 'True' (delete_after) and 'file_id' as per Main branch requirements
     send_to_all_destinations.delay(processed_file, True, file_id)
 
-    # 3a. Trigger PDF/A archival conversion if enabled
+    # 3a. Trigger PDF/A archival conversion if enabled (from feature branch)
     if settings.enable_pdfa_conversion:
-        from app.tasks.convert_to_pdfa import convert_to_pdfa
+        try:
+            from app.tasks.convert_to_pdfa import convert_to_pdfa
+            logger.info(f"[{task_id}] PDF/A conversion enabled, queueing archival conversion")
+            log_task_progress(
+                task_id,
+                "finalize_document_storage",
+                "in_progress",
+                "Queueing PDF/A archival conversion",
+                file_id=file_id,
+            )
+            convert_to_pdfa.delay(file_id)
+        except Exception as e:
+            logger.warning(f"[{task_id}] Could not queue PDF/A conversion: {e}")
 
-        logger.info(f"[{task_id}] PDF/A conversion enabled, queueing archival conversion")
-        log_task_progress(
-            task_id,
-            "finalize_document_storage",
-            "in_progress",
-            "Queueing PDF/A archival conversion",
-            file_id=file_id,
-        )
-        convert_to_pdfa.delay(file_id)
+    # 3b. Queue embedding computation (from main branch)
+    if file_id is not None:
+        try:
+            from app.tasks.compute_embedding import compute_document_embedding
+            compute_document_embedding.delay(file_id)
+            logger.info(f"[{task_id}] Queued embedding computation for file {file_id}")
+        except Exception as e:
+            logger.warning(f"[{task_id}] Could not queue embedding task: {e}")
 
-    # 4. Send Notification (From Copilot)
-    # Note: This notification is sent after processing is complete but while uploads
-    # are being queued.
+    # 4. Send Notification
     try:
         # Get file information
         file_size = os.path.getsize(processed_file) if os.path.exists(processed_file) else 0
         filename = os.path.basename(processed_file)
 
         notify_file_processed(
-            filename=filename, file_size=file_size, metadata=metadata, destinations=configured_destinations
+            filename=filename, 
+            file_size=file_size, 
+            metadata=metadata, 
+            destinations=configured_destinations
         )
     except Exception as e:
         logger.warning(f"[WARNING] Failed to send file processed notification: {e}")
 
-    return {"status": "Completed", "file": processed_file}
+    return {"status": "Completed", "file": processed_file}

app/utils/similarity.py

@@ -38,13 +38,15 @@ def _get_embedding_client() -> Any:
     )
 
 
-def generate_embedding(text: str, model: str = "text-embedding-3-small") -> list[float]:
+def generate_embedding(text: str, model: str | None = None) -> list[float]:
     """Generate a text embedding vector using the OpenAI-compatible API.
 
     Args:
-        text: The input text to embed.  Truncated to ~8000 tokens worth of
-            characters to stay within model limits.
-        model: The embedding model to use.  Defaults to ``text-embedding-3-small``.
+        text: The input text to embed.  Truncated to stay within the
+            model's context window based on ``settings.embedding_max_tokens``
+            (default 8 000 tokens ≈ 24 000 characters).
+        model: The embedding model to use.  When ``None`` (the default), the
+            value of ``settings.embedding_model`` is used.
 
     Returns:
         A list of floats representing the embedding vector.
@@ -53,12 +55,25 @@ def generate_embedding(text: str, model: str = "text-embedding-3-small") -> list
         RuntimeError: If the OpenAI client cannot be created.
         Exception: If the API call fails.
     """
-    # Truncate very long texts to stay within token limits (~4 chars per token)
-    max_chars = 30000
+    if model is None:
+        model = settings.embedding_model
+
+    # Truncate to stay within the model's context window.
+    # Conservative 3 chars/token estimate (actual ratio varies by language;
+    # English averages ~4 chars/token but 3 gives a safety margin).
+    max_chars = settings.embedding_max_tokens * 3
     if len(text) > max_chars:
+        logger.debug(
+            "Truncating text from %d to %d chars (~%d tokens) for model %s",
+            len(text),
+            max_chars,
+            settings.embedding_max_tokens,
+            model,
+        )
         text = text[:max_chars]
 
     client = _get_embedding_client()
+    logger.debug("Generating embedding for %d chars using model=%s", len(text), model)
     response = client.embeddings.create(input=text, model=model)
     return response.data[0].embedding
 
@@ -89,8 +104,41 @@ def cosine_similarity(vec_a: list[float], vec_b: list[float]) -> float:
     return max(0.0, min(1.0, similarity))
 
 
-def _get_or_compute_embedding(db: Session, file_record: Any) -> list[float] | None:
-    """Retrieve a cached embedding or compute and store a new one.
+def _get_cached_embedding(file_record: Any) -> list[float] | None:
+    """Return the cached embedding for a file record, or ``None``.
+
+    This is a **read-only** helper — it never triggers an API call.  Use
+    :func:`compute_and_store_embedding` when you need to generate a new
+    embedding.
+
+    Args:
+        file_record: A ``FileRecord`` instance (or any object with ``id``
+            and ``embedding`` attributes).
+
+    Returns:
+        The parsed embedding vector, or ``None`` if no valid cached
+        embedding exists.
+    """
+    raw = file_record.embedding if hasattr(file_record, "embedding") else None
+    if not raw:
+        return None
+    try:
+        cached = json.loads(raw)
+        logger.debug("Using cached embedding for file %s (%d dimensions)", file_record.id, len(cached))
+        return cached
+    except (json.JSONDecodeError, TypeError):
+        logger.warning("Invalid cached embedding for file %s", file_record.id)
+        return None
+
+
+def compute_and_store_embedding(db: Session, file_record: Any) -> list[float] | None:
+    """Generate an embedding for a file and persist it in the database.
+
+    Called during document ingestion (Celery task) or via the manual
+    ``POST /api/files/{id}/compute-embedding`` debug endpoint.  The
+    similarity query path (:func:`find_similar_documents`) intentionally
+    does **not** call this — it only reads pre-computed embeddings so
+    that it returns instantly without blocking on external API calls.
 
     Args:
         db: Active database session.
@@ -100,40 +148,54 @@ def _get_or_compute_embedding(db: Session, file_record: Any) -> list[float] | No
         The embedding vector, or ``None`` if the document has no OCR text
         or embedding generation fails.
     """
-    # Return cached embedding if available
+    # Return cached embedding if already present
     if file_record.embedding:
         try:
-            return json.loads(file_record.embedding)
+            cached = json.loads(file_record.embedding)
+            logger.debug("Embedding already cached for file %s (%d dims)", file_record.id, len(cached))
+            return cached
         except (json.JSONDecodeError, TypeError):
-            logger.warning(f"Invalid cached embedding for file {file_record.id}, recomputing")
+            logger.warning("Invalid cached embedding for file %s, recomputing", file_record.id)
 
     # Need OCR text to generate an embedding
     if not file_record.ocr_text or not file_record.ocr_text.strip():
+        logger.debug("No OCR text for file %s, cannot generate embedding", file_record.id)
         return None
 
     try:
+        logger.info("Computing embedding for file %s (%d chars of OCR text)", file_record.id, len(file_record.ocr_text))
         embedding = generate_embedding(file_record.ocr_text)
-        # Cache the embedding in the database
+        # Persist in the database
         file_record.embedding = json.dumps(embedding)
         db.commit()
+        logger.info("Embedding computed and cached for file %s (%d dimensions)", file_record.id, len(embedding))
         return embedding
     except Exception as e:
         db.rollback()
-        logger.error(f"Failed to generate embedding for file {file_record.id}: {e}")
+        logger.error("Failed to generate embedding for file %s: %s", file_record.id, e)
         return None
 
 
+# Keep the legacy alias so that existing callers (e.g. tests) keep working.
+_get_or_compute_embedding = compute_and_store_embedding
+
+
 def find_similar_documents(
     db: Session,
     file_id: int,
     limit: int = 5,
     threshold: float = 0.3,
 ) -> list[dict[str, Any]]:
-    """Find documents similar to the given file.
+    """Find documents similar to the given file using **pre-computed** embeddings.
 
-    Computes cosine similarity between the target document's embedding and
-    all other documents that have OCR text.  Results are sorted by descending
-    similarity score.
+    Only documents whose embeddings were already generated (during
+    ingestion or via the debug endpoint) are considered.  No external API
+    calls are made — the function reads cached vectors from the database
+    and computes cosine similarity in-process.
+
+    To keep memory usage bounded for large corpora (100 k+ documents) the
+    candidate query fetches only the columns needed for scoring and
+    iterates in chunks via ``yield_per``.
 
     Args:
         db: Active database session.
@@ -152,43 +214,54 @@ def find_similar_documents(
     """
     from app.models import FileRecord
 
-    # Get the target document
+    # Get the target document's cached embedding (read-only, no API call)
     target = db.query(FileRecord).filter(FileRecord.id == file_id).first()
     if not target:
         return []
 
-    # Get the target embedding
-    target_embedding = _get_or_compute_embedding(db, target)
+    target_embedding = _get_cached_embedding(target)
     if not target_embedding:
+        logger.info("No cached embedding for target file %s — skipping similarity search", file_id)
         return []
 
-    # Get candidate documents (those with OCR text, excluding the target)
+    # Query only candidates that already have a pre-computed embedding.
+    # Fetch only the columns needed for scoring to minimise memory use.
+    # yield_per streams rows in chunks so we never materialise all 100k+
+    # records at once.
     candidates = (
-        db.query(FileRecord)
+        db.query(
+            FileRecord.id,
+            FileRecord.original_filename,
+            FileRecord.document_title,
+            FileRecord.mime_type,
+            FileRecord.created_at,
+            FileRecord.embedding,
+        )
         .filter(
             FileRecord.id != file_id,
-            FileRecord.ocr_text.isnot(None),
-            FileRecord.ocr_text != "",
+            FileRecord.embedding.isnot(None),
+            FileRecord.embedding != "",
         )
-        .all()
+        .yield_per(500)
     )
 
-    results = []
-    for candidate in candidates:
-        candidate_embedding = _get_or_compute_embedding(db, candidate)
-        if not candidate_embedding:
+    results: list[dict[str, Any]] = []
+    for row in candidates:
+        try:
+            candidate_embedding: list[float] = json.loads(row.embedding)
+        except (json.JSONDecodeError, TypeError):
             continue
 
         score = cosine_similarity(target_embedding, candidate_embedding)
         if score >= threshold:
             results.append(
                 {
-                    "file_id": candidate.id,
-                    "original_filename": candidate.original_filename,
-                    "document_title": candidate.document_title,
+                    "file_id": row.id,
+                    "original_filename": row.original_filename,
+                    "document_title": row.document_title,
                     "similarity_score": round(score, 4),
-                    "mime_type": candidate.mime_type,
-                    "created_at": candidate.created_at.isoformat() if candidate.created_at else None,
+                    "mime_type": row.mime_type,
+                    "created_at": row.created_at.isoformat() if row.created_at else None,
                 }
             )
 

app/utils/step_manager.py

@@ -23,6 +23,7 @@ BASE_MAIN_PROCESSING_STEPS = [
     "embed_metadata_into_pdf",
     "finalize_document_storage",
     "send_to_all_destinations",
+    "compute_embedding",
 ]
 
 OPTIONAL_PROCESSING_STEPS = {
@@ -217,6 +218,7 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict:
         "embed_metadata_into_pdf",
         "finalize_document_storage",
         "send_to_all_destinations",
+        "compute_embedding",
     }
 
     # Add check_for_duplicates if deduplication is enabled
@@ -325,6 +327,7 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
         "embed_metadata_into_pdf",
         "finalize_document_storage",
         "send_to_all_destinations",
+        "compute_embedding",
     }
 
     # Add check_for_duplicates if deduplication is enabled

app/views/files.py

@@ -389,8 +389,12 @@ def _compute_processing_flow(logs):
         },
         "extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]},
         "embed_metadata_into_pdf": {"label": "Embed Metadata into PDF", "next": ["finalize_document_storage"]},
-        "finalize_document_storage": {"label": "Finalize & Queue Distribution", "next": ["send_to_all_destinations"]},
+        "finalize_document_storage": {
+            "label": "Finalize & Queue Distribution",
+            "next": ["send_to_all_destinations", "compute_embedding"],
+        },
         "send_to_all_destinations": {"label": "Upload to Destinations", "next": [], "has_branches": True},
+        "compute_embedding": {"label": "Compute Embedding", "next": []},
     }
 
     # Filter out deduplication step if not enabled or if not showing it
@@ -815,3 +819,54 @@ def duplicates_page(
                 "error": str(e),
             },
         )
+
+
+@router.get("/similarity")
+@require_login
+def similarity_dashboard_page(
+    request: Request,
+    db: Session = Depends(get_db),
+):
+    """Render the corpus-wide similarity dashboard.
+
+    Passes the configured threshold and embedding coverage stats so the
+    template can display them immediately while the JS fetches the actual
+    pairs from the API asynchronously.
+    """
+    from app.config import settings
+    from app.models import FileRecord
+
+    try:
+        total_files = db.query(FileRecord).count()
+        files_with_embedding = (
+            db.query(FileRecord).filter(FileRecord.embedding.isnot(None), FileRecord.embedding != "").count()
+        )
+        files_with_ocr = db.query(FileRecord).filter(FileRecord.ocr_text.isnot(None), FileRecord.ocr_text != "").count()
+
+        return templates.TemplateResponse(
+            "similarity_dashboard.html",
+            {
+                "request": request,
+                "default_threshold": settings.near_duplicate_threshold,
+                "embedding_model": settings.embedding_model,
+                "total_files": total_files,
+                "files_with_embedding": files_with_embedding,
+                "files_with_ocr": files_with_ocr,
+                "files_missing_embedding": files_with_ocr - files_with_embedding,
+            },
+        )
+    except Exception as e:
+        logger.error(f"Error rendering similarity dashboard: {e}")
+        return templates.TemplateResponse(
+            "similarity_dashboard.html",
+            {
+                "request": request,
+                "default_threshold": 0.85,
+                "embedding_model": "text-embedding-3-small",
+                "total_files": 0,
+                "files_with_embedding": 0,
+                "files_with_ocr": 0,
+                "files_missing_embedding": 0,
+                "error": str(e),
+            },
+        )