diff --git a/app/api/similarity.py b/app/api/similarity.py index 9c14ce1b..ca15b23e 100644 --- a/app/api/similarity.py +++ b/app/api/similarity.py @@ -1,9 +1,11 @@ """Document similarity API endpoints. -Provides an endpoint to find documents similar to a given file based on -text embeddings and cosine similarity scoring. +Provides endpoints to find documents similar to a given file based on +text embeddings and cosine similarity scoring, plus debug/diagnostic +endpoints for inspecting and triggering embedding computation. """ +import json import logging from typing import Annotated @@ -11,6 +13,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from sqlalchemy.orm import Session from app.auth import require_login +from app.config import settings from app.database import get_db from app.models import FileRecord @@ -95,3 +98,236 @@ def get_similar_documents( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to compute document similarity", ) + + +# --------------------------------------------------------------------------- +# Debug / diagnostic endpoints +# --------------------------------------------------------------------------- + + +@router.get("/files/{file_id}/embedding-status") +@require_login +def get_embedding_status( + request: Request, + file_id: int, + db: DbSession, +): + """Return the embedding status for a single file. + + Useful for debugging whether the embedding has been computed + and cached for a given document. + + Response: + ```json + { + "file_id": 42, + "has_embedding": true, + "embedding_dimensions": 1536, + "has_ocr_text": true, + "ocr_text_length": 4200, + "embedding_model": "text-embedding-3-small" + } + ``` + """ + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + if not file_record: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") + + has_embedding = False + embedding_dimensions = None + if file_record.embedding: + try: + parsed = json.loads(file_record.embedding) + has_embedding = True + embedding_dimensions = len(parsed) + except (json.JSONDecodeError, TypeError): + pass + + has_ocr_text = bool(file_record.ocr_text and file_record.ocr_text.strip()) + + return { + "file_id": file_id, + "has_embedding": has_embedding, + "embedding_dimensions": embedding_dimensions, + "has_ocr_text": has_ocr_text, + "ocr_text_length": len(file_record.ocr_text) if file_record.ocr_text else 0, + "embedding_model": settings.embedding_model, + } + + +@router.post("/files/{file_id}/compute-embedding") +@require_login +def trigger_compute_embedding( + request: Request, + file_id: int, + db: DbSession, +): + """Trigger embedding computation for a single file. + + If the file already has a cached embedding it will be recomputed. + The computation happens synchronously so the caller receives the + result immediately. + + Response: + ```json + { + "file_id": 42, + "status": "success", + "embedding_dimensions": 1536 + } + ``` + """ + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + if not file_record: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") + + if not file_record.ocr_text or not file_record.ocr_text.strip(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="File has no OCR text — cannot generate embedding", + ) + + try: + from app.utils.similarity import generate_embedding + + # Clear cached embedding to force recomputation + file_record.embedding = None + db.flush() + + embedding = generate_embedding(file_record.ocr_text) + file_record.embedding = json.dumps(embedding) + db.commit() + + return { + "file_id": file_id, + "status": "success", + "embedding_dimensions": len(embedding), + } + except Exception as e: + db.rollback() + logger.error(f"Failed to compute embedding for file {file_id}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Embedding computation failed: {e}", + ) + + +@router.get("/diagnostic/embeddings") +@require_login +def get_embeddings_overview( + request: Request, + db: DbSession, +): + """Return an overview of embedding status across all files. + + Provides aggregate counts as well as a per-file breakdown so an + administrator can quickly identify documents that are missing + embeddings. + + Response: + ```json + { + "total_files": 120, + "files_with_ocr_text": 95, + "files_with_embedding": 42, + "files_missing_embedding": 53, + "embedding_model": "text-embedding-3-small", + "files": [ + { + "file_id": 1, + "original_filename": "invoice.pdf", + "has_ocr_text": true, + "has_embedding": true, + "embedding_dimensions": 1536 + } + ] + } + ``` + """ + all_files = db.query(FileRecord).order_by(FileRecord.id.desc()).all() + + files_info = [] + total_with_ocr = 0 + total_with_embedding = 0 + + for f in all_files: + has_ocr = bool(f.ocr_text and f.ocr_text.strip()) + has_emb = False + emb_dims = None + + if f.embedding: + try: + parsed = json.loads(f.embedding) + has_emb = True + emb_dims = len(parsed) + except (json.JSONDecodeError, TypeError): + pass + + if has_ocr: + total_with_ocr += 1 + if has_emb: + total_with_embedding += 1 + + files_info.append( + { + "file_id": f.id, + "original_filename": f.original_filename, + "has_ocr_text": has_ocr, + "has_embedding": has_emb, + "embedding_dimensions": emb_dims, + } + ) + + return { + "total_files": len(all_files), + "files_with_ocr_text": total_with_ocr, + "files_with_embedding": total_with_embedding, + "files_missing_embedding": total_with_ocr - total_with_embedding, + "embedding_model": settings.embedding_model, + "files": files_info, + } + + +@router.post("/diagnostic/compute-all-embeddings") +@require_login +def trigger_compute_all_embeddings( + request: Request, + db: DbSession, +): + """Queue embedding computation for all files that have OCR text but no embedding. + + Each file is processed as a separate Celery task so the endpoint + returns immediately. + + Response: + ```json + { + "status": "queued", + "files_queued": 53 + } + ``` + """ + candidates = ( + db.query(FileRecord) + .filter( + FileRecord.ocr_text.isnot(None), + FileRecord.ocr_text != "", + (FileRecord.embedding.is_(None)) | (FileRecord.embedding == ""), + ) + .all() + ) + + queued = 0 + for f in candidates: + try: + from app.tasks.compute_embedding import compute_document_embedding + + compute_document_embedding.delay(f.id) + queued += 1 + except Exception as e: + logger.warning(f"Could not queue embedding for file {f.id}: {e}") + + return { + "status": "queued", + "files_queued": queued, + } diff --git a/app/celery_worker.py b/app/celery_worker.py index d6bd83d4..d04f5916 100644 --- a/app/celery_worker.py +++ b/app/celery_worker.py @@ -9,6 +9,7 @@ from app import tasks # noqa: F401 - Imports app/tasks.py so Celery can registe from app.celery_app import celery from app.config import settings from app.tasks.check_credentials import check_credentials +from app.tasks.compute_embedding import backfill_missing_embeddings, compute_document_embedding # noqa: F401 from app.tasks.convert_to_pdf import convert_to_pdf # noqa: F401 from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf # noqa: F401 from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt # noqa: F401 @@ -91,6 +92,13 @@ celery.conf.beat_schedule = { "schedule": crontab(minute="*/1"), # Every minute "options": {"expires": 55}, # Must complete within 55 seconds }, + # Backfill embeddings for files that were processed before the + # embedding pipeline was enabled, or where the embedding task failed. + "backfill-missing-embeddings": { + "task": "backfill_missing_embeddings", + "schedule": crontab(minute="*/5"), # Every 5 minutes + "options": {"expires": 240}, # 4 minutes expiry + }, } # Remove None entries from beat_schedule diff --git a/app/config.py b/app/config.py index 5a4c2fd1..62d87607 100644 --- a/app/config.py +++ b/app/config.py @@ -323,6 +323,13 @@ class Settings(BaseSettings): "them near-duplicates. Higher values require closer content matches. Default: 0.85." ), ) + embedding_model: str = Field( + default="text-embedding-3-small", + description=( + "Model name used for generating text embeddings via the OpenAI-compatible API. " + "Embeddings drive the document similarity feature. Default: text-embedding-3-small." + ), + ) # Text quality check - AI-based assessment of embedded PDF text enable_text_quality_check: bool = Field( diff --git a/app/tasks/compute_embedding.py b/app/tasks/compute_embedding.py new file mode 100644 index 00000000..0df05175 --- /dev/null +++ b/app/tasks/compute_embedding.py @@ -0,0 +1,148 @@ +"""Celery task for pre-computing document text embeddings. + +Runs after document processing to ensure embeddings are available for +the similarity feature without requiring a user to trigger them on first +access. +""" + +import logging + +from app.celery_app import celery +from app.database import SessionLocal +from app.models import FileRecord +from app.tasks.retry_config import BaseTaskWithRetry +from app.utils import log_task_progress + +logger = logging.getLogger(__name__) + + +@celery.task(base=BaseTaskWithRetry, bind=True, name="compute_document_embedding") +def compute_document_embedding(self, file_id: int) -> dict: + """Compute and cache the text embedding for a single document. + + Skips silently when the file has no OCR text or already has a cached + embedding. The result is stored in ``FileRecord.embedding`` for + subsequent similarity queries. + + Args: + file_id: Primary key of the :class:`~app.models.FileRecord`. + + Returns: + A dict with ``status`` (``"success"`` / ``"skipped"`` / ``"error"``) + and optional ``detail`` message. + """ + task_id = self.request.id + logger.info("[%s] Computing embedding for file %s", task_id, file_id) + log_task_progress( + task_id, + "compute_embedding", + "in_progress", + f"Computing text embedding for file {file_id}", + file_id=file_id, + ) + + with SessionLocal() as db: + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + if not file_record: + logger.warning("[%s] File %s not found, skipping embedding", task_id, file_id) + return {"status": "skipped", "detail": "File not found"} + + # Already has a cached embedding – nothing to do + if file_record.embedding: + logger.info("[%s] File %s already has a cached embedding", task_id, file_id) + log_task_progress( + task_id, + "compute_embedding", + "success", + "Embedding already cached", + file_id=file_id, + ) + return {"status": "skipped", "detail": "Embedding already cached"} + + if not file_record.ocr_text or not file_record.ocr_text.strip(): + logger.info("[%s] File %s has no OCR text, skipping embedding", task_id, file_id) + log_task_progress( + task_id, + "compute_embedding", + "skipped", + "No OCR text available", + file_id=file_id, + ) + return {"status": "skipped", "detail": "No OCR text available"} + + try: + from app.utils.similarity import compute_and_store_embedding + + embedding = compute_and_store_embedding(db, file_record) + if embedding: + log_task_progress( + task_id, + "compute_embedding", + "success", + f"Embedding computed ({len(embedding)} dimensions)", + file_id=file_id, + ) + return { + "status": "success", + "detail": f"Embedding computed ({len(embedding)} dimensions)", + } + else: + log_task_progress( + task_id, + "compute_embedding", + "failure", + "Embedding computation returned None", + file_id=file_id, + ) + return {"status": "error", "detail": "Embedding computation returned None"} + except Exception as exc: + logger.exception("[%s] Embedding computation failed for file %s: %s", task_id, file_id, exc) + log_task_progress( + task_id, + "compute_embedding", + "failure", + f"Exception: {exc}", + file_id=file_id, + ) + return {"status": "error", "detail": str(exc)} + + +@celery.task(bind=True, name="backfill_missing_embeddings") +def backfill_missing_embeddings(self) -> dict: + """Periodic task that computes embeddings for documents that lack them. + + Iterates over all ``FileRecord`` rows that have OCR text but no + cached embedding and queues a :func:`compute_document_embedding` + task for each one. A configurable ``batch_size`` caps the number + of tasks queued per run to avoid overwhelming the worker or the + embedding API. + + Returns: + A dict with the number of tasks ``queued``. + """ + batch_size = 50 # max files to queue per run + task_id = self.request.id + logger.info("[%s] Backfill: scanning for files missing embeddings (batch_size=%d)", task_id, batch_size) + + with SessionLocal() as db: + candidates = ( + db.query(FileRecord.id) + .filter( + FileRecord.ocr_text.isnot(None), + FileRecord.ocr_text != "", + (FileRecord.embedding.is_(None)) | (FileRecord.embedding == ""), + ) + .limit(batch_size) + .all() + ) + + queued = 0 + for (file_id,) in candidates: + try: + compute_document_embedding.delay(file_id) + queued += 1 + except Exception as exc: + logger.warning("[%s] Could not queue embedding for file %s: %s", task_id, file_id, exc) + + logger.info("[%s] Backfill: queued %d embedding tasks", task_id, queued) + return {"queued": queued} diff --git a/app/tasks/finalize_document_storage.py b/app/tasks/finalize_document_storage.py index 1876e73d..228b9b09 100644 --- a/app/tasks/finalize_document_storage.py +++ b/app/tasks/finalize_document_storage.py @@ -76,6 +76,16 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met # We pass 'True' (delete_after) and 'file_id' as per Main branch requirements send_to_all_destinations.delay(processed_file, True, file_id) + # 3a. Queue embedding computation so similarity scores are ready for queries + if file_id is not None: + try: + from app.tasks.compute_embedding import compute_document_embedding + + compute_document_embedding.delay(file_id) + logger.info(f"[{task_id}] Queued embedding computation for file {file_id}") + except Exception as e: + logger.warning(f"[{task_id}] Could not queue embedding task: {e}") + # 4. Send Notification (From Copilot) # Note: This notification is sent after processing is complete but while uploads # are being queued. diff --git a/app/utils/similarity.py b/app/utils/similarity.py index 617f1976..c1e69c15 100644 --- a/app/utils/similarity.py +++ b/app/utils/similarity.py @@ -38,13 +38,14 @@ def _get_embedding_client() -> Any: ) -def generate_embedding(text: str, model: str = "text-embedding-3-small") -> list[float]: +def generate_embedding(text: str, model: str | None = None) -> list[float]: """Generate a text embedding vector using the OpenAI-compatible API. Args: text: The input text to embed. Truncated to ~8000 tokens worth of characters to stay within model limits. - model: The embedding model to use. Defaults to ``text-embedding-3-small``. + model: The embedding model to use. When ``None`` (the default), the + value of ``settings.embedding_model`` is used. Returns: A list of floats representing the embedding vector. @@ -53,12 +54,16 @@ def generate_embedding(text: str, model: str = "text-embedding-3-small") -> list RuntimeError: If the OpenAI client cannot be created. Exception: If the API call fails. """ + if model is None: + model = settings.embedding_model + # Truncate very long texts to stay within token limits (~4 chars per token) max_chars = 30000 if len(text) > max_chars: text = text[:max_chars] client = _get_embedding_client() + logger.debug("Generating embedding for %d chars using model=%s", len(text), model) response = client.embeddings.create(input=text, model=model) return response.data[0].embedding @@ -89,8 +94,41 @@ def cosine_similarity(vec_a: list[float], vec_b: list[float]) -> float: return max(0.0, min(1.0, similarity)) -def _get_or_compute_embedding(db: Session, file_record: Any) -> list[float] | None: - """Retrieve a cached embedding or compute and store a new one. +def _get_cached_embedding(file_record: Any) -> list[float] | None: + """Return the cached embedding for a file record, or ``None``. + + This is a **read-only** helper — it never triggers an API call. Use + :func:`compute_and_store_embedding` when you need to generate a new + embedding. + + Args: + file_record: A ``FileRecord`` instance (or any object with ``id`` + and ``embedding`` attributes). + + Returns: + The parsed embedding vector, or ``None`` if no valid cached + embedding exists. + """ + raw = file_record.embedding if hasattr(file_record, "embedding") else None + if not raw: + return None + try: + cached = json.loads(raw) + logger.debug("Using cached embedding for file %s (%d dimensions)", file_record.id, len(cached)) + return cached + except (json.JSONDecodeError, TypeError): + logger.warning("Invalid cached embedding for file %s", file_record.id) + return None + + +def compute_and_store_embedding(db: Session, file_record: Any) -> list[float] | None: + """Generate an embedding for a file and persist it in the database. + + Called during document ingestion (Celery task) or via the manual + ``POST /api/files/{id}/compute-embedding`` debug endpoint. The + similarity query path (:func:`find_similar_documents`) intentionally + does **not** call this — it only reads pre-computed embeddings so + that it returns instantly without blocking on external API calls. Args: db: Active database session. @@ -100,40 +138,54 @@ def _get_or_compute_embedding(db: Session, file_record: Any) -> list[float] | No The embedding vector, or ``None`` if the document has no OCR text or embedding generation fails. """ - # Return cached embedding if available + # Return cached embedding if already present if file_record.embedding: try: - return json.loads(file_record.embedding) + cached = json.loads(file_record.embedding) + logger.debug("Embedding already cached for file %s (%d dims)", file_record.id, len(cached)) + return cached except (json.JSONDecodeError, TypeError): - logger.warning(f"Invalid cached embedding for file {file_record.id}, recomputing") + logger.warning("Invalid cached embedding for file %s, recomputing", file_record.id) # Need OCR text to generate an embedding if not file_record.ocr_text or not file_record.ocr_text.strip(): + logger.debug("No OCR text for file %s, cannot generate embedding", file_record.id) return None try: + logger.info("Computing embedding for file %s (%d chars of OCR text)", file_record.id, len(file_record.ocr_text)) embedding = generate_embedding(file_record.ocr_text) - # Cache the embedding in the database + # Persist in the database file_record.embedding = json.dumps(embedding) db.commit() + logger.info("Embedding computed and cached for file %s (%d dimensions)", file_record.id, len(embedding)) return embedding except Exception as e: db.rollback() - logger.error(f"Failed to generate embedding for file {file_record.id}: {e}") + logger.error("Failed to generate embedding for file %s: %s", file_record.id, e) return None +# Keep the legacy alias so that existing callers (e.g. tests) keep working. +_get_or_compute_embedding = compute_and_store_embedding + + def find_similar_documents( db: Session, file_id: int, limit: int = 5, threshold: float = 0.3, ) -> list[dict[str, Any]]: - """Find documents similar to the given file. + """Find documents similar to the given file using **pre-computed** embeddings. - Computes cosine similarity between the target document's embedding and - all other documents that have OCR text. Results are sorted by descending - similarity score. + Only documents whose embeddings were already generated (during + ingestion or via the debug endpoint) are considered. No external API + calls are made — the function reads cached vectors from the database + and computes cosine similarity in-process. + + To keep memory usage bounded for large corpora (100 k+ documents) the + candidate query fetches only the columns needed for scoring and + iterates in chunks via ``yield_per``. Args: db: Active database session. @@ -152,43 +204,54 @@ def find_similar_documents( """ from app.models import FileRecord - # Get the target document + # Get the target document's cached embedding (read-only, no API call) target = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not target: return [] - # Get the target embedding - target_embedding = _get_or_compute_embedding(db, target) + target_embedding = _get_cached_embedding(target) if not target_embedding: + logger.info("No cached embedding for target file %s — skipping similarity search", file_id) return [] - # Get candidate documents (those with OCR text, excluding the target) + # Query only candidates that already have a pre-computed embedding. + # Fetch only the columns needed for scoring to minimise memory use. + # yield_per streams rows in chunks so we never materialise all 100k+ + # records at once. candidates = ( - db.query(FileRecord) + db.query( + FileRecord.id, + FileRecord.original_filename, + FileRecord.document_title, + FileRecord.mime_type, + FileRecord.created_at, + FileRecord.embedding, + ) .filter( FileRecord.id != file_id, - FileRecord.ocr_text.isnot(None), - FileRecord.ocr_text != "", + FileRecord.embedding.isnot(None), + FileRecord.embedding != "", ) - .all() + .yield_per(500) ) - results = [] - for candidate in candidates: - candidate_embedding = _get_or_compute_embedding(db, candidate) - if not candidate_embedding: + results: list[dict[str, Any]] = [] + for row in candidates: + try: + candidate_embedding: list[float] = json.loads(row.embedding) + except (json.JSONDecodeError, TypeError): continue score = cosine_similarity(target_embedding, candidate_embedding) if score >= threshold: results.append( { - "file_id": candidate.id, - "original_filename": candidate.original_filename, - "document_title": candidate.document_title, + "file_id": row.id, + "original_filename": row.original_filename, + "document_title": row.document_title, "similarity_score": round(score, 4), - "mime_type": candidate.mime_type, - "created_at": candidate.created_at.isoformat() if candidate.created_at else None, + "mime_type": row.mime_type, + "created_at": row.created_at.isoformat() if row.created_at else None, } ) diff --git a/frontend/templates/file_detail.html b/frontend/templates/file_detail.html index e57cd92d..c9d34f57 100644 --- a/frontend/templates/file_detail.html +++ b/frontend/templates/file_detail.html @@ -952,10 +952,75 @@ loadPDF('processed', fileId); {% endif %} - // Load similar documents + // Load similar documents and embedding status loadSimilarDocuments(fileId); + loadEmbeddingStatus(fileId); }); + // Fetch and display embedding debug info + async function loadEmbeddingStatus(fileId) { + const statusDiv = document.getElementById('embedding-status'); + const actionsDiv = document.getElementById('embedding-actions'); + try { + const response = await fetch(`/api/files/${fileId}/embedding-status`); + if (!response.ok) return; + const data = await response.json(); + + let html = ''; + if (data.has_embedding) { + html = ` Embedding: ${data.embedding_dimensions} dimensions (model: ${data.embedding_model})`; + statusDiv.style.backgroundColor = '#f0fff4'; + statusDiv.style.color = '#276749'; + statusDiv.style.border = '1px solid #c6f6d5'; + } else if (data.has_ocr_text) { + html = ` Embedding: not yet computed — OCR text available (${data.ocr_text_length} chars). Click "Recompute Embedding" to generate.`; + statusDiv.style.backgroundColor = '#fffff0'; + statusDiv.style.color = '#975a16'; + statusDiv.style.border = '1px solid #fefcbf'; + } else { + html = ` Embedding: unavailable — no OCR text extracted for this file.`; + statusDiv.style.backgroundColor = '#fff5f5'; + statusDiv.style.color = '#9b2c2c'; + statusDiv.style.border = '1px solid #fed7d7'; + } + statusDiv.innerHTML = html; + statusDiv.style.display = 'block'; + actionsDiv.style.display = data.has_ocr_text ? 'block' : 'none'; + } catch (error) { + console.error('Error loading embedding status:', error); + } + } + + // Trigger embedding recomputation + async function recomputeEmbedding(fileId) { + const btn = document.getElementById('recompute-embedding-btn'); + btn.disabled = true; + btn.innerHTML = ' Computing…'; + + try { + const response = await fetch(`/api/files/${fileId}/compute-embedding`, { method: 'POST' }); + const data = await response.json(); + if (!response.ok) { + throw new Error(data.detail || 'Request failed'); + } + btn.innerHTML = ' Done'; + btn.style.backgroundColor = '#48bb78'; + // Refresh both embedding status and similar documents + loadEmbeddingStatus(fileId); + loadSimilarDocuments(fileId); + } catch (error) { + btn.innerHTML = ' Failed'; + btn.style.backgroundColor = '#e53e3e'; + console.error('Recompute embedding failed:', error); + } finally { + setTimeout(() => { + btn.disabled = false; + btn.innerHTML = ' Recompute Embedding'; + btn.style.backgroundColor = '#4299e1'; + }, 3000); + } + } + // Similar documents loading async function loadSimilarDocuments(fileId) { const loadingDiv = document.getElementById('similar-documents-loading'); @@ -1602,6 +1667,8 @@

Similar Documents

+ +

Searching for similar documents…

@@ -1615,6 +1682,12 @@

Failed to load similar documents.

+ +
diff --git a/tests/test_similarity.py b/tests/test_similarity.py index b81fd053..c790b51b 100644 --- a/tests/test_similarity.py +++ b/tests/test_similarity.py @@ -443,3 +443,389 @@ class TestSimilarDocumentsAPI: assert "similarity_score" in doc assert "mime_type" in doc assert "created_at" in doc + + +# --------------------------------------------------------------------------- +# Tests for embedding status endpoint +# --------------------------------------------------------------------------- + + +class TestEmbeddingStatusAPI: + """Integration tests for GET /api/files/{file_id}/embedding-status.""" + + @pytest.mark.integration + def test_file_not_found(self, client: TestClient): + """Should return 404 for non-existent file.""" + response = client.get("/api/files/9999/embedding-status") + assert response.status_code == 404 + + @pytest.mark.integration + def test_file_without_embedding_or_ocr(self, client: TestClient, db_session): + """Should report no embedding and no OCR text.""" + file_record = FileRecord( + filehash="abc1", + local_filename="/tmp/test.pdf", + file_size=100, + original_filename="test.pdf", + ) + db_session.add(file_record) + db_session.commit() + + response = client.get(f"/api/files/{file_record.id}/embedding-status") + assert response.status_code == 200 + data = response.json() + assert data["file_id"] == file_record.id + assert data["has_embedding"] is False + assert data["embedding_dimensions"] is None + assert data["has_ocr_text"] is False + assert data["ocr_text_length"] == 0 + assert "embedding_model" in data + + @pytest.mark.integration + def test_file_with_ocr_text_no_embedding(self, client: TestClient, db_session): + """Should report OCR text present but no embedding.""" + file_record = FileRecord( + filehash="abc2", + local_filename="/tmp/test2.pdf", + file_size=100, + original_filename="test2.pdf", + ocr_text="Some OCR text content", + ) + db_session.add(file_record) + db_session.commit() + + response = client.get(f"/api/files/{file_record.id}/embedding-status") + assert response.status_code == 200 + data = response.json() + assert data["has_embedding"] is False + assert data["has_ocr_text"] is True + assert data["ocr_text_length"] == 21 + + @pytest.mark.integration + def test_file_with_cached_embedding(self, client: TestClient, db_session): + """Should report embedding present with correct dimensions.""" + embedding = [0.1, 0.2, 0.3, 0.4, 0.5] + file_record = FileRecord( + filehash="abc3", + local_filename="/tmp/test3.pdf", + file_size=100, + original_filename="test3.pdf", + ocr_text="Some text", + embedding=json.dumps(embedding), + ) + db_session.add(file_record) + db_session.commit() + + response = client.get(f"/api/files/{file_record.id}/embedding-status") + assert response.status_code == 200 + data = response.json() + assert data["has_embedding"] is True + assert data["embedding_dimensions"] == 5 + assert data["has_ocr_text"] is True + + +# --------------------------------------------------------------------------- +# Tests for compute-embedding endpoint +# --------------------------------------------------------------------------- + + +class TestComputeEmbeddingAPI: + """Integration tests for POST /api/files/{file_id}/compute-embedding.""" + + @pytest.mark.integration + def test_file_not_found(self, client: TestClient): + """Should return 404 for non-existent file.""" + response = client.post("/api/files/9999/compute-embedding") + assert response.status_code == 404 + + @pytest.mark.integration + def test_no_ocr_text(self, client: TestClient, db_session): + """Should return 400 when file has no OCR text.""" + file_record = FileRecord( + filehash="emb1", + local_filename="/tmp/emb1.pdf", + file_size=100, + original_filename="emb1.pdf", + ) + db_session.add(file_record) + db_session.commit() + + response = client.post(f"/api/files/{file_record.id}/compute-embedding") + assert response.status_code == 400 + + @pytest.mark.integration + @patch("app.utils.similarity.generate_embedding") + def test_computes_embedding(self, mock_embed, client: TestClient, db_session): + """Should compute and store an embedding.""" + mock_embed.return_value = [0.1, 0.2, 0.3] + + file_record = FileRecord( + filehash="emb2", + local_filename="/tmp/emb2.pdf", + file_size=100, + original_filename="emb2.pdf", + ocr_text="Some document text", + ) + db_session.add(file_record) + db_session.commit() + + response = client.post(f"/api/files/{file_record.id}/compute-embedding") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert data["embedding_dimensions"] == 3 + + # Verify embedding is stored + db_session.refresh(file_record) + assert file_record.embedding is not None + stored = json.loads(file_record.embedding) + assert len(stored) == 3 + + @pytest.mark.integration + @patch("app.utils.similarity.generate_embedding") + def test_recomputes_existing_embedding(self, mock_embed, client: TestClient, db_session): + """Should overwrite existing embedding when recomputing.""" + mock_embed.return_value = [0.9, 0.8, 0.7] + + file_record = FileRecord( + filehash="emb3", + local_filename="/tmp/emb3.pdf", + file_size=100, + original_filename="emb3.pdf", + ocr_text="Some text", + embedding=json.dumps([0.1, 0.2, 0.3]), + ) + db_session.add(file_record) + db_session.commit() + + response = client.post(f"/api/files/{file_record.id}/compute-embedding") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + + db_session.refresh(file_record) + stored = json.loads(file_record.embedding) + assert stored == [0.9, 0.8, 0.7] + + +# --------------------------------------------------------------------------- +# Tests for diagnostic embeddings overview endpoint +# --------------------------------------------------------------------------- + + +class TestEmbeddingsOverviewAPI: + """Integration tests for GET /api/diagnostic/embeddings.""" + + @pytest.mark.integration + def test_empty_database(self, client: TestClient): + """Should return zero counts on empty database.""" + response = client.get("/api/diagnostic/embeddings") + assert response.status_code == 200 + data = response.json() + assert data["total_files"] == 0 + assert data["files_with_ocr_text"] == 0 + assert data["files_with_embedding"] == 0 + assert data["files_missing_embedding"] == 0 + assert "embedding_model" in data + assert data["files"] == [] + + @pytest.mark.integration + def test_mixed_files(self, client: TestClient, db_session): + """Should report correct counts for mixed embedding states.""" + # File with both OCR text and embedding + f1 = FileRecord( + filehash="diag1", + local_filename="/tmp/d1.pdf", + file_size=100, + original_filename="d1.pdf", + ocr_text="Some text", + embedding=json.dumps([0.1, 0.2]), + ) + # File with OCR text but no embedding + f2 = FileRecord( + filehash="diag2", + local_filename="/tmp/d2.pdf", + file_size=100, + original_filename="d2.pdf", + ocr_text="More text", + ) + # File with no OCR text + f3 = FileRecord( + filehash="diag3", + local_filename="/tmp/d3.pdf", + file_size=100, + original_filename="d3.pdf", + ) + db_session.add_all([f1, f2, f3]) + db_session.commit() + + response = client.get("/api/diagnostic/embeddings") + assert response.status_code == 200 + data = response.json() + assert data["total_files"] == 3 + assert data["files_with_ocr_text"] == 2 + assert data["files_with_embedding"] == 1 + assert data["files_missing_embedding"] == 1 + assert len(data["files"]) == 3 + + # Check per-file info + files_by_id = {f["file_id"]: f for f in data["files"]} + assert files_by_id[f1.id]["has_embedding"] is True + assert files_by_id[f1.id]["embedding_dimensions"] == 2 + assert files_by_id[f2.id]["has_embedding"] is False + assert files_by_id[f2.id]["has_ocr_text"] is True + assert files_by_id[f3.id]["has_ocr_text"] is False + + +# --------------------------------------------------------------------------- +# Tests for compute-all-embeddings endpoint +# --------------------------------------------------------------------------- + + +class TestComputeAllEmbeddingsAPI: + """Integration tests for POST /api/diagnostic/compute-all-embeddings.""" + + @pytest.mark.integration + @patch("app.tasks.compute_embedding.compute_document_embedding.delay") + def test_queues_tasks_for_files_missing_embeddings(self, mock_delay, client: TestClient, db_session): + """Should queue embedding tasks for files with OCR text but no embedding.""" + # File with OCR text but no embedding -> should be queued + f1 = FileRecord( + filehash="all1", + local_filename="/tmp/a1.pdf", + file_size=100, + original_filename="a1.pdf", + ocr_text="Text for embedding", + ) + # File already with embedding -> should NOT be queued + f2 = FileRecord( + filehash="all2", + local_filename="/tmp/a2.pdf", + file_size=100, + original_filename="a2.pdf", + ocr_text="More text", + embedding=json.dumps([0.1, 0.2]), + ) + # File without OCR text -> should NOT be queued + f3 = FileRecord( + filehash="all3", + local_filename="/tmp/a3.pdf", + file_size=100, + original_filename="a3.pdf", + ) + db_session.add_all([f1, f2, f3]) + db_session.commit() + + response = client.post("/api/diagnostic/compute-all-embeddings") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "queued" + assert data["files_queued"] == 1 + mock_delay.assert_called_once_with(f1.id) + + @pytest.mark.integration + @patch("app.tasks.compute_embedding.compute_document_embedding.delay") + def test_empty_database_queues_nothing(self, mock_delay, client: TestClient): + """Should queue nothing when database is empty.""" + response = client.post("/api/diagnostic/compute-all-embeddings") + assert response.status_code == 200 + data = response.json() + assert data["files_queued"] == 0 + mock_delay.assert_not_called() + + +# --------------------------------------------------------------------------- +# Tests for compute_document_embedding Celery task +# --------------------------------------------------------------------------- + + +class TestComputeDocumentEmbeddingTask: + """Unit tests for the compute_document_embedding Celery task.""" + + @pytest.mark.unit + @patch("app.utils.similarity.generate_embedding") + def test_computes_embedding_for_file(self, mock_embed, db_session): + """Should compute and store embedding when file has OCR text.""" + mock_embed.return_value = [0.1, 0.2, 0.3] + + file_record = FileRecord( + filehash="task1", + local_filename="/tmp/task1.pdf", + file_size=100, + original_filename="task1.pdf", + ocr_text="Some document text", + ) + db_session.add(file_record) + db_session.commit() + + from app.tasks.compute_embedding import compute_document_embedding + + # Patch SessionLocal to return our test session + with patch("app.tasks.compute_embedding.SessionLocal") as mock_session_local: + mock_session_local.return_value.__enter__ = lambda self: db_session + mock_session_local.return_value.__exit__ = lambda self, *args: None + + result = compute_document_embedding(file_record.id) + + assert result["status"] == "success" + assert "dimensions" in result["detail"] + + @pytest.mark.unit + def test_skips_missing_file(self, db_session): + """Should skip when file ID does not exist.""" + from app.tasks.compute_embedding import compute_document_embedding + + with patch("app.tasks.compute_embedding.SessionLocal") as mock_session_local: + mock_session_local.return_value.__enter__ = lambda self: db_session + mock_session_local.return_value.__exit__ = lambda self, *args: None + + result = compute_document_embedding(9999) + + assert result["status"] == "skipped" + + @pytest.mark.unit + def test_skips_file_without_ocr_text(self, db_session): + """Should skip when file has no OCR text.""" + file_record = FileRecord( + filehash="task2", + local_filename="/tmp/task2.pdf", + file_size=100, + original_filename="task2.pdf", + ) + db_session.add(file_record) + db_session.commit() + + from app.tasks.compute_embedding import compute_document_embedding + + with patch("app.tasks.compute_embedding.SessionLocal") as mock_session_local: + mock_session_local.return_value.__enter__ = lambda self: db_session + mock_session_local.return_value.__exit__ = lambda self, *args: None + + result = compute_document_embedding(file_record.id) + + assert result["status"] == "skipped" + + @pytest.mark.unit + def test_skips_file_with_existing_embedding(self, db_session): + """Should skip when file already has a cached embedding.""" + file_record = FileRecord( + filehash="task3", + local_filename="/tmp/task3.pdf", + file_size=100, + original_filename="task3.pdf", + ocr_text="Some text", + embedding=json.dumps([0.1, 0.2]), + ) + db_session.add(file_record) + db_session.commit() + + from app.tasks.compute_embedding import compute_document_embedding + + with patch("app.tasks.compute_embedding.SessionLocal") as mock_session_local: + mock_session_local.return_value.__enter__ = lambda self: db_session + mock_session_local.return_value.__exit__ = lambda self, *args: None + + result = compute_document_embedding(file_record.id) + + assert result["status"] == "skipped" + assert "already cached" in result["detail"]