diff --git a/app/api/similarity.py b/app/api/similarity.py index 26bb7dc6..e7b5f844 100644 --- a/app/api/similarity.py +++ b/app/api/similarity.py @@ -257,7 +257,17 @@ def get_embeddings_overview( } ``` """ - all_files = db.query(FileRecord).order_by(FileRecord.id.desc()).all() + # Use column-only query to avoid loading full ORM objects into memory + all_files = ( + db.query( + FileRecord.id, + FileRecord.original_filename, + FileRecord.ocr_text, + FileRecord.embedding, + ) + .order_by(FileRecord.id.desc()) + .all() + ) files_info = [] total_with_ocr = 0 diff --git a/app/config.py b/app/config.py index df5e60bc..91f2237d 100644 --- a/app/config.py +++ b/app/config.py @@ -339,6 +339,13 @@ class Settings(BaseSettings): "Set this below the model's context window (e.g. 8000 for an 8192-token model)." ), ) + embedding_backfill_batch_size: int = Field( + default=50, + description=( + "Maximum number of files to queue for embedding computation per " + "backfill run. Keeps the worker and embedding API load bounded." + ), + ) # Text quality check - AI-based assessment of embedded PDF text enable_text_quality_check: bool = Field( diff --git a/app/tasks/compute_embedding.py b/app/tasks/compute_embedding.py index fedc6585..50fc3cd6 100644 --- a/app/tasks/compute_embedding.py +++ b/app/tasks/compute_embedding.py @@ -9,6 +9,7 @@ import logging from datetime import datetime, timezone from app.celery_app import celery +from app.config import settings from app.database import SessionLocal from app.models import FileRecord from app.tasks.retry_config import BaseTaskWithRetry @@ -145,7 +146,7 @@ def backfill_missing_embeddings(self) -> dict: Returns: A dict with the number of tasks ``queued``. """ - batch_size = 50 # max files to queue per run + batch_size = settings.embedding_backfill_batch_size task_id = self.request.id logger.info("[%s] Backfill: scanning for files missing embeddings (batch_size=%d)", task_id, batch_size) diff --git a/app/utils/similarity.py b/app/utils/similarity.py index b5948719..590084e1 100644 --- a/app/utils/similarity.py +++ b/app/utils/similarity.py @@ -59,8 +59,8 @@ def generate_embedding(text: str, model: str | None = None) -> list[float]: model = settings.embedding_model # Truncate to stay within the model's context window. - # Use a conservative estimate of ~3 characters per token so that the - # resulting text fits comfortably within ``embedding_max_tokens``. + # Conservative 3 chars/token estimate (actual ratio varies by language; + # English averages ~4 chars/token but 3 gives a safety margin). max_chars = settings.embedding_max_tokens * 3 if len(text) > max_chars: logger.debug( diff --git a/frontend/templates/similarity_dashboard.html b/frontend/templates/similarity_dashboard.html index 1fdf2c9a..1090ce3a 100644 --- a/frontend/templates/similarity_dashboard.html +++ b/frontend/templates/similarity_dashboard.html @@ -133,7 +133,7 @@ {{ files_missing_embedding }} file(s) have OCR text but no embedding yet. The background task will compute them automatically every 5 minutes, or you can .