fix(similarity): truncate text to fit embedding model context window, fix step tracking

- Add EMBEDDING_MAX_TOKENS config (default 8000) for safe text truncation
- Use conservative 3 chars/token estimate (was 4) to prevent ContextWindowExceededError
- Add compute_embedding to REAL_MAIN_STEPS in both get_file_overall_status and get_step_summary
- Fix test_near_duplicates_returned to use pre-computed embeddings
- Update .env.demo and docs with EMBEDDING_MAX_TOKENS setting

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/utils/similarity.py

@@ -42,8 +42,9 @@ def generate_embedding(text: str, model: str | None = None) -> list[float]:
     """Generate a text embedding vector using the OpenAI-compatible API.
 
     Args:
-        text: The input text to embed.  Truncated to ~8000 tokens worth of
-            characters to stay within model limits.
+        text: The input text to embed.  Truncated to stay within the
+            model's context window based on ``settings.embedding_max_tokens``
+            (default 8 000 tokens ≈ 24 000 characters).
         model: The embedding model to use.  When ``None`` (the default), the
             value of ``settings.embedding_model`` is used.
 
@@ -57,9 +58,18 @@ def generate_embedding(text: str, model: str | None = None) -> list[float]:
     if model is None:
         model = settings.embedding_model
 
-    # Truncate very long texts to stay within token limits (~4 chars per token)
-    max_chars = 30000
+    # Truncate to stay within the model's context window.
+    # Use a conservative estimate of ~3 characters per token so that the
+    # resulting text fits comfortably within ``embedding_max_tokens``.
+    max_chars = settings.embedding_max_tokens * 3
     if len(text) > max_chars:
+        logger.debug(
+            "Truncating text from %d to %d chars (~%d tokens) for model %s",
+            len(text),
+            max_chars,
+            settings.embedding_max_tokens,
+            model,
+        )
         text = text[:max_chars]
 
     client = _get_embedding_client()

app/utils/step_manager.py

@@ -218,6 +218,7 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict:
         "embed_metadata_into_pdf",
         "finalize_document_storage",
         "send_to_all_destinations",
+        "compute_embedding",
     }
 
     # Add check_for_duplicates if deduplication is enabled
@@ -326,6 +327,7 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
         "embed_metadata_into_pdf",
         "finalize_document_storage",
         "send_to_all_destinations",
+        "compute_embedding",
     }
 
     # Add check_for_duplicates if deduplication is enabled