Merge pull request #377 from christianlouis/copilot/add-status-page-ai-check-button
feat(status): Add AI extraction test tool to /status page
This commit is contained in:
Christian Krakau-Louis
+133
-3
@@ -1,14 +1,20 @@
|
||||
"""
|
||||
AI provider and OpenAI API endpoints.
|
||||
|
||||
Exposes two endpoints:
|
||||
- GET /api/ai/test – tests the currently configured AI provider (generic, provider-agnostic)
|
||||
- GET /api/openai/test – backward-compatible alias that tests the OpenAI API specifically
|
||||
Exposes three endpoints:
|
||||
- GET /api/ai/test – tests the currently configured AI provider (generic, provider-agnostic)
|
||||
- GET /api/openai/test – backward-compatible alias that tests the OpenAI API specifically
|
||||
- POST /api/ai/test-extraction – runs the metadata-extraction prompt against the configured AI provider
|
||||
with caller-supplied plaintext and returns the raw response, parsed JSON,
|
||||
and extracted tags so operators can evaluate model quality.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
@@ -18,6 +24,10 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# Maximum number of characters accepted for a test-extraction request.
|
||||
# Keeps individual requests reasonable without blocking any real-world document.
|
||||
_MAX_EXTRACTION_TEXT_LEN = 50_000
|
||||
|
||||
|
||||
def _get_exception_chain_detail(exc: Exception) -> str:
|
||||
"""
|
||||
@@ -202,3 +212,123 @@ async def test_ai_provider_connection(request: Request):
|
||||
"message": f"Connection failed: {detail}",
|
||||
"provider": provider_name,
|
||||
}
|
||||
|
||||
|
||||
class ExtractionTestRequest(BaseModel):
|
||||
"""Request body for the AI extraction test endpoint."""
|
||||
|
||||
text: str = Field(..., min_length=1, max_length=_MAX_EXTRACTION_TEXT_LEN, description="Plain-text document content")
|
||||
|
||||
|
||||
def _build_extraction_prompt(text: str) -> str:
|
||||
"""Return the metadata-extraction prompt used in the standard processing pipeline."""
|
||||
return (
|
||||
"You are a specialized document analyzer trained to extract structured metadata from documents.\n"
|
||||
"Your task is to analyze the given text and return a well-structured JSON object.\n\n"
|
||||
"Extract and return the following fields:\n"
|
||||
"1. **filename**: Machine-readable filename "
|
||||
"(YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, periods, and underscores).\n"
|
||||
'2. **empfaenger**: The recipient, or "Unknown" if not found.\n'
|
||||
'3. **absender**: The sender, or "Unknown" if not found.\n'
|
||||
"4. **correspondent**: The entity or company that issued the document "
|
||||
'(shortest possible name, e.g., "Amazon" instead of "Amazon EU SARL, German branch").\n'
|
||||
"5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, "
|
||||
"Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].\n"
|
||||
"6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, "
|
||||
"Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, "
|
||||
"Private_Korrespondenz, Sonstige_Informationen].\n"
|
||||
"7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).\n"
|
||||
"8. **tags**: A list of up to 4 relevant thematic keywords.\n"
|
||||
'9. **language**: Detected document language (ISO 639-1 code, e.g., "de" or "en").\n'
|
||||
"10. **title**: A human-readable title summarizing the document content.\n"
|
||||
"11. **confidence_score**: A numeric value (0-100) indicating the confidence level "
|
||||
"of the extracted metadata.\n"
|
||||
"12. **reference_number**: Extracted invoice/order/reference number if available.\n"
|
||||
"13. **monetary_amounts**: A list of key monetary values detected in the document.\n\n"
|
||||
"### Important Rules:\n"
|
||||
"- **OCR Correction**: Assume the text has been corrected for OCR errors.\n"
|
||||
"- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.\n"
|
||||
"- **Title**: Concise, no addresses, and contains key identifying features.\n"
|
||||
"- **Date Selection**: Use the most relevant date if multiple are found.\n"
|
||||
"- **Output Language**: Maintain the document's original language.\n\n"
|
||||
f"Extracted text:\n{text}\n\n"
|
||||
"Return only valid JSON with no additional commentary.\n"
|
||||
)
|
||||
|
||||
|
||||
def _extract_json_from_text(text: str):
|
||||
"""Try to extract a JSON object from the LLM response text."""
|
||||
pattern = r"```(?:json)?\s*(\{.*?\})\s*```"
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1)
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start != -1 and end != -1 and end > start:
|
||||
return text[start : end + 1]
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/ai/test-extraction")
|
||||
@require_login
|
||||
async def test_ai_extraction(body: ExtractionTestRequest, request: Request):
|
||||
"""
|
||||
Run the metadata-extraction prompt against the configured AI provider.
|
||||
|
||||
Accepts plain-text document content, sends it through the same prompt used
|
||||
by the background processing pipeline, and returns:
|
||||
- ``raw_response``: verbatim LLM output
|
||||
- ``parsed_json``: the extracted JSON object (null when parsing fails)
|
||||
- ``tags``: the ``tags`` list from the parsed JSON (empty list on failure)
|
||||
- ``provider`` / ``model``: which provider / model was used
|
||||
"""
|
||||
from app.utils.ai_provider import get_ai_provider
|
||||
|
||||
provider_name = settings.ai_provider
|
||||
model = settings.ai_model or settings.openai_model
|
||||
|
||||
logger.info(f"AI extraction test requested: provider={provider_name}, model={model}")
|
||||
|
||||
try:
|
||||
provider = get_ai_provider()
|
||||
prompt = _build_extraction_prompt(body.text)
|
||||
raw_response = provider.chat_completion(
|
||||
messages=[
|
||||
{"role": "system", "content": "You are an intelligent document classifier."},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
model=model,
|
||||
temperature=0,
|
||||
)
|
||||
except ValueError as e:
|
||||
logger.warning(f"AI extraction test – configuration error: {e}")
|
||||
return {"status": "error", "message": str(e), "provider": provider_name}
|
||||
except Exception as e:
|
||||
detail = _get_exception_chain_detail(e)
|
||||
logger.error(f"AI extraction test failed for provider '{provider_name}': {detail}", exc_info=True)
|
||||
return {"status": "error", "message": f"AI call failed: {detail}", "provider": provider_name}
|
||||
|
||||
# Attempt to parse JSON from the response
|
||||
parsed_json = None
|
||||
tags: list = []
|
||||
parse_error = None
|
||||
json_text = _extract_json_from_text(raw_response)
|
||||
if json_text:
|
||||
try:
|
||||
parsed_json = json.loads(json_text)
|
||||
tags = parsed_json.get("tags", [])
|
||||
except json.JSONDecodeError as exc:
|
||||
parse_error = str(exc)
|
||||
logger.warning(f"AI extraction test: JSON parse error: {exc}")
|
||||
else:
|
||||
parse_error = "No JSON object found in response"
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"provider": provider_name,
|
||||
"model": model,
|
||||
"raw_response": raw_response,
|
||||
"parsed_json": parsed_json,
|
||||
"tags": tags,
|
||||
"parse_error": parse_error,
|
||||
}
|
||||
|
||||
@@ -181,6 +181,12 @@
|
||||
data-provider="ai_provider">
|
||||
Test Connection
|
||||
</button>
|
||||
<button
|
||||
id="testAiExtractionBtn"
|
||||
class="inline-flex items-center px-2.5 py-1.5 border border-transparent text-xs font-medium rounded text-indigo-700 bg-indigo-100 hover:bg-indigo-200 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500">
|
||||
<i class="fa-solid fa-flask mr-1"></i>
|
||||
Test Extraction
|
||||
</button>
|
||||
{% endif %}
|
||||
{% elif name == "Azure AI" %}
|
||||
{% if provider.configured %}
|
||||
@@ -273,6 +279,85 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- AI Extraction Test Modal -->
|
||||
<div id="aiExtractionModal" class="fixed inset-0 bg-gray-600 bg-opacity-50 hidden overflow-y-auto h-full w-full z-50" aria-modal="true" role="dialog">
|
||||
<div class="relative top-10 mx-auto p-5 border w-11/12 md:w-3/4 lg:w-2/3 xl:w-1/2 shadow-lg rounded-md bg-white">
|
||||
<div class="absolute top-0 right-0 pt-4 pr-4">
|
||||
<button type="button" id="closeAiExtractionModal" class="text-gray-400 hover:text-gray-500">
|
||||
<span class="sr-only">Close</span>
|
||||
<i class="fa-solid fa-xmark h-6 w-6"></i>
|
||||
</button>
|
||||
</div>
|
||||
<div class="mt-3">
|
||||
<h3 class="text-lg leading-6 font-medium text-gray-900 mb-1">
|
||||
<i class="fa-solid fa-flask mr-2 text-indigo-600"></i>AI Extraction Test
|
||||
</h3>
|
||||
<p class="text-sm text-gray-500 mb-4">
|
||||
Paste the plain-text content of a document below and run it through the configured AI provider
|
||||
to inspect the raw response, extracted JSON, and tags.
|
||||
</p>
|
||||
|
||||
<!-- Input area -->
|
||||
<div id="aiExtractionInput">
|
||||
<label for="aiExtractionText" class="block text-sm font-medium text-gray-700 mb-1">Document Text</label>
|
||||
<textarea
|
||||
id="aiExtractionText"
|
||||
rows="10"
|
||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm text-sm font-mono focus:outline-none focus:ring-indigo-500 focus:border-indigo-500"
|
||||
placeholder="Paste the plain-text content of your document here…"></textarea>
|
||||
<div class="mt-3 flex justify-end space-x-2">
|
||||
<button type="button" id="cancelAiExtractionBtn"
|
||||
class="px-4 py-2 border border-gray-300 text-sm font-medium rounded-md text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500">
|
||||
Cancel
|
||||
</button>
|
||||
<button type="button" id="runAiExtractionBtn"
|
||||
class="inline-flex items-center px-4 py-2 border border-transparent text-sm font-medium rounded-md shadow-sm text-white bg-indigo-600 hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500">
|
||||
<i class="fa-solid fa-play mr-2"></i>Run Extraction
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Results area (hidden until extraction completes) -->
|
||||
<div id="aiExtractionResults" class="hidden mt-4 space-y-4">
|
||||
<!-- Provider / model badge -->
|
||||
<div id="aiExtractionMeta" class="text-xs text-gray-500"></div>
|
||||
|
||||
<!-- Parse warning -->
|
||||
<div id="aiParseWarning" class="hidden bg-yellow-50 border-l-4 border-yellow-400 p-3 text-sm text-yellow-800"></div>
|
||||
|
||||
<!-- Raw response -->
|
||||
<div>
|
||||
<h4 class="text-sm font-medium text-gray-900 mb-1">Raw LLM Response</h4>
|
||||
<pre id="aiRawResponse" class="bg-gray-50 border border-gray-200 rounded p-3 text-xs overflow-auto max-h-48 whitespace-pre-wrap break-words"></pre>
|
||||
</div>
|
||||
|
||||
<!-- Parsed JSON -->
|
||||
<div id="aiParsedJsonSection">
|
||||
<h4 class="text-sm font-medium text-gray-900 mb-1">Parsed JSON</h4>
|
||||
<pre id="aiParsedJson" class="bg-gray-50 border border-gray-200 rounded p-3 text-xs overflow-auto max-h-64 whitespace-pre-wrap break-words"></pre>
|
||||
</div>
|
||||
|
||||
<!-- Tags -->
|
||||
<div id="aiTagsSection">
|
||||
<h4 class="text-sm font-medium text-gray-900 mb-1">Extracted Tags</h4>
|
||||
<div id="aiTags" class="flex flex-wrap gap-2"></div>
|
||||
</div>
|
||||
|
||||
<div class="flex justify-end space-x-2 pt-2">
|
||||
<button type="button" id="aiExtractionBackBtn"
|
||||
class="px-4 py-2 border border-gray-300 text-sm font-medium rounded-md text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500">
|
||||
<i class="fa-solid fa-arrow-left mr-1"></i>Back
|
||||
</button>
|
||||
<button type="button" id="aiExtractionCloseBtn"
|
||||
class="px-4 py-2 bg-indigo-600 text-white text-sm font-medium rounded-md shadow-sm hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-600">
|
||||
Close
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
@@ -573,6 +658,152 @@ document.addEventListener('DOMContentLoaded', function() {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ── AI Extraction Test Modal ──────────────────────────────────────────────
|
||||
|
||||
const aiExtractionModal = document.getElementById('aiExtractionModal');
|
||||
const aiExtractionText = document.getElementById('aiExtractionText');
|
||||
const aiExtractionInput = document.getElementById('aiExtractionInput');
|
||||
const aiExtractionResults = document.getElementById('aiExtractionResults');
|
||||
const aiExtractionMeta = document.getElementById('aiExtractionMeta');
|
||||
const aiRawResponse = document.getElementById('aiRawResponse');
|
||||
const aiParsedJson = document.getElementById('aiParsedJson');
|
||||
const aiParsedJsonSection = document.getElementById('aiParsedJsonSection');
|
||||
const aiTags = document.getElementById('aiTags');
|
||||
const aiTagsSection = document.getElementById('aiTagsSection');
|
||||
const aiParseWarning = document.getElementById('aiParseWarning');
|
||||
const testAiExtractionBtn = document.getElementById('testAiExtractionBtn');
|
||||
const runAiExtractionBtn = document.getElementById('runAiExtractionBtn');
|
||||
const cancelAiExtractionBtn= document.getElementById('cancelAiExtractionBtn');
|
||||
const aiExtractionBackBtn = document.getElementById('aiExtractionBackBtn');
|
||||
const aiExtractionCloseBtn = document.getElementById('aiExtractionCloseBtn');
|
||||
const closeAiExtractionModal = document.getElementById('closeAiExtractionModal');
|
||||
|
||||
function openAiExtractionModal() {
|
||||
// Reset to input view
|
||||
aiExtractionText.value = '';
|
||||
aiExtractionInput.classList.remove('hidden');
|
||||
aiExtractionResults.classList.add('hidden');
|
||||
aiExtractionModal.classList.remove('hidden');
|
||||
}
|
||||
|
||||
function closeAiExtractionModalFn() {
|
||||
aiExtractionModal.classList.add('hidden');
|
||||
}
|
||||
|
||||
function showAiExtractionResults(data) {
|
||||
// Provider / model meta
|
||||
aiExtractionMeta.textContent = `Provider: ${data.provider || '—'} | Model: ${data.model || '—'}`;
|
||||
|
||||
// Raw response
|
||||
aiRawResponse.textContent = data.raw_response || '(empty)';
|
||||
|
||||
// Parse warning
|
||||
if (data.parse_error) {
|
||||
aiParseWarning.textContent = `JSON parse issue: ${data.parse_error}`;
|
||||
aiParseWarning.classList.remove('hidden');
|
||||
} else {
|
||||
aiParseWarning.classList.add('hidden');
|
||||
}
|
||||
|
||||
// Parsed JSON
|
||||
if (data.parsed_json) {
|
||||
aiParsedJson.textContent = JSON.stringify(data.parsed_json, null, 2);
|
||||
aiParsedJsonSection.classList.remove('hidden');
|
||||
} else {
|
||||
aiParsedJsonSection.classList.add('hidden');
|
||||
}
|
||||
|
||||
// Tags
|
||||
aiTags.innerHTML = '';
|
||||
const tags = Array.isArray(data.tags) ? data.tags : [];
|
||||
if (tags.length > 0) {
|
||||
tags.forEach(tag => {
|
||||
const span = document.createElement('span');
|
||||
span.className = 'inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium bg-indigo-100 text-indigo-800';
|
||||
span.textContent = tag;
|
||||
aiTags.appendChild(span);
|
||||
});
|
||||
aiTagsSection.classList.remove('hidden');
|
||||
} else {
|
||||
aiTagsSection.classList.add('hidden');
|
||||
}
|
||||
|
||||
// Switch view
|
||||
aiExtractionInput.classList.add('hidden');
|
||||
aiExtractionResults.classList.remove('hidden');
|
||||
}
|
||||
|
||||
if (testAiExtractionBtn) {
|
||||
testAiExtractionBtn.addEventListener('click', openAiExtractionModal);
|
||||
}
|
||||
|
||||
if (closeAiExtractionModal) {
|
||||
closeAiExtractionModal.addEventListener('click', closeAiExtractionModalFn);
|
||||
}
|
||||
|
||||
if (cancelAiExtractionBtn) {
|
||||
cancelAiExtractionBtn.addEventListener('click', closeAiExtractionModalFn);
|
||||
}
|
||||
|
||||
if (aiExtractionCloseBtn) {
|
||||
aiExtractionCloseBtn.addEventListener('click', closeAiExtractionModalFn);
|
||||
}
|
||||
|
||||
if (aiExtractionBackBtn) {
|
||||
aiExtractionBackBtn.addEventListener('click', function() {
|
||||
aiExtractionResults.classList.add('hidden');
|
||||
aiExtractionInput.classList.remove('hidden');
|
||||
});
|
||||
}
|
||||
|
||||
// Close when clicking the backdrop
|
||||
if (aiExtractionModal) {
|
||||
aiExtractionModal.addEventListener('click', function(e) {
|
||||
if (e.target === aiExtractionModal) {
|
||||
closeAiExtractionModalFn();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (runAiExtractionBtn) {
|
||||
runAiExtractionBtn.addEventListener('click', function() {
|
||||
const text = aiExtractionText.value.trim();
|
||||
if (!text) {
|
||||
aiExtractionText.focus();
|
||||
return;
|
||||
}
|
||||
|
||||
const originalHTML = runAiExtractionBtn.innerHTML;
|
||||
runAiExtractionBtn.innerHTML = '<i class="fa-solid fa-spinner fa-spin mr-2"></i>Running…';
|
||||
runAiExtractionBtn.disabled = true;
|
||||
cancelAiExtractionBtn.disabled = true;
|
||||
|
||||
fetch('/api/ai/test-extraction', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ text: text }),
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
if (data.status === 'success') {
|
||||
showAiExtractionResults(data);
|
||||
} else {
|
||||
closeAiExtractionModalFn();
|
||||
showModal('error', 'AI Extraction Failed', data.message || 'Unknown error');
|
||||
}
|
||||
})
|
||||
.catch(error => {
|
||||
closeAiExtractionModalFn();
|
||||
showModal('error', 'Connection Error', 'Error running extraction: ' + error.message);
|
||||
})
|
||||
.finally(() => {
|
||||
runAiExtractionBtn.innerHTML = originalHTML;
|
||||
runAiExtractionBtn.disabled = false;
|
||||
cancelAiExtractionBtn.disabled = false;
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
@@ -309,3 +309,363 @@ class TestOpenAIConnectionErrors:
|
||||
|
||||
assert data["status"] == "error"
|
||||
assert data.get("is_auth_error") is False
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestAiExtractionEndpoint:
|
||||
"""Tests for POST /api/ai/test-extraction endpoint."""
|
||||
|
||||
@patch("app.api.openai.settings")
|
||||
def test_extraction_missing_text_returns_422(self, mock_settings, client):
|
||||
"""Test that missing text body returns 422 validation error."""
|
||||
response = client.post("/api/ai/test-extraction", json={})
|
||||
assert response.status_code == 422
|
||||
|
||||
@patch("app.api.openai.settings")
|
||||
def test_extraction_empty_text_returns_422(self, mock_settings, client):
|
||||
"""Test that empty string text returns 422 validation error."""
|
||||
response = client.post("/api/ai/test-extraction", json={"text": ""})
|
||||
assert response.status_code == 422
|
||||
|
||||
@patch("app.utils.ai_provider.get_ai_provider")
|
||||
@patch("app.api.openai.settings")
|
||||
def test_extraction_returns_raw_response_and_parsed_json(self, mock_settings, mock_get_provider, client):
|
||||
"""Test successful extraction returns raw response, parsed JSON, and tags."""
|
||||
import json as json_module
|
||||
|
||||
mock_settings.ai_provider = "openai"
|
||||
mock_settings.ai_model = "gpt-4o-mini"
|
||||
mock_settings.openai_model = "gpt-4o-mini"
|
||||
|
||||
expected_json = {
|
||||
"filename": "2024-01-01_Invoice",
|
||||
"tags": ["invoice", "payment"],
|
||||
"title": "January Invoice",
|
||||
"document_type": "Invoice",
|
||||
}
|
||||
raw = "```json\n" + json_module.dumps(expected_json) + "\n```"
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.chat_completion.return_value = raw
|
||||
mock_get_provider.return_value = mock_provider
|
||||
|
||||
response = client.post("/api/ai/test-extraction", json={"text": "Invoice content here"})
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert data["status"] == "success"
|
||||
assert data["raw_response"] == raw
|
||||
assert data["parsed_json"]["filename"] == "2024-01-01_Invoice"
|
||||
assert data["tags"] == ["invoice", "payment"]
|
||||
assert data["parse_error"] is None
|
||||
assert data["provider"] == "openai"
|
||||
assert data["model"] == "gpt-4o-mini"
|
||||
|
||||
@patch("app.utils.ai_provider.get_ai_provider")
|
||||
@patch("app.api.openai.settings")
|
||||
def test_extraction_handles_invalid_json_in_response(self, mock_settings, mock_get_provider, client):
|
||||
"""Test that invalid JSON in LLM response is reported via parse_error."""
|
||||
mock_settings.ai_provider = "openai"
|
||||
mock_settings.ai_model = "gpt-4o-mini"
|
||||
mock_settings.openai_model = "gpt-4o-mini"
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.chat_completion.return_value = "Sorry, I cannot help with that."
|
||||
mock_get_provider.return_value = mock_provider
|
||||
|
||||
response = client.post("/api/ai/test-extraction", json={"text": "Some document text"})
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert data["status"] == "success"
|
||||
assert data["parsed_json"] is None
|
||||
assert data["tags"] == []
|
||||
assert data["parse_error"] is not None
|
||||
|
||||
@patch("app.utils.ai_provider.get_ai_provider")
|
||||
@patch("app.api.openai.settings")
|
||||
def test_extraction_provider_config_error(self, mock_settings, mock_get_provider, client):
|
||||
"""Test that configuration errors (missing keys) are returned as error status."""
|
||||
mock_settings.ai_provider = "anthropic"
|
||||
mock_settings.ai_model = "claude-3"
|
||||
mock_settings.openai_model = "gpt-4o-mini"
|
||||
|
||||
mock_get_provider.side_effect = ValueError("ANTHROPIC_API_KEY must be set")
|
||||
|
||||
response = client.post("/api/ai/test-extraction", json={"text": "Some document text"})
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert data["status"] == "error"
|
||||
assert "ANTHROPIC_API_KEY" in data["message"]
|
||||
|
||||
@patch("app.utils.ai_provider.get_ai_provider")
|
||||
@patch("app.api.openai.settings")
|
||||
def test_extraction_provider_runtime_error(self, mock_settings, mock_get_provider, client):
|
||||
"""Test that runtime errors during AI call are returned as error status."""
|
||||
mock_settings.ai_provider = "openai"
|
||||
mock_settings.ai_model = "gpt-4o-mini"
|
||||
mock_settings.openai_model = "gpt-4o-mini"
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.chat_completion.side_effect = Exception("Connection refused")
|
||||
mock_get_provider.return_value = mock_provider
|
||||
|
||||
response = client.post("/api/ai/test-extraction", json={"text": "Some document text"})
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert data["status"] == "error"
|
||||
assert "Connection refused" in data["message"]
|
||||
|
||||
@patch("app.utils.ai_provider.get_ai_provider")
|
||||
@patch("app.api.openai.settings")
|
||||
def test_extraction_plain_json_without_code_fences(self, mock_settings, mock_get_provider, client):
|
||||
"""Test that JSON returned without code fences is still parsed correctly."""
|
||||
import json as json_module
|
||||
|
||||
mock_settings.ai_provider = "openai"
|
||||
mock_settings.ai_model = "gpt-4o-mini"
|
||||
mock_settings.openai_model = "gpt-4o-mini"
|
||||
|
||||
expected_json = {"tags": ["contract"], "title": "Service Agreement"}
|
||||
raw = json_module.dumps(expected_json)
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.chat_completion.return_value = raw
|
||||
mock_get_provider.return_value = mock_provider
|
||||
|
||||
response = client.post("/api/ai/test-extraction", json={"text": "Contract content"})
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert data["status"] == "success"
|
||||
assert data["tags"] == ["contract"]
|
||||
assert data["parse_error"] is None
|
||||
|
||||
@patch("app.utils.ai_provider.get_ai_provider")
|
||||
@patch("app.api.openai.settings")
|
||||
def test_extraction_json_found_but_invalid_reports_parse_error(self, mock_settings, mock_get_provider, client):
|
||||
"""Test JSONDecodeError branch: response contains '{...}' but is not valid JSON."""
|
||||
mock_settings.ai_provider = "openai"
|
||||
mock_settings.ai_model = "gpt-4o-mini"
|
||||
mock_settings.openai_model = "gpt-4o-mini"
|
||||
|
||||
# Looks like JSON (has { and }) but is NOT parseable
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.chat_completion.return_value = "{this is not: valid json!!}"
|
||||
mock_get_provider.return_value = mock_provider
|
||||
|
||||
response = client.post("/api/ai/test-extraction", json={"text": "Some document text"})
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert data["status"] == "success"
|
||||
assert data["parsed_json"] is None
|
||||
assert data["tags"] == []
|
||||
assert data["parse_error"] is not None
|
||||
assert "raw_response" in data
|
||||
|
||||
@patch("app.api.openai.settings")
|
||||
def test_extraction_text_too_long_returns_422(self, mock_settings, client):
|
||||
"""Test that text exceeding max length returns 422 validation error."""
|
||||
from app.api.openai import _MAX_EXTRACTION_TEXT_LEN
|
||||
|
||||
oversized_text = "x" * (_MAX_EXTRACTION_TEXT_LEN + 1)
|
||||
response = client.post("/api/ai/test-extraction", json={"text": oversized_text})
|
||||
assert response.status_code == 422
|
||||
|
||||
@patch("app.utils.ai_provider.get_ai_provider")
|
||||
@patch("app.api.openai.settings")
|
||||
def test_extraction_response_with_no_tags_key(self, mock_settings, mock_get_provider, client):
|
||||
"""Test extraction where parsed JSON has no 'tags' key returns empty tags list."""
|
||||
import json as json_module
|
||||
|
||||
mock_settings.ai_provider = "openai"
|
||||
mock_settings.ai_model = "gpt-4o-mini"
|
||||
mock_settings.openai_model = "gpt-4o-mini"
|
||||
|
||||
# Valid JSON but no 'tags' key
|
||||
payload = {"title": "Report", "document_type": "Report"}
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.chat_completion.return_value = json_module.dumps(payload)
|
||||
mock_get_provider.return_value = mock_provider
|
||||
|
||||
response = client.post("/api/ai/test-extraction", json={"text": "Report content"})
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert data["status"] == "success"
|
||||
assert data["tags"] == []
|
||||
assert data["parsed_json"]["title"] == "Report"
|
||||
assert data["parse_error"] is None
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestAiProviderTestEndpoint:
|
||||
"""Tests for GET /api/ai/test endpoint."""
|
||||
|
||||
@patch("app.utils.ai_provider.get_ai_provider")
|
||||
@patch("app.api.openai.settings")
|
||||
def test_ai_test_success(self, mock_settings, mock_get_provider, client):
|
||||
"""Test successful AI provider connection."""
|
||||
mock_settings.ai_provider = "openai"
|
||||
mock_settings.ai_model = "gpt-4o-mini"
|
||||
mock_settings.openai_model = "gpt-4o-mini"
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.chat_completion.return_value = "ok"
|
||||
mock_get_provider.return_value = mock_provider
|
||||
|
||||
response = client.get("/api/ai/test")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert data["status"] == "success"
|
||||
assert "reachable" in data["message"].lower()
|
||||
assert data["provider"] == "openai"
|
||||
assert data["model"] == "gpt-4o-mini"
|
||||
assert "response_preview" in data
|
||||
|
||||
@patch("app.utils.ai_provider.get_ai_provider")
|
||||
@patch("app.api.openai.settings")
|
||||
def test_ai_test_value_error(self, mock_settings, mock_get_provider, client):
|
||||
"""Test GET /api/ai/test returns error on provider configuration ValueError."""
|
||||
mock_settings.ai_provider = "anthropic"
|
||||
mock_settings.ai_model = None
|
||||
mock_settings.openai_model = "gpt-4o-mini"
|
||||
|
||||
mock_get_provider.side_effect = ValueError("ANTHROPIC_API_KEY must be set")
|
||||
|
||||
response = client.get("/api/ai/test")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert data["status"] == "error"
|
||||
assert "ANTHROPIC_API_KEY" in data["message"]
|
||||
assert data["provider"] == "anthropic"
|
||||
|
||||
@patch("app.utils.ai_provider.get_ai_provider")
|
||||
@patch("app.api.openai.settings")
|
||||
def test_ai_test_connection_exception(self, mock_settings, mock_get_provider, client):
|
||||
"""Test GET /api/ai/test returns error on provider runtime exception."""
|
||||
mock_settings.ai_provider = "openai"
|
||||
mock_settings.ai_model = "gpt-4o-mini"
|
||||
mock_settings.openai_model = "gpt-4o-mini"
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.chat_completion.side_effect = Exception("Connection refused")
|
||||
mock_get_provider.return_value = mock_provider
|
||||
|
||||
response = client.get("/api/ai/test")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert data["status"] == "error"
|
||||
assert "Connection refused" in data["message"]
|
||||
assert data["provider"] == "openai"
|
||||
|
||||
@patch("app.utils.ai_provider.get_ai_provider")
|
||||
@patch("app.api.openai.settings")
|
||||
def test_ai_test_uses_openai_model_fallback(self, mock_settings, mock_get_provider, client):
|
||||
"""Test that ai_model=None falls back to openai_model."""
|
||||
mock_settings.ai_provider = "openai"
|
||||
mock_settings.ai_model = None
|
||||
mock_settings.openai_model = "gpt-3.5-turbo"
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.chat_completion.return_value = "ok"
|
||||
mock_get_provider.return_value = mock_provider
|
||||
|
||||
response = client.get("/api/ai/test")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert data["status"] == "success"
|
||||
assert data["model"] == "gpt-3.5-turbo"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestExceptionChainDetail:
|
||||
"""Unit tests for the _get_exception_chain_detail helper."""
|
||||
|
||||
def test_single_exception_no_chain(self):
|
||||
"""Test with a simple exception that has no cause."""
|
||||
from app.api.openai import _get_exception_chain_detail
|
||||
|
||||
exc = ValueError("root cause")
|
||||
result = _get_exception_chain_detail(exc)
|
||||
assert result == "root cause"
|
||||
|
||||
def test_exception_with_cause(self):
|
||||
"""Test that chained exceptions are surfaced in the message."""
|
||||
from app.api.openai import _get_exception_chain_detail
|
||||
|
||||
inner = OSError("DNS resolution failed")
|
||||
outer = ConnectionError("Connection failed")
|
||||
outer.__cause__ = inner
|
||||
|
||||
result = _get_exception_chain_detail(outer)
|
||||
assert "Connection failed" in result
|
||||
assert "DNS resolution failed" in result
|
||||
assert "caused by" in result
|
||||
|
||||
def test_empty_cause_string_skipped(self):
|
||||
"""Test that a cause with empty string representation is not appended."""
|
||||
from app.api.openai import _get_exception_chain_detail
|
||||
|
||||
inner = Exception("") # str() returns ""
|
||||
outer = RuntimeError("outer error")
|
||||
outer.__cause__ = inner
|
||||
|
||||
result = _get_exception_chain_detail(outer)
|
||||
# The empty-string cause should be skipped (branch 47->49)
|
||||
assert result == "outer error"
|
||||
|
||||
def test_duplicate_cause_string_skipped(self):
|
||||
"""Test that a cause whose str() is already in parts is not duplicated."""
|
||||
from app.api.openai import _get_exception_chain_detail
|
||||
|
||||
outer = RuntimeError("same message")
|
||||
inner = RuntimeError("same message") # same text as outer
|
||||
outer.__cause__ = inner
|
||||
|
||||
result = _get_exception_chain_detail(outer)
|
||||
# "same message" should appear only once (branch: cause_str already in parts)
|
||||
assert result.count("same message") == 1
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestExtractJsonFromText:
|
||||
"""Unit tests for the _extract_json_from_text helper."""
|
||||
|
||||
def test_json_in_code_fence(self):
|
||||
"""Test extraction from markdown code fence."""
|
||||
from app.api.openai import _extract_json_from_text
|
||||
|
||||
text = '```json\n{"key": "value"}\n```'
|
||||
result = _extract_json_from_text(text)
|
||||
assert result == '{"key": "value"}'
|
||||
|
||||
def test_json_in_plain_code_fence(self):
|
||||
"""Test extraction from plain (non-json) code fence."""
|
||||
from app.api.openai import _extract_json_from_text
|
||||
|
||||
text = '```\n{"key": "value"}\n```'
|
||||
result = _extract_json_from_text(text)
|
||||
assert result == '{"key": "value"}'
|
||||
|
||||
def test_bare_json_object(self):
|
||||
"""Test extraction of a bare JSON object without code fence."""
|
||||
from app.api.openai import _extract_json_from_text
|
||||
|
||||
text = 'Here is the result: {"title": "Invoice"} done.'
|
||||
result = _extract_json_from_text(text)
|
||||
assert result == '{"title": "Invoice"}'
|
||||
|
||||
def test_no_json_returns_none(self):
|
||||
"""Test that text with no JSON object returns None."""
|
||||
from app.api.openai import _extract_json_from_text
|
||||
|
||||
result = _extract_json_from_text("No JSON here at all.")
|
||||
assert result is None
|
||||
|
||||
Reference in New Issue
Block a user