gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add OpenAI and Azure AI API endpoints with connection testing functionality

Browse Source
This commit is contained in:
Christian Krakau-Louis
2025-04-09 07:08:38 +02:00
parent d66fef7ad4
commit 2f822adeb7
13 changed files with 308 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/__init__.py
+4
View File
@@ -11,6 +11,8 @@ from app.api.process import router as process_router
from app.api.diagnostic import router as diagnostic_router
from app.api.onedrive import router as onedrive_router
from app.api.dropbox import router as dropbox_router
from app.api.openai import router as openai_router
from app.api.azure import router as azure_router
# Set up logging
logger = logging.getLogger(__name__)
@@ -25,3 +27,5 @@ router.include_router(process_router)
router.include_router(diagnostic_router)
router.include_router(onedrive_router)
router.include_router(dropbox_router)
router.include_router(openai_router)
router.include_router(azure_router)
app/api/azure.py
+124
View File
@@ -0,0 +1,124 @@
"""
Azure AI API endpoints
"""
from fastapi import APIRouter, Request, HTTPException, status
import logging
import os
from app.auth import require_login
from app.config import settings
# Import the Azure modules including the administration client
from azure.core.credentials import AzureKeyCredential
from azure.ai.documentintelligence import DocumentIntelligenceClient
from azure.ai.documentintelligence import DocumentIntelligenceAdministrationClient
import azure.core.exceptions
# Set up logging
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/azure/test")
@require_login
async def test_azure_connection(request: Request):
"""
Test if the configured Azure Document Intelligence connection is valid.
Uses the DocumentIntelligenceAdministrationClient for testing the connection.
"""
try:
logger.info("Testing Azure Document Intelligence connection")
# Check if Azure configuration is present
if not settings.azure_endpoint or not settings.azure_ai_key:
logger.warning("Azure Document Intelligence configuration is incomplete")
missing = []
if not settings.azure_endpoint:
missing.append("endpoint")
if not settings.azure_ai_key:
missing.append("API key")
return {
"status": "error",
"message": f"Azure Document Intelligence configuration is incomplete. Missing: {', '.join(missing)}"
}
# Try to initialize the admin client and make a request to list operations
try:
# Initialize the admin client with credentials
admin_client = DocumentIntelligenceAdministrationClient(
endpoint=settings.azure_endpoint,
credential=AzureKeyCredential(settings.azure_ai_key)
)
# Test the connection by listing operations - this is a documented method in the admin client
operations = list(admin_client.list_operations())
# Successfully initialized client and made a request
logger.info("Azure Document Intelligence Admin connection successfully tested")
# Return success with available operations info
operations_info = []
try:
for op in operations:
if hasattr(op, 'operation_id') and op.operation_id:
op_info = {
"id": op.operation_id,
"status": op.status if hasattr(op, 'status') else "Unknown",
"created": str(op.created_on) if hasattr(op, 'created_on') else "Unknown",
"kind": op.kind if hasattr(op, 'kind') else "Unknown"
}
operations_info.append(op_info)
operation_count = len(operations_info)
return {
"status": "success",
"message": f"Azure Document Intelligence connection is valid. Found {operation_count} operations.",
"endpoint": settings.azure_endpoint,
"operations_count": operation_count,
"recent_operations": operations_info[:3] if operations_info else []
}
except Exception as e:
# If error occurs while processing operations info, still return success
logger.warning(f"Connected to Azure but couldn't parse operations: {e}")
return {
"status": "success",
"message": "Azure Document Intelligence connection is valid, but couldn't retrieve operations details.",
"endpoint": settings.azure_endpoint
}
except azure.core.exceptions.ClientAuthenticationError as e:
logger.error(f"Azure authentication error: {e}")
return {
"status": "error",
"message": f"Authentication error: Invalid API key or credentials",
"detail": str(e)
}
except azure.core.exceptions.ServiceRequestError as e:
logger.error(f"Azure service request error: {e}")
return {
"status": "error",
"message": f"Service request error: Could not reach the Azure endpoint",
"detail": str(e)
}
except ValueError as e:
logger.error(f"Azure configuration value error: {e}")
return {
"status": "error",
"message": f"Configuration error: {str(e)}",
"detail": str(e)
}
except Exception as e:
logger.error(f"Azure connection test failed with unexpected error: {e}")
return {
"status": "error",
"message": f"Connection test failed with unexpected error",
"detail": str(e)
}
except Exception as e:
logger.exception("Unexpected error testing Azure Document Intelligence connection")
return {
"status": "error",
"message": f"Unexpected error: {str(e)}"
}
app/api/diagnostic.py
+3
View File
@@ -31,12 +31,15 @@ async def diagnostic_settings(request: Request, current_user: dict = Depends(get
"email": bool(getattr(settings, 'email_host', None)),
"s3": bool(getattr(settings, 's3_bucket_name', None)),
"dropbox": bool(getattr(settings, 'dropbox_refresh_token', None)),
"onedrive": bool(getattr(settings, 'onedrive_refresh_token', None)),
"nextcloud": bool(getattr(settings, 'nextcloud_upload_url', None)),
"sftp": bool(getattr(settings, 'sftp_host', None)),
"paperless": bool(getattr(settings, 'paperless_host', None)),
"google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)),
"uptime_kuma": bool(getattr(settings, 'uptime_kuma_url', None)),
"auth": bool(getattr(settings, 'authentik_config_url', None)),
"openai": bool(getattr(settings, 'openai_api_key', None)),
"azure": bool(getattr(settings, 'azure_api_key', None) and getattr(settings, 'azure_endpoint', None)),
},
"imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)),
}
app/api/openai.py
+75
View File
@@ -0,0 +1,75 @@
"""
OpenAI API endpoints
"""
from fastapi import APIRouter, Request, HTTPException, status
import logging
import os
import requests
from app.auth import require_login
from app.config import settings
# Set up logging
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/openai/test")
@require_login
async def test_openai_connection(request: Request):
"""
Test if the configured OpenAI API key is valid.
"""
try:
import openai
logger.info("Testing OpenAI API key validity")
# Check if API key is configured
if not settings.openai_api_key:
logger.warning("No OpenAI API key configured")
return {
"status": "error",
"message": "No OpenAI API key is configured"
}
# Configure the client
client = openai.OpenAI(api_key=settings.openai_api_key)
# Try to make a simple request to validate the key
try:
# Use a models list endpoint as a simple validation
models = client.models.list()
# If we got here, the key is valid
logger.info("OpenAI API key is valid")
return {
"status": "success",
"message": "OpenAI API key is valid",
"models_available": len(models.data) if hasattr(models, "data") else "Unknown"
}
except Exception as e:
error_msg = str(e)
logger.error(f"OpenAI API key test failed: {error_msg}")
# Determine if this is an authentication error
is_auth_error = "auth" in error_msg.lower() or "api key" in error_msg.lower()
return {
"status": "error",
"message": f"API key validation failed: {error_msg}",
"is_auth_error": is_auth_error
}
except ImportError:
logger.exception("OpenAI package not installed")
return {
"status": "error",
"message": "OpenAI package not installed"
}
except Exception as e:
logger.exception("Unexpected error testing OpenAI connection")
return {
"status": "error",
"message": f"Unexpected error: {str(e)}"
}
app/config.py
+9
View File
@@ -165,8 +165,17 @@ class Settings(BaseSettings):
class Config:
env_file = ".env"
# Convert string representations of booleans to actual booleans
# and strip quotes from string values
@classmethod
def parse_env_var(cls, field_name: str, raw_val: str) -> Any:
# First, strip quotes from the value if it's a string
if isinstance(raw_val, str):
if (raw_val.startswith('"') and raw_val.endswith('"')) or \
(raw_val.startswith("'") and raw_val.endswith("'")):
raw_val = raw_val[1:-1]
raw_val = raw_val.strip()
# Convert string representations of booleans to actual booleans
if field_name.endswith('_enabled') or field_name == 'debug':
if raw_val.lower() in ('false', '0', 'no', 'n', 'f'):
return False
app/tasks/extract_metadata_with_gpt.py
+13 -5
View File
@@ -9,12 +9,20 @@ from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
# Import the shared Celery instance
from app.celery_app import celery
import openai
import logging
# Initialize OpenAI client dynamically
client = openai.OpenAI(
api_key=settings.openai_api_key,
base_url=settings.openai_base_url
)
logger = logging.getLogger(__name__)
# Initialize OpenAI client dynamically with better error handling
try:
client = openai.OpenAI(
api_key=settings.openai_api_key,
base_url=settings.openai_base_url
)
logger.info("OpenAI client initialized successfully")
except Exception as e:
logger.error(f"Failed to initialize OpenAI client: {e}")
client = None
def extract_json_from_text(text):
"""
app/tasks/process_with_azure_document_intelligence.py
+14 -5
View File
@@ -4,6 +4,7 @@ import PyPDF2
from azure.core.credentials import AzureKeyCredential
from azure.ai.documentintelligence import DocumentIntelligenceClient
from azure.ai.documentintelligence.models import AnalyzeOutputOption, AnalyzeResult
import azure.core.exceptions
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
@@ -12,11 +13,19 @@ from app.celery_app import celery
logger = logging.getLogger(__name__)
# Initialize Azure Document Intelligence client
document_intelligence_client = DocumentIntelligenceClient(
endpoint=settings.azure_endpoint,
credential=AzureKeyCredential(settings.azure_ai_key)
)
# Initialize Azure Document Intelligence client with error handling
try:
document_intelligence_client = DocumentIntelligenceClient(
endpoint=settings.azure_endpoint,
credential=AzureKeyCredential(settings.azure_ai_key)
)
logger.info("Azure Document Intelligence client initialized successfully")
except (ValueError, azure.core.exceptions.ClientAuthenticationError) as e:
logger.error(f"Failed to initialize Azure Document Intelligence client: {e}")
document_intelligence_client = None
except Exception as e:
logger.error(f"Unexpected error initializing Azure Document Intelligence client: {e}")
document_intelligence_client = None
# Azure Document Intelligence service limits for Standard S0 tier
AZURE_DOC_INTELLIGENCE_LIMITS = {
app/utils/config_validator.py
+36 -27
View File
@@ -138,40 +138,49 @@ def validate_storage_configs():
return issues
def mask_sensitive_value(value):
"""Helper function to mask sensitive values consistently"""
if not value:
return "Not set"
if isinstance(value, str):
if len(value) > 10:
visible_start = max(1, len(value) // 3)
visible_end = max(1, len(value) // 4)
return f"{value[:visible_start]}{'*' * (len(value) - visible_start - visible_end)}{value[-visible_end:]}"
else:
return f"{value[:2]}{'*' * (len(value) - 4)}{value[-2:]}" if len(value) > 4 else "****"
elif not isinstance(value, (bool, int, float)):
return "**Configured Value**"
return str(value)
"""
Masks sensitive values like API keys in logs and output
"""
# Return masked value for sensitive data
if value and isinstance(value, str) and len(value) > 8:
return value[:4] + "*" * (len(value) - 4)
return value
def get_provider_status():
"""
Returns status information for all configured providers
Font Awesome icons used:
- fa-brands fa-dropbox: Dropbox icon
- fa-solid fa-envelope: Email icon
- fa-solid fa-server: FTP Server icon
- fa-brands fa-google-drive: Google Drive icon
- fa-solid fa-cloud: NextCloud icon
- fa-brands fa-microsoft: Microsoft/OneDrive icon
- fa-solid fa-file-lines: Document/Paperless icon
- fa-brands fa-aws: AWS/S3 icon
- fa-solid fa-lock: SFTP icon (secure)
- fa-solid fa-heart-pulse: Uptime/health monitoring
- fa-solid fa-globe: WebDAV/web icon
"""
providers = {}
# Add AI services first
providers["OpenAI"] = {
"name": "OpenAI",
"icon": "fa-brands fa-openai",
"configured": bool(getattr(settings, 'openai_api_key', None) and
str(getattr(settings, 'openai_api_key', '')).startswith('sk-')),
"enabled": True,
"description": "AI-powered document analysis and metadata extraction",
"details": {
"api_key": mask_sensitive_value(getattr(settings, 'openai_api_key', None)),
"base_url": getattr(settings, 'openai_base_url', 'https://api.openai.com/v1'),
"model": getattr(settings, 'openai_model', 'gpt-4')
}
}
providers["Azure AI"] = {
"name": "Azure AI",
"icon": "fa-solid fa-robot",
"configured": bool(getattr(settings, 'azure_ai_key', None) and
getattr(settings, 'azure_endpoint', None)),
"enabled": True,
"description": "Microsoft Azure Document Intelligence",
"details": {
"api_key": mask_sensitive_value(getattr(settings, 'azure_ai_key', None)),
"endpoint": getattr(settings, 'azure_endpoint', 'Not set'),
"region": getattr(settings, 'azure_region', 'Not set')
}
}
# Add Dropbox configuration - alphabetically ordered providers
providers["Dropbox"] = {
"name": "Dropbox",
app/views/status.py
+3
View File
@@ -2,12 +2,15 @@
Status and configuration views for the application.
"""
from fastapi import Request
from fastapi.responses import JSONResponse
from datetime import datetime
import os
import subprocess
import logging
from app.views.base import APIRouter, templates, require_login, settings
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/status")
build-setup.sh
View File
docker-build.sh
View File
docker/build-scripts/container-init.sh
View File
frontend/templates/status_dashboard.html
+20
View File
@@ -136,6 +136,22 @@
Configure Now
</a>
{% endif %}
{% elif name == "OpenAI" %}
{% if provider.configured %}
<button
class="test-provider-btn inline-flex items-center px-2.5 py-1.5 border border-gray-300 text-xs font-medium rounded text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500"
data-provider="openai">
Test Connection
</button>
{% endif %}
{% elif name == "Azure AI" %}
{% if provider.configured %}
<button
class="test-provider-btn inline-flex items-center px-2.5 py-1.5 border border-gray-300 text-xs font-medium rounded text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500"
data-provider="azure">
Test Connection
</button>
{% endif %}
{% elif provider.configured and name == "Paperless-ngx" %}
<a href="{{ provider.details.host }}" target="_blank" class="inline-flex items-center px-2.5 py-1.5 border border-gray-300 text-xs font-medium rounded text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500">
<i class="fa-solid fa-external-link h-3 w-3 mr-1"></i>
@@ -380,6 +396,10 @@ document.addEventListener('DOMContentLoaded', function() {
endpoint = '/api/dropbox/test-token';
} else if (provider === 'onedrive') {
endpoint = '/api/onedrive/test-token';
} else if (provider === 'openai') {
endpoint = '/api/openai/test';
} else if (provider === 'azure') {
endpoint = '/api/azure/test';
}
fetch(endpoint)