From b258dd2f29012283475b39805cae385b8dba31b7 Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Wed, 9 Apr 2025 07:08:38 +0200 Subject: [PATCH] feat: Add OpenAI and Azure AI API endpoints with connection testing functionality --- app/api/__init__.py | 4 + app/api/azure.py | 124 ++++++++++++++++++ app/api/diagnostic.py | 3 + app/api/openai.py | 75 +++++++++++ app/config.py | 9 ++ app/tasks/extract_metadata_with_gpt.py | 18 ++- ...rocess_with_azure_document_intelligence.py | 19 ++- app/utils/config_validator.py | 77 ++++++----- app/views/status.py | 3 + build-setup.sh | 0 docker-build.sh | 0 docker/build-scripts/container-init.sh | 0 frontend/templates/status_dashboard.html | 20 +++ 13 files changed, 308 insertions(+), 44 deletions(-) create mode 100644 app/api/azure.py create mode 100644 app/api/openai.py create mode 100644 build-setup.sh create mode 100644 docker-build.sh create mode 100644 docker/build-scripts/container-init.sh diff --git a/app/api/__init__.py b/app/api/__init__.py index a2871ca9..468b09b7 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -11,6 +11,8 @@ from app.api.process import router as process_router from app.api.diagnostic import router as diagnostic_router from app.api.onedrive import router as onedrive_router from app.api.dropbox import router as dropbox_router +from app.api.openai import router as openai_router +from app.api.azure import router as azure_router # Set up logging logger = logging.getLogger(__name__) @@ -25,3 +27,5 @@ router.include_router(process_router) router.include_router(diagnostic_router) router.include_router(onedrive_router) router.include_router(dropbox_router) +router.include_router(openai_router) +router.include_router(azure_router) diff --git a/app/api/azure.py b/app/api/azure.py new file mode 100644 index 00000000..022d8ddf --- /dev/null +++ b/app/api/azure.py @@ -0,0 +1,124 @@ +""" +Azure AI API endpoints +""" +from fastapi import APIRouter, Request, HTTPException, status +import logging +import os + +from app.auth import require_login +from app.config import settings + +# Import the Azure modules including the administration client +from azure.core.credentials import AzureKeyCredential +from azure.ai.documentintelligence import DocumentIntelligenceClient +from azure.ai.documentintelligence import DocumentIntelligenceAdministrationClient +import azure.core.exceptions + +# Set up logging +logger = logging.getLogger(__name__) + +router = APIRouter() + +@router.get("/azure/test") +@require_login +async def test_azure_connection(request: Request): + """ + Test if the configured Azure Document Intelligence connection is valid. + Uses the DocumentIntelligenceAdministrationClient for testing the connection. + """ + try: + logger.info("Testing Azure Document Intelligence connection") + + # Check if Azure configuration is present + if not settings.azure_endpoint or not settings.azure_ai_key: + logger.warning("Azure Document Intelligence configuration is incomplete") + missing = [] + if not settings.azure_endpoint: + missing.append("endpoint") + if not settings.azure_ai_key: + missing.append("API key") + + return { + "status": "error", + "message": f"Azure Document Intelligence configuration is incomplete. Missing: {', '.join(missing)}" + } + + # Try to initialize the admin client and make a request to list operations + try: + # Initialize the admin client with credentials + admin_client = DocumentIntelligenceAdministrationClient( + endpoint=settings.azure_endpoint, + credential=AzureKeyCredential(settings.azure_ai_key) + ) + + # Test the connection by listing operations - this is a documented method in the admin client + operations = list(admin_client.list_operations()) + + # Successfully initialized client and made a request + logger.info("Azure Document Intelligence Admin connection successfully tested") + + # Return success with available operations info + operations_info = [] + try: + for op in operations: + if hasattr(op, 'operation_id') and op.operation_id: + op_info = { + "id": op.operation_id, + "status": op.status if hasattr(op, 'status') else "Unknown", + "created": str(op.created_on) if hasattr(op, 'created_on') else "Unknown", + "kind": op.kind if hasattr(op, 'kind') else "Unknown" + } + operations_info.append(op_info) + + operation_count = len(operations_info) + return { + "status": "success", + "message": f"Azure Document Intelligence connection is valid. Found {operation_count} operations.", + "endpoint": settings.azure_endpoint, + "operations_count": operation_count, + "recent_operations": operations_info[:3] if operations_info else [] + } + except Exception as e: + # If error occurs while processing operations info, still return success + logger.warning(f"Connected to Azure but couldn't parse operations: {e}") + return { + "status": "success", + "message": "Azure Document Intelligence connection is valid, but couldn't retrieve operations details.", + "endpoint": settings.azure_endpoint + } + + except azure.core.exceptions.ClientAuthenticationError as e: + logger.error(f"Azure authentication error: {e}") + return { + "status": "error", + "message": f"Authentication error: Invalid API key or credentials", + "detail": str(e) + } + except azure.core.exceptions.ServiceRequestError as e: + logger.error(f"Azure service request error: {e}") + return { + "status": "error", + "message": f"Service request error: Could not reach the Azure endpoint", + "detail": str(e) + } + except ValueError as e: + logger.error(f"Azure configuration value error: {e}") + return { + "status": "error", + "message": f"Configuration error: {str(e)}", + "detail": str(e) + } + except Exception as e: + logger.error(f"Azure connection test failed with unexpected error: {e}") + return { + "status": "error", + "message": f"Connection test failed with unexpected error", + "detail": str(e) + } + + except Exception as e: + logger.exception("Unexpected error testing Azure Document Intelligence connection") + return { + "status": "error", + "message": f"Unexpected error: {str(e)}" + } diff --git a/app/api/diagnostic.py b/app/api/diagnostic.py index 9691057e..1d3977ed 100644 --- a/app/api/diagnostic.py +++ b/app/api/diagnostic.py @@ -31,12 +31,15 @@ async def diagnostic_settings(request: Request, current_user: dict = Depends(get "email": bool(getattr(settings, 'email_host', None)), "s3": bool(getattr(settings, 's3_bucket_name', None)), "dropbox": bool(getattr(settings, 'dropbox_refresh_token', None)), + "onedrive": bool(getattr(settings, 'onedrive_refresh_token', None)), "nextcloud": bool(getattr(settings, 'nextcloud_upload_url', None)), "sftp": bool(getattr(settings, 'sftp_host', None)), "paperless": bool(getattr(settings, 'paperless_host', None)), "google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)), "uptime_kuma": bool(getattr(settings, 'uptime_kuma_url', None)), "auth": bool(getattr(settings, 'authentik_config_url', None)), + "openai": bool(getattr(settings, 'openai_api_key', None)), + "azure": bool(getattr(settings, 'azure_api_key', None) and getattr(settings, 'azure_endpoint', None)), }, "imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)), } diff --git a/app/api/openai.py b/app/api/openai.py new file mode 100644 index 00000000..155c956d --- /dev/null +++ b/app/api/openai.py @@ -0,0 +1,75 @@ +""" +OpenAI API endpoints +""" +from fastapi import APIRouter, Request, HTTPException, status +import logging +import os +import requests + +from app.auth import require_login +from app.config import settings + +# Set up logging +logger = logging.getLogger(__name__) + +router = APIRouter() + +@router.get("/openai/test") +@require_login +async def test_openai_connection(request: Request): + """ + Test if the configured OpenAI API key is valid. + """ + try: + import openai + + logger.info("Testing OpenAI API key validity") + + # Check if API key is configured + if not settings.openai_api_key: + logger.warning("No OpenAI API key configured") + return { + "status": "error", + "message": "No OpenAI API key is configured" + } + + # Configure the client + client = openai.OpenAI(api_key=settings.openai_api_key) + + # Try to make a simple request to validate the key + try: + # Use a models list endpoint as a simple validation + models = client.models.list() + + # If we got here, the key is valid + logger.info("OpenAI API key is valid") + return { + "status": "success", + "message": "OpenAI API key is valid", + "models_available": len(models.data) if hasattr(models, "data") else "Unknown" + } + except Exception as e: + error_msg = str(e) + logger.error(f"OpenAI API key test failed: {error_msg}") + + # Determine if this is an authentication error + is_auth_error = "auth" in error_msg.lower() or "api key" in error_msg.lower() + + return { + "status": "error", + "message": f"API key validation failed: {error_msg}", + "is_auth_error": is_auth_error + } + + except ImportError: + logger.exception("OpenAI package not installed") + return { + "status": "error", + "message": "OpenAI package not installed" + } + except Exception as e: + logger.exception("Unexpected error testing OpenAI connection") + return { + "status": "error", + "message": f"Unexpected error: {str(e)}" + } diff --git a/app/config.py b/app/config.py index 869606c0..559d2aec 100644 --- a/app/config.py +++ b/app/config.py @@ -165,8 +165,17 @@ class Settings(BaseSettings): class Config: env_file = ".env" # Convert string representations of booleans to actual booleans + # and strip quotes from string values @classmethod def parse_env_var(cls, field_name: str, raw_val: str) -> Any: + # First, strip quotes from the value if it's a string + if isinstance(raw_val, str): + if (raw_val.startswith('"') and raw_val.endswith('"')) or \ + (raw_val.startswith("'") and raw_val.endswith("'")): + raw_val = raw_val[1:-1] + raw_val = raw_val.strip() + + # Convert string representations of booleans to actual booleans if field_name.endswith('_enabled') or field_name == 'debug': if raw_val.lower() in ('false', '0', 'no', 'n', 'f'): return False diff --git a/app/tasks/extract_metadata_with_gpt.py b/app/tasks/extract_metadata_with_gpt.py index ea20440d..2aab5283 100644 --- a/app/tasks/extract_metadata_with_gpt.py +++ b/app/tasks/extract_metadata_with_gpt.py @@ -9,12 +9,20 @@ from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf # Import the shared Celery instance from app.celery_app import celery import openai +import logging -# Initialize OpenAI client dynamically -client = openai.OpenAI( - api_key=settings.openai_api_key, - base_url=settings.openai_base_url -) +logger = logging.getLogger(__name__) + +# Initialize OpenAI client dynamically with better error handling +try: + client = openai.OpenAI( + api_key=settings.openai_api_key, + base_url=settings.openai_base_url + ) + logger.info("OpenAI client initialized successfully") +except Exception as e: + logger.error(f"Failed to initialize OpenAI client: {e}") + client = None def extract_json_from_text(text): """ diff --git a/app/tasks/process_with_azure_document_intelligence.py b/app/tasks/process_with_azure_document_intelligence.py index ecf9ee2f..1dbed541 100644 --- a/app/tasks/process_with_azure_document_intelligence.py +++ b/app/tasks/process_with_azure_document_intelligence.py @@ -4,6 +4,7 @@ import PyPDF2 from azure.core.credentials import AzureKeyCredential from azure.ai.documentintelligence import DocumentIntelligenceClient from azure.ai.documentintelligence.models import AnalyzeOutputOption, AnalyzeResult +import azure.core.exceptions from app.config import settings from app.tasks.retry_config import BaseTaskWithRetry @@ -12,11 +13,19 @@ from app.celery_app import celery logger = logging.getLogger(__name__) -# Initialize Azure Document Intelligence client -document_intelligence_client = DocumentIntelligenceClient( - endpoint=settings.azure_endpoint, - credential=AzureKeyCredential(settings.azure_ai_key) -) +# Initialize Azure Document Intelligence client with error handling +try: + document_intelligence_client = DocumentIntelligenceClient( + endpoint=settings.azure_endpoint, + credential=AzureKeyCredential(settings.azure_ai_key) + ) + logger.info("Azure Document Intelligence client initialized successfully") +except (ValueError, azure.core.exceptions.ClientAuthenticationError) as e: + logger.error(f"Failed to initialize Azure Document Intelligence client: {e}") + document_intelligence_client = None +except Exception as e: + logger.error(f"Unexpected error initializing Azure Document Intelligence client: {e}") + document_intelligence_client = None # Azure Document Intelligence service limits for Standard S0 tier AZURE_DOC_INTELLIGENCE_LIMITS = { diff --git a/app/utils/config_validator.py b/app/utils/config_validator.py index 28e58421..0c2a7f13 100644 --- a/app/utils/config_validator.py +++ b/app/utils/config_validator.py @@ -138,40 +138,49 @@ def validate_storage_configs(): return issues def mask_sensitive_value(value): - """Helper function to mask sensitive values consistently""" - if not value: - return "Not set" - - if isinstance(value, str): - if len(value) > 10: - visible_start = max(1, len(value) // 3) - visible_end = max(1, len(value) // 4) - return f"{value[:visible_start]}{'*' * (len(value) - visible_start - visible_end)}{value[-visible_end:]}" - else: - return f"{value[:2]}{'*' * (len(value) - 4)}{value[-2:]}" if len(value) > 4 else "****" - elif not isinstance(value, (bool, int, float)): - return "**Configured Value**" - return str(value) + """ + Masks sensitive values like API keys in logs and output + """ + # Return masked value for sensitive data + if value and isinstance(value, str) and len(value) > 8: + return value[:4] + "*" * (len(value) - 4) + return value def get_provider_status(): """ Returns status information for all configured providers - - Font Awesome icons used: - - fa-brands fa-dropbox: Dropbox icon - - fa-solid fa-envelope: Email icon - - fa-solid fa-server: FTP Server icon - - fa-brands fa-google-drive: Google Drive icon - - fa-solid fa-cloud: NextCloud icon - - fa-brands fa-microsoft: Microsoft/OneDrive icon - - fa-solid fa-file-lines: Document/Paperless icon - - fa-brands fa-aws: AWS/S3 icon - - fa-solid fa-lock: SFTP icon (secure) - - fa-solid fa-heart-pulse: Uptime/health monitoring - - fa-solid fa-globe: WebDAV/web icon """ providers = {} + # Add AI services first + providers["OpenAI"] = { + "name": "OpenAI", + "icon": "fa-brands fa-openai", + "configured": bool(getattr(settings, 'openai_api_key', None) and + str(getattr(settings, 'openai_api_key', '')).startswith('sk-')), + "enabled": True, + "description": "AI-powered document analysis and metadata extraction", + "details": { + "api_key": mask_sensitive_value(getattr(settings, 'openai_api_key', None)), + "base_url": getattr(settings, 'openai_base_url', 'https://api.openai.com/v1'), + "model": getattr(settings, 'openai_model', 'gpt-4') + } + } + + providers["Azure AI"] = { + "name": "Azure AI", + "icon": "fa-solid fa-robot", + "configured": bool(getattr(settings, 'azure_ai_key', None) and + getattr(settings, 'azure_endpoint', None)), + "enabled": True, + "description": "Microsoft Azure Document Intelligence", + "details": { + "api_key": mask_sensitive_value(getattr(settings, 'azure_ai_key', None)), + "endpoint": getattr(settings, 'azure_endpoint', 'Not set'), + "region": getattr(settings, 'azure_region', 'Not set') + } + } + # Add Dropbox configuration - alphabetically ordered providers providers["Dropbox"] = { "name": "Dropbox", @@ -210,7 +219,7 @@ def get_provider_status(): # Add FTP configuration to providers providers["FTP Storage"] = { - "name": "FTP Storage", + "name": "FTP Storage", "icon": "fa-solid fa-server", "configured": bool(getattr(settings, 'ftp_host', None) and getattr(settings, 'ftp_username', None) and @@ -301,7 +310,7 @@ def get_provider_status(): # Check S3 configuration providers["S3 Storage"] = { - "name": "S3 Storage", + "name": "S3 Storage", "icon": "fa-brands fa-aws", "configured": bool(getattr(settings, 's3_bucket_name', None) and getattr(settings, 'aws_access_key_id', None) and @@ -339,10 +348,10 @@ def get_provider_status(): "folder": getattr(settings, 'sftp_folder', 'Not set') } } - + # Add Uptime Kuma configuration providers["Uptime Kuma"] = { - "name": "Uptime Kuma", + "name": "Uptime Kuma", "icon": "fa-solid fa-heart-pulse", "configured": bool(getattr(settings, 'uptime_kuma_url', None)), "enabled": True, @@ -352,7 +361,7 @@ def get_provider_status(): "ping_interval": getattr(settings, 'uptime_kuma_ping_interval', 'Not set') } } - + # Check WebDAV configuration providers["WebDAV"] = { "name": "WebDAV", @@ -394,7 +403,7 @@ def dump_all_settings(): def get_settings_for_display(show_values=False): """ Group settings into logical categories and check if they are configured. - Returns a dictionary with categories as keys and lists of setting items as values. + Returns a dictionary with categories as keys and lists of setting items as values. Each setting item is a dict with name, value, and is_configured. If show_values is False, sensitive values are masked. @@ -403,7 +412,7 @@ def get_settings_for_display(show_values=False): result = { "System Info": [ { - "name": "App Version", + "name": "App Version", "value": settings.version, "is_configured": True } diff --git a/app/views/status.py b/app/views/status.py index a7297d34..92d69818 100644 --- a/app/views/status.py +++ b/app/views/status.py @@ -2,12 +2,15 @@ Status and configuration views for the application. """ from fastapi import Request +from fastapi.responses import JSONResponse from datetime import datetime import os import subprocess +import logging from app.views.base import APIRouter, templates, require_login, settings +logger = logging.getLogger(__name__) router = APIRouter() @router.get("/status") diff --git a/build-setup.sh b/build-setup.sh new file mode 100644 index 00000000..e69de29b diff --git a/docker-build.sh b/docker-build.sh new file mode 100644 index 00000000..e69de29b diff --git a/docker/build-scripts/container-init.sh b/docker/build-scripts/container-init.sh new file mode 100644 index 00000000..e69de29b diff --git a/frontend/templates/status_dashboard.html b/frontend/templates/status_dashboard.html index 66baea96..b9460f23 100644 --- a/frontend/templates/status_dashboard.html +++ b/frontend/templates/status_dashboard.html @@ -136,6 +136,22 @@ Configure Now {% endif %} + {% elif name == "OpenAI" %} + {% if provider.configured %} + + {% endif %} + {% elif name == "Azure AI" %} + {% if provider.configured %} + + {% endif %} {% elif provider.configured and name == "Paperless-ngx" %} @@ -380,6 +396,10 @@ document.addEventListener('DOMContentLoaded', function() { endpoint = '/api/dropbox/test-token'; } else if (provider === 'onedrive') { endpoint = '/api/onedrive/test-token'; + } else if (provider === 'openai') { + endpoint = '/api/openai/test'; + } else if (provider === 'azure') { + endpoint = '/api/azure/test'; } fetch(endpoint)