From d66fef7ad49fe5185ec98862d1f37155f529edf8 Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Mon, 7 Apr 2025 00:43:14 +0200 Subject: [PATCH 1/3] feat: Enhance Docker setup and application status reporting with build date and container info --- Dockerfile | 31 +++++------ app/views/general.py | 20 +++++-- app/views/status.py | 68 +++++++++++++++++++++++- docker-compose.yaml | 5 +- docker/build-scripts/save-build-date.sh | 11 ---- frontend/templates/status_dashboard.html | 10 ++++ 6 files changed, 109 insertions(+), 36 deletions(-) delete mode 100644 docker/build-scripts/save-build-date.sh diff --git a/Dockerfile b/Dockerfile index 9f58d8f6..d16c09a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,41 +1,38 @@ -# Stage 1: Build dependencies +# Use multi-stage build for a smaller final image FROM python:3.13 AS builder WORKDIR /app +# Copy requirements first for better layer caching COPY requirements.txt /app/ RUN pip install --no-cache-dir -r requirements.txt -# Stage 2: Final image +# Second stage for the actual runtime FROM python:3.13.2-slim WORKDIR /app -# Copy installed dependencies +# Copy installed packages from builder stage COPY --from=builder /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages COPY --from=builder /usr/local/bin /usr/local/bin -# Copy application files correctly +# Copy application code COPY ./app /app/app -COPY ./frontend /app/frontend COPY ./VERSION /app/VERSION -COPY ./LICENSE /app/LICENSE +COPY ./frontend /app/frontend -# Copy build script and generate build date -COPY ./docker/build-scripts/save-build-date.sh /tmp/ -RUN mkdir -p /app/docker/build-scripts/ && \ - cp /tmp/save-build-date.sh /app/docker/build-scripts/ && \ - chmod +x /tmp/save-build-date.sh && \ - /tmp/save-build-date.sh +# Create runtime_info directory +RUN mkdir -p /app/runtime_info -# Set build date as environment variable -#ENV BUILD_DATE=$(cat /app/BUILD_DATE) +# Create necessary directories +RUN mkdir -p /workdir -# Set Python path explicitly +# Set environment variables ENV PYTHONPATH=/app +ENV PYTHONUNBUFFERED=1 -# Expose API port +# Expose the port the app runs on EXPOSE 8000 -WORKDIR /app +# Default command CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/app/views/general.py b/app/views/general.py index ed51af7d..5b010a3f 100644 --- a/app/views/general.py +++ b/app/views/general.py @@ -1,18 +1,19 @@ """ General routes for the application homepage and basic pages. """ -from fastapi import Request, HTTPException +from fastapi import Request, HTTPException, Depends from fastapi.responses import FileResponse from pathlib import Path from datetime import date +from sqlalchemy.orm import Session -from app.views.base import APIRouter, templates, require_login +from app.views.base import APIRouter, templates, require_login, get_db from app.utils.config_validator import get_provider_status, validate_storage_configs router = APIRouter() @router.get("/", include_in_schema=False) -async def serve_index(request: Request): +async def serve_index(request: Request, db: Session = Depends(get_db)): """Serve the index/home page.""" # Get provider information from config validator providers = get_provider_status() @@ -27,9 +28,20 @@ async def serve_index(request: Request): 's3', 'ftp', 'webdav', 'google_drive', 'onedrive']) + # Query the actual file count from the database + processed_files = 0 + try: + # Import the model here to avoid circular imports + from app.models import FileRecord + processed_files = db.query(FileRecord).count() + except Exception as e: + # Log error but continue (don't break the page if DB query fails) + from app.views.base import logger + logger.error(f"Error counting files: {str(e)}") + # Create stats object to pass to the template stats = { - "processed_files": 0, # Placeholder - would need actual DB query + "processed_files": processed_files, "active_integrations": configured_providers, "storage_targets": configured_storage_targets } diff --git a/app/views/status.py b/app/views/status.py index a1a93cc9..a7297d34 100644 --- a/app/views/status.py +++ b/app/views/status.py @@ -3,6 +3,8 @@ Status and configuration views for the application. """ from fastapi import Request from datetime import datetime +import os +import subprocess from app.views.base import APIRouter, templates, require_login, settings @@ -19,14 +21,78 @@ async def status_dashboard(request: Request): # Get provider status providers = get_provider_status() + # Get build date from settings + build_date = getattr(settings, 'build_date', 'Unknown') + + # Try to get container information + container_info = {} + try: + # Check for Docker environment + if os.path.exists('/.dockerenv'): + # We're inside a Docker container + container_info['is_docker'] = True + + # Try to get container ID + try: + with open('/proc/self/cgroup', 'r') as f: + for line in f: + if 'docker' in line: + container_id = line.split('/')[-1].strip() + container_info['id'] = container_id[:12] # Short ID format + break + except Exception: + container_info['id'] = 'Unknown' + + # Try to get Git commit SHA from runtime info + try: + # First check runtime info directory + if os.path.exists('/app/runtime_info/GIT_SHA'): + with open('/app/runtime_info/GIT_SHA', 'r') as f: + git_sha = f.read().strip() + # Then try environment variable + else: + git_sha = os.environ.get('GIT_COMMIT_SHA', '') + + # If still not found, try the original file location + if not git_sha and os.path.exists('/.git-commit-sha'): + with open('/.git-commit-sha', 'r') as f: + git_sha = f.read().strip() + + container_info['git_sha'] = git_sha[:7] if git_sha and git_sha != 'unknown' else 'Unknown' + except Exception: + container_info['git_sha'] = 'Unknown' + + # Try to get runtime information + try: + if os.path.exists('/app/runtime_info/RUNTIME_INFO'): + with open('/app/runtime_info/RUNTIME_INFO', 'r') as f: + container_info['runtime_info'] = f.read().strip() + except Exception: + pass + else: + container_info['is_docker'] = False + + # If not in Docker, try to get Git info directly + try: + git_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD'], + stderr=subprocess.DEVNULL, + text=True).strip()[:7] + container_info['git_sha'] = git_sha + except (subprocess.SubprocessError, FileNotFoundError): + container_info['git_sha'] = 'Unknown' + except Exception: + container_info = {'is_docker': False, 'id': 'Unknown', 'git_sha': 'Unknown'} + return templates.TemplateResponse( "status_dashboard.html", { "request": request, "providers": providers, "app_version": settings.version, + "build_date": build_date, "debug_enabled": getattr(settings, 'debug', False), - "last_check": datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "last_check": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "container_info": container_info } ) diff --git a/docker-compose.yaml b/docker-compose.yaml index d7faf546..e09c5b23 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,7 +10,7 @@ services: working_dir: /workdir # We'll run uvicorn from the container's /app code - command: ["sh", "-c", "cd /app && uvicorn app.main:app --host 0.0.0.0 --port 8000 --proxy-headers"] + command: ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"] # Environment variables environment: @@ -50,7 +50,7 @@ services: - redis - gotenberg - # Mount the shared directory (and optionally your code if you want dev mode) + # Mount the shared directory volumes: - /var/docparse/workdir:/workdir @@ -59,7 +59,6 @@ services: container_name: gotenberg restart: always - redis: image: redis:alpine container_name: document_redis diff --git a/docker/build-scripts/save-build-date.sh b/docker/build-scripts/save-build-date.sh deleted file mode 100644 index 3a459f50..00000000 --- a/docker/build-scripts/save-build-date.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Get current date in Month DD, YYYY format (e.g., May 15, 2024) -BUILD_DATE=$(date +"%B %d, %Y") - -# Save it to the BUILD_DATE file -echo $BUILD_DATE > /app/BUILD_DATE - -# Also set it as an environment variable -echo "Setting BUILD_DATE=$BUILD_DATE" -export BUILD_DATE diff --git a/frontend/templates/status_dashboard.html b/frontend/templates/status_dashboard.html index 7ebdc987..66baea96 100644 --- a/frontend/templates/status_dashboard.html +++ b/frontend/templates/status_dashboard.html @@ -14,10 +14,20 @@

From 2f822adeb7f539a709c2502daa60527425875d32 Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Wed, 9 Apr 2025 07:08:38 +0200 Subject: [PATCH 2/3] feat: Add OpenAI and Azure AI API endpoints with connection testing functionality --- app/api/__init__.py | 4 + app/api/azure.py | 124 ++++++++++++++++++ app/api/diagnostic.py | 3 + app/api/openai.py | 75 +++++++++++ app/config.py | 9 ++ app/tasks/extract_metadata_with_gpt.py | 18 ++- ...rocess_with_azure_document_intelligence.py | 19 ++- app/utils/config_validator.py | 77 ++++++----- app/views/status.py | 3 + build-setup.sh | 0 docker-build.sh | 0 docker/build-scripts/container-init.sh | 0 frontend/templates/status_dashboard.html | 20 +++ 13 files changed, 308 insertions(+), 44 deletions(-) create mode 100644 app/api/azure.py create mode 100644 app/api/openai.py create mode 100644 build-setup.sh create mode 100644 docker-build.sh create mode 100644 docker/build-scripts/container-init.sh diff --git a/app/api/__init__.py b/app/api/__init__.py index a2871ca9..468b09b7 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -11,6 +11,8 @@ from app.api.process import router as process_router from app.api.diagnostic import router as diagnostic_router from app.api.onedrive import router as onedrive_router from app.api.dropbox import router as dropbox_router +from app.api.openai import router as openai_router +from app.api.azure import router as azure_router # Set up logging logger = logging.getLogger(__name__) @@ -25,3 +27,5 @@ router.include_router(process_router) router.include_router(diagnostic_router) router.include_router(onedrive_router) router.include_router(dropbox_router) +router.include_router(openai_router) +router.include_router(azure_router) diff --git a/app/api/azure.py b/app/api/azure.py new file mode 100644 index 00000000..022d8ddf --- /dev/null +++ b/app/api/azure.py @@ -0,0 +1,124 @@ +""" +Azure AI API endpoints +""" +from fastapi import APIRouter, Request, HTTPException, status +import logging +import os + +from app.auth import require_login +from app.config import settings + +# Import the Azure modules including the administration client +from azure.core.credentials import AzureKeyCredential +from azure.ai.documentintelligence import DocumentIntelligenceClient +from azure.ai.documentintelligence import DocumentIntelligenceAdministrationClient +import azure.core.exceptions + +# Set up logging +logger = logging.getLogger(__name__) + +router = APIRouter() + +@router.get("/azure/test") +@require_login +async def test_azure_connection(request: Request): + """ + Test if the configured Azure Document Intelligence connection is valid. + Uses the DocumentIntelligenceAdministrationClient for testing the connection. + """ + try: + logger.info("Testing Azure Document Intelligence connection") + + # Check if Azure configuration is present + if not settings.azure_endpoint or not settings.azure_ai_key: + logger.warning("Azure Document Intelligence configuration is incomplete") + missing = [] + if not settings.azure_endpoint: + missing.append("endpoint") + if not settings.azure_ai_key: + missing.append("API key") + + return { + "status": "error", + "message": f"Azure Document Intelligence configuration is incomplete. Missing: {', '.join(missing)}" + } + + # Try to initialize the admin client and make a request to list operations + try: + # Initialize the admin client with credentials + admin_client = DocumentIntelligenceAdministrationClient( + endpoint=settings.azure_endpoint, + credential=AzureKeyCredential(settings.azure_ai_key) + ) + + # Test the connection by listing operations - this is a documented method in the admin client + operations = list(admin_client.list_operations()) + + # Successfully initialized client and made a request + logger.info("Azure Document Intelligence Admin connection successfully tested") + + # Return success with available operations info + operations_info = [] + try: + for op in operations: + if hasattr(op, 'operation_id') and op.operation_id: + op_info = { + "id": op.operation_id, + "status": op.status if hasattr(op, 'status') else "Unknown", + "created": str(op.created_on) if hasattr(op, 'created_on') else "Unknown", + "kind": op.kind if hasattr(op, 'kind') else "Unknown" + } + operations_info.append(op_info) + + operation_count = len(operations_info) + return { + "status": "success", + "message": f"Azure Document Intelligence connection is valid. Found {operation_count} operations.", + "endpoint": settings.azure_endpoint, + "operations_count": operation_count, + "recent_operations": operations_info[:3] if operations_info else [] + } + except Exception as e: + # If error occurs while processing operations info, still return success + logger.warning(f"Connected to Azure but couldn't parse operations: {e}") + return { + "status": "success", + "message": "Azure Document Intelligence connection is valid, but couldn't retrieve operations details.", + "endpoint": settings.azure_endpoint + } + + except azure.core.exceptions.ClientAuthenticationError as e: + logger.error(f"Azure authentication error: {e}") + return { + "status": "error", + "message": f"Authentication error: Invalid API key or credentials", + "detail": str(e) + } + except azure.core.exceptions.ServiceRequestError as e: + logger.error(f"Azure service request error: {e}") + return { + "status": "error", + "message": f"Service request error: Could not reach the Azure endpoint", + "detail": str(e) + } + except ValueError as e: + logger.error(f"Azure configuration value error: {e}") + return { + "status": "error", + "message": f"Configuration error: {str(e)}", + "detail": str(e) + } + except Exception as e: + logger.error(f"Azure connection test failed with unexpected error: {e}") + return { + "status": "error", + "message": f"Connection test failed with unexpected error", + "detail": str(e) + } + + except Exception as e: + logger.exception("Unexpected error testing Azure Document Intelligence connection") + return { + "status": "error", + "message": f"Unexpected error: {str(e)}" + } diff --git a/app/api/diagnostic.py b/app/api/diagnostic.py index 9691057e..1d3977ed 100644 --- a/app/api/diagnostic.py +++ b/app/api/diagnostic.py @@ -31,12 +31,15 @@ async def diagnostic_settings(request: Request, current_user: dict = Depends(get "email": bool(getattr(settings, 'email_host', None)), "s3": bool(getattr(settings, 's3_bucket_name', None)), "dropbox": bool(getattr(settings, 'dropbox_refresh_token', None)), + "onedrive": bool(getattr(settings, 'onedrive_refresh_token', None)), "nextcloud": bool(getattr(settings, 'nextcloud_upload_url', None)), "sftp": bool(getattr(settings, 'sftp_host', None)), "paperless": bool(getattr(settings, 'paperless_host', None)), "google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)), "uptime_kuma": bool(getattr(settings, 'uptime_kuma_url', None)), "auth": bool(getattr(settings, 'authentik_config_url', None)), + "openai": bool(getattr(settings, 'openai_api_key', None)), + "azure": bool(getattr(settings, 'azure_api_key', None) and getattr(settings, 'azure_endpoint', None)), }, "imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)), } diff --git a/app/api/openai.py b/app/api/openai.py new file mode 100644 index 00000000..155c956d --- /dev/null +++ b/app/api/openai.py @@ -0,0 +1,75 @@ +""" +OpenAI API endpoints +""" +from fastapi import APIRouter, Request, HTTPException, status +import logging +import os +import requests + +from app.auth import require_login +from app.config import settings + +# Set up logging +logger = logging.getLogger(__name__) + +router = APIRouter() + +@router.get("/openai/test") +@require_login +async def test_openai_connection(request: Request): + """ + Test if the configured OpenAI API key is valid. + """ + try: + import openai + + logger.info("Testing OpenAI API key validity") + + # Check if API key is configured + if not settings.openai_api_key: + logger.warning("No OpenAI API key configured") + return { + "status": "error", + "message": "No OpenAI API key is configured" + } + + # Configure the client + client = openai.OpenAI(api_key=settings.openai_api_key) + + # Try to make a simple request to validate the key + try: + # Use a models list endpoint as a simple validation + models = client.models.list() + + # If we got here, the key is valid + logger.info("OpenAI API key is valid") + return { + "status": "success", + "message": "OpenAI API key is valid", + "models_available": len(models.data) if hasattr(models, "data") else "Unknown" + } + except Exception as e: + error_msg = str(e) + logger.error(f"OpenAI API key test failed: {error_msg}") + + # Determine if this is an authentication error + is_auth_error = "auth" in error_msg.lower() or "api key" in error_msg.lower() + + return { + "status": "error", + "message": f"API key validation failed: {error_msg}", + "is_auth_error": is_auth_error + } + + except ImportError: + logger.exception("OpenAI package not installed") + return { + "status": "error", + "message": "OpenAI package not installed" + } + except Exception as e: + logger.exception("Unexpected error testing OpenAI connection") + return { + "status": "error", + "message": f"Unexpected error: {str(e)}" + } diff --git a/app/config.py b/app/config.py index 869606c0..559d2aec 100644 --- a/app/config.py +++ b/app/config.py @@ -165,8 +165,17 @@ class Settings(BaseSettings): class Config: env_file = ".env" # Convert string representations of booleans to actual booleans + # and strip quotes from string values @classmethod def parse_env_var(cls, field_name: str, raw_val: str) -> Any: + # First, strip quotes from the value if it's a string + if isinstance(raw_val, str): + if (raw_val.startswith('"') and raw_val.endswith('"')) or \ + (raw_val.startswith("'") and raw_val.endswith("'")): + raw_val = raw_val[1:-1] + raw_val = raw_val.strip() + + # Convert string representations of booleans to actual booleans if field_name.endswith('_enabled') or field_name == 'debug': if raw_val.lower() in ('false', '0', 'no', 'n', 'f'): return False diff --git a/app/tasks/extract_metadata_with_gpt.py b/app/tasks/extract_metadata_with_gpt.py index ea20440d..2aab5283 100644 --- a/app/tasks/extract_metadata_with_gpt.py +++ b/app/tasks/extract_metadata_with_gpt.py @@ -9,12 +9,20 @@ from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf # Import the shared Celery instance from app.celery_app import celery import openai +import logging -# Initialize OpenAI client dynamically -client = openai.OpenAI( - api_key=settings.openai_api_key, - base_url=settings.openai_base_url -) +logger = logging.getLogger(__name__) + +# Initialize OpenAI client dynamically with better error handling +try: + client = openai.OpenAI( + api_key=settings.openai_api_key, + base_url=settings.openai_base_url + ) + logger.info("OpenAI client initialized successfully") +except Exception as e: + logger.error(f"Failed to initialize OpenAI client: {e}") + client = None def extract_json_from_text(text): """ diff --git a/app/tasks/process_with_azure_document_intelligence.py b/app/tasks/process_with_azure_document_intelligence.py index ecf9ee2f..1dbed541 100644 --- a/app/tasks/process_with_azure_document_intelligence.py +++ b/app/tasks/process_with_azure_document_intelligence.py @@ -4,6 +4,7 @@ import PyPDF2 from azure.core.credentials import AzureKeyCredential from azure.ai.documentintelligence import DocumentIntelligenceClient from azure.ai.documentintelligence.models import AnalyzeOutputOption, AnalyzeResult +import azure.core.exceptions from app.config import settings from app.tasks.retry_config import BaseTaskWithRetry @@ -12,11 +13,19 @@ from app.celery_app import celery logger = logging.getLogger(__name__) -# Initialize Azure Document Intelligence client -document_intelligence_client = DocumentIntelligenceClient( - endpoint=settings.azure_endpoint, - credential=AzureKeyCredential(settings.azure_ai_key) -) +# Initialize Azure Document Intelligence client with error handling +try: + document_intelligence_client = DocumentIntelligenceClient( + endpoint=settings.azure_endpoint, + credential=AzureKeyCredential(settings.azure_ai_key) + ) + logger.info("Azure Document Intelligence client initialized successfully") +except (ValueError, azure.core.exceptions.ClientAuthenticationError) as e: + logger.error(f"Failed to initialize Azure Document Intelligence client: {e}") + document_intelligence_client = None +except Exception as e: + logger.error(f"Unexpected error initializing Azure Document Intelligence client: {e}") + document_intelligence_client = None # Azure Document Intelligence service limits for Standard S0 tier AZURE_DOC_INTELLIGENCE_LIMITS = { diff --git a/app/utils/config_validator.py b/app/utils/config_validator.py index 28e58421..0c2a7f13 100644 --- a/app/utils/config_validator.py +++ b/app/utils/config_validator.py @@ -138,40 +138,49 @@ def validate_storage_configs(): return issues def mask_sensitive_value(value): - """Helper function to mask sensitive values consistently""" - if not value: - return "Not set" - - if isinstance(value, str): - if len(value) > 10: - visible_start = max(1, len(value) // 3) - visible_end = max(1, len(value) // 4) - return f"{value[:visible_start]}{'*' * (len(value) - visible_start - visible_end)}{value[-visible_end:]}" - else: - return f"{value[:2]}{'*' * (len(value) - 4)}{value[-2:]}" if len(value) > 4 else "****" - elif not isinstance(value, (bool, int, float)): - return "**Configured Value**" - return str(value) + """ + Masks sensitive values like API keys in logs and output + """ + # Return masked value for sensitive data + if value and isinstance(value, str) and len(value) > 8: + return value[:4] + "*" * (len(value) - 4) + return value def get_provider_status(): """ Returns status information for all configured providers - - Font Awesome icons used: - - fa-brands fa-dropbox: Dropbox icon - - fa-solid fa-envelope: Email icon - - fa-solid fa-server: FTP Server icon - - fa-brands fa-google-drive: Google Drive icon - - fa-solid fa-cloud: NextCloud icon - - fa-brands fa-microsoft: Microsoft/OneDrive icon - - fa-solid fa-file-lines: Document/Paperless icon - - fa-brands fa-aws: AWS/S3 icon - - fa-solid fa-lock: SFTP icon (secure) - - fa-solid fa-heart-pulse: Uptime/health monitoring - - fa-solid fa-globe: WebDAV/web icon """ providers = {} + # Add AI services first + providers["OpenAI"] = { + "name": "OpenAI", + "icon": "fa-brands fa-openai", + "configured": bool(getattr(settings, 'openai_api_key', None) and + str(getattr(settings, 'openai_api_key', '')).startswith('sk-')), + "enabled": True, + "description": "AI-powered document analysis and metadata extraction", + "details": { + "api_key": mask_sensitive_value(getattr(settings, 'openai_api_key', None)), + "base_url": getattr(settings, 'openai_base_url', 'https://api.openai.com/v1'), + "model": getattr(settings, 'openai_model', 'gpt-4') + } + } + + providers["Azure AI"] = { + "name": "Azure AI", + "icon": "fa-solid fa-robot", + "configured": bool(getattr(settings, 'azure_ai_key', None) and + getattr(settings, 'azure_endpoint', None)), + "enabled": True, + "description": "Microsoft Azure Document Intelligence", + "details": { + "api_key": mask_sensitive_value(getattr(settings, 'azure_ai_key', None)), + "endpoint": getattr(settings, 'azure_endpoint', 'Not set'), + "region": getattr(settings, 'azure_region', 'Not set') + } + } + # Add Dropbox configuration - alphabetically ordered providers providers["Dropbox"] = { "name": "Dropbox", @@ -210,7 +219,7 @@ def get_provider_status(): # Add FTP configuration to providers providers["FTP Storage"] = { - "name": "FTP Storage", + "name": "FTP Storage", "icon": "fa-solid fa-server", "configured": bool(getattr(settings, 'ftp_host', None) and getattr(settings, 'ftp_username', None) and @@ -301,7 +310,7 @@ def get_provider_status(): # Check S3 configuration providers["S3 Storage"] = { - "name": "S3 Storage", + "name": "S3 Storage", "icon": "fa-brands fa-aws", "configured": bool(getattr(settings, 's3_bucket_name', None) and getattr(settings, 'aws_access_key_id', None) and @@ -339,10 +348,10 @@ def get_provider_status(): "folder": getattr(settings, 'sftp_folder', 'Not set') } } - + # Add Uptime Kuma configuration providers["Uptime Kuma"] = { - "name": "Uptime Kuma", + "name": "Uptime Kuma", "icon": "fa-solid fa-heart-pulse", "configured": bool(getattr(settings, 'uptime_kuma_url', None)), "enabled": True, @@ -352,7 +361,7 @@ def get_provider_status(): "ping_interval": getattr(settings, 'uptime_kuma_ping_interval', 'Not set') } } - + # Check WebDAV configuration providers["WebDAV"] = { "name": "WebDAV", @@ -394,7 +403,7 @@ def dump_all_settings(): def get_settings_for_display(show_values=False): """ Group settings into logical categories and check if they are configured. - Returns a dictionary with categories as keys and lists of setting items as values. + Returns a dictionary with categories as keys and lists of setting items as values. Each setting item is a dict with name, value, and is_configured. If show_values is False, sensitive values are masked. @@ -403,7 +412,7 @@ def get_settings_for_display(show_values=False): result = { "System Info": [ { - "name": "App Version", + "name": "App Version", "value": settings.version, "is_configured": True } diff --git a/app/views/status.py b/app/views/status.py index a7297d34..92d69818 100644 --- a/app/views/status.py +++ b/app/views/status.py @@ -2,12 +2,15 @@ Status and configuration views for the application. """ from fastapi import Request +from fastapi.responses import JSONResponse from datetime import datetime import os import subprocess +import logging from app.views.base import APIRouter, templates, require_login, settings +logger = logging.getLogger(__name__) router = APIRouter() @router.get("/status") diff --git a/build-setup.sh b/build-setup.sh new file mode 100644 index 00000000..e69de29b diff --git a/docker-build.sh b/docker-build.sh new file mode 100644 index 00000000..e69de29b diff --git a/docker/build-scripts/container-init.sh b/docker/build-scripts/container-init.sh new file mode 100644 index 00000000..e69de29b diff --git a/frontend/templates/status_dashboard.html b/frontend/templates/status_dashboard.html index 66baea96..b9460f23 100644 --- a/frontend/templates/status_dashboard.html +++ b/frontend/templates/status_dashboard.html @@ -136,6 +136,22 @@ Configure Now {% endif %} + {% elif name == "OpenAI" %} + {% if provider.configured %} + + {% endif %} + {% elif name == "Azure AI" %} + {% if provider.configured %} + + {% endif %} {% elif provider.configured and name == "Paperless-ngx" %} @@ -380,6 +396,10 @@ document.addEventListener('DOMContentLoaded', function() { endpoint = '/api/dropbox/test-token'; } else if (provider === 'onedrive') { endpoint = '/api/onedrive/test-token'; + } else if (provider === 'openai') { + endpoint = '/api/openai/test'; + } else if (provider === 'azure') { + endpoint = '/api/azure/test'; } fetch(endpoint) From 9823935927a9a326ecb80e23db8fcb0b75b81333 Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Wed, 9 Apr 2025 07:29:42 +0200 Subject: [PATCH 3/3] feat: Replace PyMuPDF with PyPDF2 for PDF metadata editing and text extraction --- app/tasks/embed_metadata_into_pdf.py | 35 +++++++++++++++++----------- app/tasks/process_document.py | 20 +++++++++------- requirements.txt | 3 +-- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/app/tasks/embed_metadata_into_pdf.py b/app/tasks/embed_metadata_into_pdf.py index d5d59254..6828e9a0 100644 --- a/app/tasks/embed_metadata_into_pdf.py +++ b/app/tasks/embed_metadata_into_pdf.py @@ -2,7 +2,7 @@ import os import shutil -import fitz # PyMuPDF for PDF metadata editing +import PyPDF2 # Replace fitz with PyPDF2 import json from app.config import settings from app.tasks.retry_config import BaseTaskWithRetry @@ -51,7 +51,6 @@ def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata: After processing, the file is moved to /processed/ where is derived from metadata["filename"]. - The output PDF is saved incrementally while preserving its original encryption. Additionally, the metadata is persisted to a JSON file with the same base name. """ # Check for file existence; if not found, try the known shared tmp directory. @@ -74,18 +73,26 @@ def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata: try: print(f"[DEBUG] Embedding metadata into {processed_file}...") - # Open the PDF - doc = fitz.open(processed_file) - # Set PDF metadata using only the standard keys. - doc.set_metadata({ - "title": metadata.get("filename", "Unknown Document"), - "author": metadata.get("absender", "Unknown"), - "subject": metadata.get("document_type", "Unknown"), - "keywords": ", ".join(metadata.get("tags", [])) - }) - # Save incrementally and preserve encryption - doc.save(processed_file, incremental=True, encryption=fitz.PDF_ENCRYPT_KEEP) - doc.close() + # Open the PDF and modify metadata + with open(processed_file, 'rb') as file: + pdf_reader = PyPDF2.PdfReader(file) + pdf_writer = PyPDF2.PdfWriter() + + # Copy all pages from the reader to the writer + for page in pdf_reader.pages: + pdf_writer.add_page(page) + + # Set PDF metadata + pdf_writer.add_metadata({ + "/Title": metadata.get("filename", "Unknown Document"), + "/Author": metadata.get("absender", "Unknown"), + "/Subject": metadata.get("document_type", "Unknown"), + "/Keywords": ", ".join(metadata.get("tags", [])) + }) + + # Write the modified PDF + with open(processed_file, 'wb') as output_file: + pdf_writer.write(output_file) print(f"[INFO] Metadata embedded successfully in {processed_file}") diff --git a/app/tasks/process_document.py b/app/tasks/process_document.py index 880264d2..fa795600 100644 --- a/app/tasks/process_document.py +++ b/app/tasks/process_document.py @@ -4,7 +4,7 @@ import os import uuid import shutil import mimetypes -import fitz # PyMuPDF for checking embedded text +import PyPDF2 # Replace fitz with PyPDF2 from app.config import settings from app.tasks.retry_config import BaseTaskWithRetry @@ -81,19 +81,23 @@ def process_document(original_local_file: str): db.commit() # 2. Check for embedded text (outside the DB session to avoid long open transactions) - pdf_doc = fitz.open(new_local_path) - has_text = any(page.get_text() for page in pdf_doc) - pdf_doc.close() + with open(new_local_path, 'rb') as file: + pdf_reader = PyPDF2.PdfReader(file) + has_text = False + for page in pdf_reader.pages: + if page.extract_text().strip(): + has_text = True + break if has_text: print(f"[INFO] PDF {original_local_file} contains embedded text. Processing locally.") # Extract text locally extracted_text = "" - pdf_doc = fitz.open(new_local_path) - for page in pdf_doc: - extracted_text += page.get_text("text") + "\n" - pdf_doc.close() + with open(new_local_path, 'rb') as file: + pdf_reader = PyPDF2.PdfReader(file) + for page in pdf_reader.pages: + extracted_text += page.extract_text() + "\n" # Call metadata extraction directly extract_metadata_with_gpt.delay(new_filename, extracted_text) diff --git a/requirements.txt b/requirements.txt index eebca058..7457c3f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,8 +5,7 @@ redis # Message broker for Celery sqlalchemy # Database ORM pydantic # Data validation openai # GPT integration for metadata extraction -pymupdf # PDF processing, text extraction, and detection (imported as 'fitz') -PyPDF2 # PDF processing for page counting and now also for rotation +PyPDF2>=3.0.0 # PDF processing for text extraction, metadata editing and rotation (replaces PyMuPDF) requests # HTTP client dropbox>=11.36.0 # Dropbox integration azure-ai-documentintelligence # Azure OCR service