From e1bb9de915318339a7b2b346a7cdfc609054eae8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 7 Feb 2026 22:31:06 +0000 Subject: [PATCH] Add settings management infrastructure: models, API, views, and database loading Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- app/api/__init__.py | 2 + app/api/settings.py | 271 +++++++++++++++++++++++++ app/main.py | 13 ++ app/models.py | 10 + app/utils/config_loader.py | 138 +++++++++++++ app/utils/settings_service.py | 326 +++++++++++++++++++++++++++++++ app/views/__init__.py | 2 + app/views/settings.py | 78 ++++++++ frontend/templates/base.html | 2 + frontend/templates/settings.html | 251 ++++++++++++++++++++++++ 10 files changed, 1093 insertions(+) create mode 100644 app/api/settings.py create mode 100644 app/utils/config_loader.py create mode 100644 app/utils/settings_service.py create mode 100644 app/views/settings.py create mode 100644 frontend/templates/settings.html diff --git a/app/api/__init__.py b/app/api/__init__.py index a621adb6..80ed4fdc 100644 --- a/app/api/__init__.py +++ b/app/api/__init__.py @@ -15,6 +15,7 @@ from app.api.openai import router as openai_router from app.api.azure import router as azure_router from app.api.google_drive import router as google_drive_router from app.api.logs import router as logs_router +from app.api.settings import router as settings_router # Set up logging logger = logging.getLogger(__name__) @@ -33,3 +34,4 @@ router.include_router(openai_router) router.include_router(azure_router) router.include_router(google_drive_router) router.include_router(logs_router) +router.include_router(settings_router) diff --git a/app/api/settings.py b/app/api/settings.py new file mode 100644 index 00000000..16c0ac27 --- /dev/null +++ b/app/api/settings.py @@ -0,0 +1,271 @@ +""" +API endpoints for managing application settings. +""" + +import logging +from typing import Dict, Any, Optional +from fastapi import APIRouter, Depends, HTTPException, Request, status +from sqlalchemy.orm import Session +from pydantic import BaseModel, Field + +from app.database import get_db +from app.config import settings +from app.utils.settings_service import ( + get_all_settings_from_db, + save_setting_to_db, + delete_setting_from_db, + get_setting_metadata, + get_settings_by_category, + validate_setting_value, + SETTING_METADATA, +) + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/api/settings", tags=["settings"]) + + +def require_admin(request: Request): + """ + Dependency to ensure the user is an admin. + """ + user = request.session.get("user") + if not user or not user.get("is_admin"): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin access required" + ) + return user + + +class SettingUpdate(BaseModel): + """Model for updating a setting""" + key: str = Field(..., description="Setting key") + value: Optional[str] = Field(None, description="Setting value (None to delete)") + + +class SettingResponse(BaseModel): + """Model for setting response""" + key: str + value: Optional[str] + metadata: Dict[str, Any] + + +class SettingsListResponse(BaseModel): + """Model for list of settings""" + settings: Dict[str, Any] + categories: Dict[str, list] + db_settings: Dict[str, str] + + +@router.get("/", response_model=SettingsListResponse) +async def get_settings( + request: Request, + db: Session = Depends(get_db), + admin: dict = Depends(require_admin) +): + """ + Get all application settings with metadata. + Admin only. + """ + try: + # Get current runtime settings + current_settings = {} + for key in SETTING_METADATA.keys(): + if hasattr(settings, key): + value = getattr(settings, key) + current_settings[key] = { + "value": value, + "metadata": get_setting_metadata(key) + } + + # Get settings stored in database + db_settings = get_all_settings_from_db(db) + + # Get settings organized by category + categories = get_settings_by_category() + + return SettingsListResponse( + settings=current_settings, + categories=categories, + db_settings=db_settings + ) + except Exception as e: + logger.error(f"Error retrieving settings: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to retrieve settings" + ) + + +@router.get("/{key}", response_model=SettingResponse) +async def get_setting( + key: str, + request: Request, + db: Session = Depends(get_db), + admin: dict = Depends(require_admin) +): + """ + Get a specific setting by key. + Admin only. + """ + try: + # Get current value + value = getattr(settings, key, None) + + # Get metadata + metadata = get_setting_metadata(key) + + return SettingResponse( + key=key, + value=str(value) if value is not None else None, + metadata=metadata + ) + except Exception as e: + logger.error(f"Error retrieving setting {key}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to retrieve setting: {key}" + ) + + +@router.post("/{key}") +async def update_setting( + key: str, + setting: SettingUpdate, + request: Request, + db: Session = Depends(get_db), + admin: dict = Depends(require_admin) +): + """ + Update a specific setting. + Admin only. + """ + try: + # Validate the setting value + if setting.value is not None: + is_valid, error_message = validate_setting_value(key, setting.value) + if not is_valid: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=error_message + ) + + # Save to database + success = save_setting_to_db(db, key, setting.value) + if not success: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to save setting to database" + ) + + # Get metadata + metadata = get_setting_metadata(key) + restart_required = metadata.get("restart_required", False) + + return { + "success": True, + "message": f"Setting '{key}' updated successfully", + "restart_required": restart_required, + "key": key, + "value": setting.value + } + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating setting {key}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to update setting: {key}" + ) + + +@router.delete("/{key}") +async def delete_setting( + key: str, + request: Request, + db: Session = Depends(get_db), + admin: dict = Depends(require_admin) +): + """ + Delete a setting from the database (reverts to environment variable or default). + Admin only. + """ + try: + success = delete_setting_from_db(db, key) + if not success: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Setting '{key}' not found in database" + ) + + return { + "success": True, + "message": f"Setting '{key}' deleted from database (will use environment variable or default)" + } + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting setting {key}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to delete setting: {key}" + ) + + +@router.post("/bulk-update") +async def bulk_update_settings( + updates: list[SettingUpdate], + request: Request, + db: Session = Depends(get_db), + admin: dict = Depends(require_admin) +): + """ + Update multiple settings at once. + Admin only. + """ + results = [] + errors = [] + + for update in updates: + try: + # Validate the setting value + if update.value is not None: + is_valid, error_message = validate_setting_value(update.key, update.value) + if not is_valid: + errors.append({ + "key": update.key, + "error": error_message + }) + continue + + # Save to database + success = save_setting_to_db(db, update.key, update.value) + if success: + results.append({ + "key": update.key, + "value": update.value, + "status": "success" + }) + else: + errors.append({ + "key": update.key, + "error": "Failed to save to database" + }) + except Exception as e: + logger.error(f"Error updating setting {update.key}: {e}") + errors.append({ + "key": update.key, + "error": str(e) + }) + + restart_required = any( + get_setting_metadata(result["key"]).get("restart_required", False) + for result in results + ) + + return { + "success": len(errors) == 0, + "updated": results, + "errors": errors, + "restart_required": restart_required + } diff --git a/app/main.py b/app/main.py index 6195838f..64961a07 100644 --- a/app/main.py +++ b/app/main.py @@ -59,6 +59,19 @@ else: @app.on_event("startup") def on_startup(): init_db() # Create tables if they don't exist + + # Load settings from database after DB initialization + from app.database import SessionLocal + from app.utils.config_loader import load_settings_from_db + + db = SessionLocal() + try: + load_settings_from_db(settings, db) + logging.info("Database settings loaded successfully") + except Exception as e: + logging.error(f"Failed to load database settings: {e}") + finally: + db.close() @app.on_event("startup") async def startup_event(): diff --git a/app/models.py b/app/models.py index 05cd4a7a..0ef98b20 100644 --- a/app/models.py +++ b/app/models.py @@ -47,3 +47,13 @@ class ProcessingLog(Base): status = Column(String) # "pending", "in_progress", "success", "failure" message = Column(String, nullable=True) # Error text or success note timestamp = Column(DateTime(timezone=True), server_default=func.now()) + +class ApplicationSettings(Base): + """Store application settings in database with precedence over environment variables""" + __tablename__ = "application_settings" + + id = Column(Integer, primary_key=True, index=True) + key = Column(String, unique=True, index=True, nullable=False) # Setting key (e.g., 'database_url') + value = Column(String, nullable=True) # Setting value (stored as string, converted as needed) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) diff --git a/app/utils/config_loader.py b/app/utils/config_loader.py new file mode 100644 index 00000000..d98b078f --- /dev/null +++ b/app/utils/config_loader.py @@ -0,0 +1,138 @@ +""" +Configuration loader that supports database-persisted settings with precedence. + +This module provides functionality to: +- Load settings from database after app initialization +- Apply database settings over environment variables +- Dynamically reload settings when changed +""" + +import logging +from typing import Any, Optional +from sqlalchemy.orm import Session + +from app.models import ApplicationSettings + +logger = logging.getLogger(__name__) + + +def load_settings_from_db(settings_obj, db_session: Session) -> None: + """ + Load settings from database and apply them to the settings object. + + Database settings take precedence over environment variables and defaults. + This function should be called after database initialization. + + Args: + settings_obj: The Settings instance to update + db_session: Database session to use for loading settings + """ + try: + db_settings = db_session.query(ApplicationSettings).all() + + if not db_settings: + logger.info("No database settings found, using environment/defaults") + return + + # Apply database settings to the settings object + updated_count = 0 + for db_setting in db_settings: + key = db_setting.key + value = db_setting.value + + # Check if the setting exists in the Settings class + if hasattr(settings_obj, key): + # Get the field info to determine the type + field_info = settings_obj.__fields__.get(key) + if field_info: + # Convert value to the appropriate type + converted_value = convert_setting_value(value, field_info.annotation) + + # Set the attribute + setattr(settings_obj, key, converted_value) + updated_count += 1 + logger.debug(f"Applied database setting: {key}") + + if updated_count > 0: + logger.info(f"Loaded {updated_count} settings from database") + else: + logger.info("No applicable database settings found") + + except Exception as e: + logger.error(f"Error loading settings from database: {e}") + # Don't fail application startup if database settings can't be loaded + logger.warning("Continuing with environment/default settings") + + +def convert_setting_value(value: Optional[str], field_type: Any) -> Any: + """ + Convert a string value from database to the appropriate type. + + Args: + value: String value from database + field_type: Target type from Pydantic field annotation + + Returns: + Converted value in the appropriate type + """ + if value is None: + return None + + # Handle Optional types + origin = getattr(field_type, '__origin__', None) + if origin is Union: + # Get the non-None type from Union (for Optional) + args = getattr(field_type, '__args__', ()) + field_type = next((arg for arg in args if arg is not type(None)), str) + + # Convert based on type + if field_type == bool: + return value.lower() in ('true', '1', 'yes', 'y', 't') + elif field_type == int: + try: + return int(value) + except ValueError: + logger.warning(f"Failed to convert '{value}' to int, returning 0") + return 0 + elif field_type == float: + try: + return float(value) + except ValueError: + logger.warning(f"Failed to convert '{value}' to float, returning 0.0") + return 0.0 + elif field_type == list or getattr(field_type, '__origin__', None) == list: + # Handle list types - assume comma-separated values + if isinstance(value, str): + return [item.strip() for item in value.split(',') if item.strip()] + return value + else: + # Default to string + return str(value) + + +def reload_settings_from_db(settings_obj) -> bool: + """ + Reload settings from database. + + This is useful after settings have been updated through the UI. + Note: Some settings require application restart to take effect. + + Args: + settings_obj: The Settings instance to update + + Returns: + True if reload was successful, False otherwise + """ + try: + from app.database import SessionLocal + + db = SessionLocal() + try: + load_settings_from_db(settings_obj, db) + logger.info("Settings reloaded from database") + return True + finally: + db.close() + except Exception as e: + logger.error(f"Error reloading settings from database: {e}") + return False diff --git a/app/utils/settings_service.py b/app/utils/settings_service.py new file mode 100644 index 00000000..410e1032 --- /dev/null +++ b/app/utils/settings_service.py @@ -0,0 +1,326 @@ +""" +Service for managing application settings with database persistence. + +This module provides functionality to: +- Load settings from database with precedence over environment variables +- Save settings to database +- Get setting metadata (descriptions, types, categories) +""" + +import logging +from typing import Any, Dict, List, Optional +from sqlalchemy.orm import Session +from sqlalchemy.exc import SQLAlchemyError + +from app.models import ApplicationSettings + +logger = logging.getLogger(__name__) + +# Define setting metadata for UI display +SETTING_METADATA = { + # Core Settings + "database_url": { + "category": "Core", + "description": "Database connection URL (e.g., sqlite:///path/to/db.sqlite)", + "type": "string", + "sensitive": False, + "required": True, + "restart_required": True, + }, + "redis_url": { + "category": "Core", + "description": "Redis connection URL for Celery task queue", + "type": "string", + "sensitive": False, + "required": True, + "restart_required": True, + }, + "workdir": { + "category": "Core", + "description": "Working directory for file storage and processing", + "type": "string", + "sensitive": False, + "required": True, + "restart_required": True, + }, + "external_hostname": { + "category": "Core", + "description": "External hostname for the application (e.g., docuelevate.example.com)", + "type": "string", + "sensitive": False, + "required": True, + "restart_required": True, + }, + "debug": { + "category": "Core", + "description": "Enable debug mode for verbose logging", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "allow_file_delete": { + "category": "Core", + "description": "Allow deleting files from the database", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "gotenberg_url": { + "category": "Core", + "description": "Gotenberg service URL for document conversion", + "type": "string", + "sensitive": False, + "required": True, + "restart_required": True, + }, + + # Authentication Settings + "auth_enabled": { + "category": "Authentication", + "description": "Enable authentication for the application", + "type": "boolean", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "session_secret": { + "category": "Authentication", + "description": "Secret key for session encryption (min 32 characters)", + "type": "string", + "sensitive": True, + "required": True, + "restart_required": True, + }, + "admin_username": { + "category": "Authentication", + "description": "Admin username for local authentication", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": True, + }, + "admin_password": { + "category": "Authentication", + "description": "Admin password for local authentication", + "type": "string", + "sensitive": True, + "required": False, + "restart_required": True, + }, + + # AI Services + "openai_api_key": { + "category": "AI Services", + "description": "OpenAI API key for metadata extraction", + "type": "string", + "sensitive": True, + "required": True, + "restart_required": False, + }, + "openai_base_url": { + "category": "AI Services", + "description": "OpenAI API base URL (default: https://api.openai.com/v1)", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "openai_model": { + "category": "AI Services", + "description": "OpenAI model to use (e.g., gpt-4o-mini)", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }, + "azure_ai_key": { + "category": "AI Services", + "description": "Azure AI key for document intelligence", + "type": "string", + "sensitive": True, + "required": True, + "restart_required": False, + }, + "azure_region": { + "category": "AI Services", + "description": "Azure region for AI services", + "type": "string", + "sensitive": False, + "required": True, + "restart_required": False, + }, + "azure_endpoint": { + "category": "AI Services", + "description": "Azure AI endpoint URL", + "type": "string", + "sensitive": False, + "required": True, + "restart_required": False, + }, + + # Add more settings metadata as needed... +} + + +def get_setting_from_db(db: Session, key: str) -> Optional[str]: + """ + Retrieve a setting value from the database. + + Args: + db: Database session + key: Setting key to retrieve + + Returns: + Setting value as string, or None if not found + """ + try: + setting = db.query(ApplicationSettings).filter(ApplicationSettings.key == key).first() + return setting.value if setting else None + except SQLAlchemyError as e: + logger.error(f"Error retrieving setting {key} from database: {e}") + return None + + +def save_setting_to_db(db: Session, key: str, value: Optional[str]) -> bool: + """ + Save or update a setting in the database. + + Args: + db: Database session + key: Setting key + value: Setting value (as string) + + Returns: + True if successful, False otherwise + """ + try: + setting = db.query(ApplicationSettings).filter(ApplicationSettings.key == key).first() + if setting: + setting.value = value + else: + setting = ApplicationSettings(key=key, value=value) + db.add(setting) + db.commit() + logger.info(f"Saved setting {key} to database") + return True + except SQLAlchemyError as e: + logger.error(f"Error saving setting {key} to database: {e}") + db.rollback() + return False + + +def get_all_settings_from_db(db: Session) -> Dict[str, str]: + """ + Retrieve all settings from the database. + + Args: + db: Database session + + Returns: + Dictionary of setting key-value pairs + """ + try: + settings = db.query(ApplicationSettings).all() + return {setting.key: setting.value for setting in settings} + except SQLAlchemyError as e: + logger.error(f"Error retrieving all settings from database: {e}") + return {} + + +def delete_setting_from_db(db: Session, key: str) -> bool: + """ + Delete a setting from the database. + + Args: + db: Database session + key: Setting key to delete + + Returns: + True if successful, False otherwise + """ + try: + setting = db.query(ApplicationSettings).filter(ApplicationSettings.key == key).first() + if setting: + db.delete(setting) + db.commit() + logger.info(f"Deleted setting {key} from database") + return True + return False + except SQLAlchemyError as e: + logger.error(f"Error deleting setting {key} from database: {e}") + db.rollback() + return False + + +def get_setting_metadata(key: str) -> Dict[str, Any]: + """ + Get metadata for a specific setting. + + Args: + key: Setting key + + Returns: + Dictionary containing setting metadata + """ + return SETTING_METADATA.get(key, { + "category": "Other", + "description": f"Setting: {key}", + "type": "string", + "sensitive": False, + "required": False, + "restart_required": False, + }) + + +def get_settings_by_category() -> Dict[str, List[str]]: + """ + Get settings organized by category. + + Returns: + Dictionary mapping category names to lists of setting keys + """ + categories = {} + for key, metadata in SETTING_METADATA.items(): + category = metadata.get("category", "Other") + if category not in categories: + categories[category] = [] + categories[category].append(key) + return categories + + +def validate_setting_value(key: str, value: str) -> tuple[bool, Optional[str]]: + """ + Validate a setting value based on its metadata. + + Args: + key: Setting key + value: Setting value to validate + + Returns: + Tuple of (is_valid, error_message) + """ + metadata = get_setting_metadata(key) + setting_type = metadata.get("type", "string") + + # Check required fields + if metadata.get("required", False) and not value: + return False, f"{key} is required" + + # Type-specific validation + if setting_type == "boolean": + if value.lower() not in ["true", "false", "1", "0", "yes", "no"]: + return False, f"{key} must be a boolean value (true/false)" + + elif setting_type == "integer": + try: + int(value) + except ValueError: + return False, f"{key} must be an integer" + + # Special validation for specific keys + if key == "session_secret" and value and len(value) < 32: + return False, "session_secret must be at least 32 characters" + + return True, None diff --git a/app/views/__init__.py b/app/views/__init__.py index 9e90df7f..630387eb 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -10,6 +10,7 @@ from app.views.onedrive import router as onedrive_router from app.views.dropbox import router as dropbox_router from app.views.google_drive import router as google_drive_router from app.views.license_routes import router as license_router # Add the license router +from app.views.settings import router as settings_router # Create a main router that includes all the view routers router = APIRouter() @@ -19,3 +20,4 @@ router.include_router(onedrive_router) router.include_router(dropbox_router) router.include_router(google_drive_router) router.include_router(license_router) # Include the license router +router.include_router(settings_router) diff --git a/app/views/settings.py b/app/views/settings.py new file mode 100644 index 00000000..162f6ef9 --- /dev/null +++ b/app/views/settings.py @@ -0,0 +1,78 @@ +""" +Settings management views for the application. +""" + +import logging +from fastapi import Request, Depends, HTTPException, status +from fastapi.responses import RedirectResponse +from sqlalchemy.orm import Session + +from app.views.base import APIRouter, templates, require_login, settings, get_db +from app.utils.settings_service import get_settings_by_category, get_setting_metadata, SETTING_METADATA +from app.utils.config_validator.masking import mask_sensitive_value + +logger = logging.getLogger(__name__) +router = APIRouter() + + +def require_admin_access(request: Request): + """Check if user is admin and redirect if not""" + user = request.session.get("user") + if not user or not user.get("is_admin"): + logger.warning(f"Non-admin user attempted to access settings page") + return RedirectResponse(url="/", status_code=status.HTTP_302_FOUND) + return None + + +@router.get("/settings") +@require_login +async def settings_page(request: Request, db: Session = Depends(get_db)): + """ + Settings management page - admin only. + """ + # Check admin access + redirect = require_admin_access(request) + if redirect: + return redirect + + try: + # Get settings organized by category + categories = get_settings_by_category() + + # Build settings data for display + settings_data = {} + for category, keys in categories.items(): + settings_data[category] = [] + for key in keys: + # Get current value from settings + value = getattr(settings, key, None) + + # Get metadata + metadata = get_setting_metadata(key) + + # Mask sensitive values + display_value = value + if metadata.get("sensitive") and value: + display_value = mask_sensitive_value(value) + + settings_data[category].append({ + "key": key, + "value": value if value is not None else "", + "display_value": display_value if display_value is not None else "", + "metadata": metadata + }) + + return templates.TemplateResponse( + "settings.html", + { + "request": request, + "settings_data": settings_data, + "app_version": settings.version + } + ) + except Exception as e: + logger.error(f"Error loading settings page: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to load settings page" + ) diff --git a/frontend/templates/base.html b/frontend/templates/base.html index 8ee7087f..a7263607 100644 --- a/frontend/templates/base.html +++ b/frontend/templates/base.html @@ -47,6 +47,7 @@ Upload Files Status + Settings About @@ -82,6 +83,7 @@ Upload Files Status + Settings About diff --git a/frontend/templates/settings.html b/frontend/templates/settings.html new file mode 100644 index 00000000..df6a3aca --- /dev/null +++ b/frontend/templates/settings.html @@ -0,0 +1,251 @@ +{% extends "base.html" %} +{% block title %}Settings - DocuElevate{% endblock %} + +{% block head_extra %} + +{% endblock %} + +{% block content %} +
+ Configure application settings through the web interface. + Settings saved here will take precedence over environment variables. +
+⚠️ Important Notes:
+