From 05d03531b9de15c7acb1139980c380f1697537c0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 23 Feb 2026 02:14:05 +0000 Subject: [PATCH 1/8] Initial plan From 90e5e0037c30fc9225ee106b7ed2ee60bc49db20 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 23 Feb 2026 02:28:08 +0000 Subject: [PATCH 2/8] feat(settings): per-option save, live worker sync, audit log, and rollback A) Per-option Save Button - Add per-setting Save button in settings.html (visible only when value changed) - Button calls POST /api/settings/{key} directly; existing bulk Save retained - Add Audit Log link in settings page header B) Immediate Worker Sync - New app/utils/settings_sync.py with notify_settings_updated() (Redis version key) and register_settings_reload_signal() (Celery task_prerun handler) - Register signal in celery_worker.py at startup - All API write paths call notify_settings_updated() after successful saves C) Audit Log - Add SettingsAuditLog model (key, old_value, new_value, changed_by, changed_at, action) - save_setting_to_db / delete_setting_from_db accept changed_by and write audit entries - New get_audit_log() service function (masks sensitive values) - New GET /api/settings/audit-log endpoint (admin-only) - New GET /admin/settings/audit-log view + audit_log.html template - Visible to all admins (per clarified requirement) D) Config Rollback / History - New get_setting_history() and rollback_setting() service functions - New GET /api/settings/{key}/history endpoint - New POST /api/settings/{key}/rollback/{history_id} endpoint - Rollback buttons in audit_log.html with confirmation dialog - Tests: 25 new tests covering audit log, rollback, worker sync helpers, and API endpoints Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- app/api/settings.py | 240 ++++++++++++++-- app/celery_worker.py | 26 +- app/models.py | 55 +++- app/utils/settings_service.py | 246 +++++++++++++++- app/utils/settings_sync.py | 86 ++++++ app/views/settings.py | 70 ++++- frontend/templates/audit_log.html | 157 +++++++++++ frontend/templates/settings.html | 65 ++++- tests/test_settings_audit_log.py | 452 ++++++++++++++++++++++++++++++ 9 files changed, 1321 insertions(+), 76 deletions(-) create mode 100644 app/utils/settings_sync.py create mode 100644 frontend/templates/audit_log.html create mode 100644 tests/test_settings_audit_log.py diff --git a/app/api/settings.py b/app/api/settings.py index 0dad89b6..dd82bbb4 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -11,16 +11,17 @@ from sqlalchemy.orm import Session from app.config import settings from app.database import get_db -from app.utils.input_validation import validate_setting_key, validate_setting_key_format -from app.utils.settings_service import ( - SETTING_METADATA, - delete_setting_from_db, - get_all_settings_from_db, - get_setting_metadata, - get_settings_by_category, - save_setting_to_db, - validate_setting_value, -) +from app.utils.input_validation import (validate_setting_key, + validate_setting_key_format) +from app.utils.settings_service import (SETTING_METADATA, + delete_setting_from_db, + get_all_settings_from_db, + get_audit_log, get_setting_history, + get_setting_metadata, + get_settings_by_category, + rollback_setting, save_setting_to_db, + validate_setting_value) +from app.utils.settings_sync import notify_settings_updated logger = logging.getLogger(__name__) router = APIRouter(prefix="/settings", tags=["settings"]) @@ -36,7 +37,9 @@ def require_admin(request: Request) -> dict: """ user = request.session.get("user") if not user or not user.get("is_admin"): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required" + ) return user @@ -79,7 +82,10 @@ async def get_settings(request: Request, db: DbSession, admin: AdminUser): for key in SETTING_METADATA.keys(): if hasattr(settings, key): value = getattr(settings, key) - current_settings[key] = {"value": value, "metadata": get_setting_metadata(key)} + current_settings[key] = { + "value": value, + "metadata": get_setting_metadata(key), + } # Get settings stored in database db_settings = get_all_settings_from_db(db) @@ -87,10 +93,15 @@ async def get_settings(request: Request, db: DbSession, admin: AdminUser): # Get settings organized by category categories = get_settings_by_category() - return SettingsListResponse(settings=current_settings, categories=categories, db_settings=db_settings) + return SettingsListResponse( + settings=current_settings, categories=categories, db_settings=db_settings + ) except Exception as e: logger.error(f"Error retrieving settings: {e}") - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to retrieve settings") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to retrieve settings", + ) @router.get("/{key}", response_model=SettingResponse) @@ -107,11 +118,14 @@ async def get_setting(key: str, request: Request, db: DbSession, admin: AdminUse # Get metadata metadata = get_setting_metadata(key) - return SettingResponse(key=key, value=str(value) if value is not None else None, metadata=metadata) + return SettingResponse( + key=key, value=str(value) if value is not None else None, metadata=metadata + ) except Exception as e: logger.error(f"Error retrieving setting {key}: {e}") raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to retrieve setting: {key}" + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to retrieve setting: {key}", ) @@ -133,15 +147,31 @@ async def update_setting( if setting.value is not None: is_valid, error_message = validate_setting_value(key, setting.value) if not is_valid: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=error_message) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=error_message + ) + + # Determine the username for the audit log + user = request.session.get("user", {}) if hasattr(request, "session") else {} + changed_by = ( + user.get("preferred_username") + or user.get("username") + or user.get("email") + or user.get("id") + or "admin" + ) # Save to database - success = save_setting_to_db(db, key, setting.value) + success = save_setting_to_db(db, key, setting.value, changed_by=changed_by) if not success: raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to save setting to database" + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to save setting to database", ) + # Notify workers that settings have changed + notify_settings_updated() + # Get metadata metadata = get_setting_metadata(key) restart_required = metadata.get("restart_required", False) @@ -158,7 +188,8 @@ async def update_setting( except Exception as e: logger.error(f"Error updating setting {key}: {e}") raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to update setting: {key}" + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to update setting: {key}", ) @@ -170,9 +201,23 @@ async def delete_setting(key: str, request: Request, db: DbSession, admin: Admin """ validate_setting_key(key) try: - success = delete_setting_from_db(db, key) + user = request.session.get("user", {}) if hasattr(request, "session") else {} + changed_by = ( + user.get("preferred_username") + or user.get("username") + or user.get("email") + or user.get("id") + or "admin" + ) + + success = delete_setting_from_db(db, key, changed_by=changed_by) if not success: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Setting '{key}' not found in database") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Setting '{key}' not found in database", + ) + + notify_settings_updated() return { "success": True, @@ -183,7 +228,8 @@ async def delete_setting(key: str, request: Request, db: DbSession, admin: Admin except Exception as e: logger.error(f"Error deleting setting {key}: {e}") raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to delete setting: {key}" + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to delete setting: {key}", ) @@ -238,11 +284,16 @@ async def list_credentials(request: Request, db: DbSession, admin: AdminUser): } except Exception as e: logger.error(f"Error retrieving credential list: {e}") - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to retrieve credentials") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to retrieve credentials", + ) @router.post("/bulk-update") -async def bulk_update_settings(updates: list[SettingUpdate], request: Request, db: DbSession, admin: AdminUser): +async def bulk_update_settings( + updates: list[SettingUpdate], request: Request, db: DbSession, admin: AdminUser +): """ Update multiple settings at once. Admin only. @@ -250,25 +301,152 @@ async def bulk_update_settings(updates: list[SettingUpdate], request: Request, d results = [] errors = [] + user = request.session.get("user", {}) if hasattr(request, "session") else {} + changed_by = ( + user.get("preferred_username") + or user.get("username") + or user.get("email") + or user.get("id") + or "admin" + ) + for update in updates: try: # Validate the setting value if update.value is not None: - is_valid, error_message = validate_setting_value(update.key, update.value) + is_valid, error_message = validate_setting_value( + update.key, update.value + ) if not is_valid: errors.append({"key": update.key, "error": error_message}) continue # Save to database - success = save_setting_to_db(db, update.key, update.value) + success = save_setting_to_db( + db, update.key, update.value, changed_by=changed_by + ) if success: - results.append({"key": update.key, "value": update.value, "status": "success"}) + results.append( + {"key": update.key, "value": update.value, "status": "success"} + ) else: - errors.append({"key": update.key, "error": "Failed to save to database"}) + errors.append( + {"key": update.key, "error": "Failed to save to database"} + ) except Exception as e: logger.error(f"Error updating setting {update.key}: {e}") errors.append({"key": update.key, "error": str(e)}) - restart_required = any(get_setting_metadata(result["key"]).get("restart_required", False) for result in results) + if results: + notify_settings_updated() - return {"success": len(errors) == 0, "updated": results, "errors": errors, "restart_required": restart_required} + restart_required = any( + get_setting_metadata(result["key"]).get("restart_required", False) + for result in results + ) + + return { + "success": len(errors) == 0, + "updated": results, + "errors": errors, + "restart_required": restart_required, + } + + +@router.get("/audit-log") +async def list_audit_log( + request: Request, + db: DbSession, + admin: AdminUser, + limit: int = 100, + offset: int = 0, +): + """ + Retrieve the settings audit log (most recent first). + + Returns all configuration changes recorded in the audit log. + Sensitive values are masked in the response. + Admin only. + """ + try: + entries = get_audit_log(db, limit=limit, offset=offset) + return {"entries": entries, "limit": limit, "offset": offset} + except Exception as e: + logger.error(f"Error retrieving audit log: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to retrieve audit log", + ) + + +@router.get("/{key}/history") +async def get_key_history(key: str, request: Request, db: DbSession, admin: AdminUser): + """ + Get the change history for a specific setting key. + + Returns all audit log entries for that key, most recent first. + Admin only. + """ + validate_setting_key_format(key) + try: + entries = get_setting_history(db, key) + return {"key": key, "history": entries} + except Exception as e: + logger.error(f"Error retrieving history for {key}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to retrieve history for setting: {key}", + ) + + +@router.post("/{key}/rollback/{history_id}") +async def rollback_setting_to_history( + key: str, + history_id: int, + request: Request, + db: DbSession, + admin: AdminUser, +): + """ + Revert a setting to the value it held at a specific point in the audit log. + + The ``history_id`` is the ID of the :class:`~app.models.SettingsAuditLog` + entry whose ``new_value`` should be reinstated. If that entry recorded a + deletion (``new_value`` is ``None``), the setting is removed from the + database and reverts to its ENV/default value. + + A new audit log entry is written to record the rollback. + Admin only. + """ + validate_setting_key_format(key) + try: + user = request.session.get("user", {}) if hasattr(request, "session") else {} + changed_by = ( + user.get("preferred_username") + or user.get("username") + or user.get("email") + or user.get("id") + or "admin" + ) + + success = rollback_setting(db, key, history_id, changed_by=changed_by) + if not success: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"History entry {history_id} not found for setting '{key}'", + ) + + notify_settings_updated() + + return { + "success": True, + "message": f"Setting '{key}' rolled back to history entry {history_id}", + } + except HTTPException: + raise + except Exception as e: + logger.error(f"Error rolling back setting {key} to history {history_id}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to roll back setting: {key}", + ) diff --git a/app/celery_worker.py b/app/celery_worker.py index 1ed82467..ddc74225 100644 --- a/app/celery_worker.py +++ b/app/celery_worker.py @@ -3,30 +3,32 @@ from celery.schedules import crontab # Ensure tasks are loaded -from app import tasks # noqa: F401 - Imports app/tasks.py so Celery can register tasks - +from app import \ + tasks # noqa: F401 - Imports app/tasks.py so Celery can register tasks # Import the shared Celery instance from app.celery_app import celery from app.config import settings from app.tasks.check_credentials import check_credentials from app.tasks.convert_to_pdf import convert_to_pdf # noqa: F401 -from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf # noqa: F401 -from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt # noqa: F401 +from app.tasks.embed_metadata_into_pdf import \ + embed_metadata_into_pdf # noqa: F401 +from app.tasks.extract_metadata_with_gpt import \ + extract_metadata_with_gpt # noqa: F401 from app.tasks.imap_tasks import pull_all_inboxes # noqa: F401 from app.tasks.monitor_stalled_steps import monitor_stalled_steps # noqa: F401 - # **Ensure all tasks are imported before Celery starts** from app.tasks.process_document import process_document # noqa: F401 -from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence # noqa: F401 +from app.tasks.process_with_azure_document_intelligence import \ + process_with_azure_document_intelligence # noqa: F401 from app.tasks.refine_text_with_gpt import refine_text_with_gpt # noqa: F401 from app.tasks.rotate_pdf_pages import rotate_pdf_pages # noqa: F401 from app.tasks.send_to_all import send_to_all_destinations # noqa: F401 - # Import new send tasks from app.tasks.upload_to_dropbox import upload_to_dropbox # noqa: F401 from app.tasks.upload_to_email import upload_to_email # noqa: F401 from app.tasks.upload_to_ftp import upload_to_ftp # noqa: F401 -from app.tasks.upload_to_google_drive import upload_to_google_drive # noqa: F401 +from app.tasks.upload_to_google_drive import \ + upload_to_google_drive # noqa: F401 from app.tasks.upload_to_nextcloud import upload_to_nextcloud # noqa: F401 from app.tasks.upload_to_onedrive import upload_to_onedrive # noqa: F401 from app.tasks.upload_to_paperless import upload_to_paperless # noqa: F401 @@ -34,6 +36,10 @@ from app.tasks.upload_to_s3 import upload_to_s3 # noqa: F401 from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401 from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401 from app.tasks.uptime_kuma_tasks import ping_uptime_kuma # noqa: F401 +# Register the settings reload signal handler so workers pick up config changes +from app.utils.settings_sync import register_settings_reload_signal + +register_settings_reload_signal() celery.conf.task_routes = { "app.tasks.*": {"queue": "default"}, @@ -89,4 +95,6 @@ celery.conf.beat_schedule = { } # Remove None entries from beat_schedule -celery.conf.beat_schedule = {k: v for k, v in celery.conf.beat_schedule.items() if v is not None} +celery.conf.beat_schedule = { + k: v for k, v in celery.conf.beat_schedule.items() if v is not None +} diff --git a/app/models.py b/app/models.py index 848f71bf..d62099d2 100644 --- a/app/models.py +++ b/app/models.py @@ -1,6 +1,7 @@ # app/models.py -from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func +from sqlalchemy import (Boolean, Column, DateTime, ForeignKey, Integer, String, + Text, UniqueConstraint, func) from app.database import Base @@ -69,21 +70,33 @@ class FileProcessingStep(Base): id = Column(Integer, primary_key=True, index=True) file_id = Column(Integer, ForeignKey(_FILES_ID_FK), nullable=False, index=True) - step_name = Column(String, nullable=False, index=True) # e.g., "hash_file", "upload_to_dropbox" - status = Column(String, nullable=False) # "pending", "in_progress", "success", "failure", "skipped" + step_name = Column( + String, nullable=False, index=True + ) # e.g., "hash_file", "upload_to_dropbox" + status = Column( + String, nullable=False + ) # "pending", "in_progress", "success", "failure", "skipped" started_at = Column(DateTime(timezone=True), nullable=True) # When step started - completed_at = Column(DateTime(timezone=True), nullable=True) # When step finished (success/failure) + completed_at = Column( + DateTime(timezone=True), nullable=True + ) # When step finished (success/failure) error_message = Column(Text, nullable=True) # Error message if status is "failure" created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) - __table_args__ = (UniqueConstraint("file_id", "step_name", name="unique_file_step"),) + __table_args__ = ( + UniqueConstraint("file_id", "step_name", name="unique_file_step"), + ) class ProcessingLog(Base): __tablename__ = "processing_logs" id = Column(Integer, primary_key=True, index=True) - file_id = Column(Integer, ForeignKey(_FILES_ID_FK), nullable=True) # Optional file association + file_id = Column( + Integer, ForeignKey(_FILES_ID_FK), nullable=True + ) # Optional file association task_id = Column(String, index=True) # Celery task ID step_name = Column(String) # e.g., "OCR", "convert_to_pdf", "upload_s3" status = Column(String) # "pending", "in_progress", "success", "failure" @@ -98,7 +111,29 @@ class ApplicationSettings(Base): __tablename__ = "application_settings" id = Column(Integer, primary_key=True, index=True) - key = Column(String, unique=True, index=True, nullable=False) # Setting key (e.g., 'database_url') - value = Column(String, nullable=True) # Setting value (stored as string, converted as needed) + key = Column( + String, unique=True, index=True, nullable=False + ) # Setting key (e.g., 'database_url') + value = Column( + String, nullable=True + ) # Setting value (stored as string, converted as needed) created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class SettingsAuditLog(Base): + """Audit log for all configuration changes made via the settings UI.""" + + __tablename__ = "settings_audit_log" + + id = Column(Integer, primary_key=True, index=True) + key = Column(String, nullable=False, index=True) # Setting key that was changed + old_value = Column(String, nullable=True) # Previous value (None if first-time set) + new_value = Column(String, nullable=True) # New value (None if deleted) + changed_by = Column( + String, nullable=False + ) # Username of the admin who made the change + changed_at = Column(DateTime(timezone=True), server_default=func.now(), index=True) + action = Column(String, nullable=False) # "update" or "delete" diff --git a/app/utils/settings_service.py b/app/utils/settings_service.py index 5fc20ce7..3b6fc85a 100644 --- a/app/utils/settings_service.py +++ b/app/utils/settings_service.py @@ -13,7 +13,7 @@ from typing import Any, Dict, List, Optional, Tuple from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session -from app.models import ApplicationSettings +from app.models import ApplicationSettings, SettingsAuditLog logger = logging.getLogger(__name__) @@ -871,7 +871,9 @@ def get_setting_from_db(db: Session, key: str) -> Optional[str]: Setting value as string (decrypted if necessary), or None if not found """ try: - setting = db.query(ApplicationSettings).filter(ApplicationSettings.key == key).first() + setting = ( + db.query(ApplicationSettings).filter(ApplicationSettings.key == key).first() + ) if not setting: return None @@ -888,16 +890,20 @@ def get_setting_from_db(db: Session, key: str) -> Optional[str]: return None -def save_setting_to_db(db: Session, key: str, value: Optional[str]) -> bool: +def save_setting_to_db( + db: Session, key: str, value: Optional[str], changed_by: str = "system" +) -> bool: """ Save or update a setting in the database. Automatically encrypts sensitive values if encryption is enabled. + Records an entry in the settings audit log. Args: db: Database session key: Setting key value: Setting value (as string) + changed_by: Username of the admin performing the change (for audit log) Returns: True if successful, False otherwise @@ -908,22 +914,53 @@ def save_setting_to_db(db: Session, key: str, value: Optional[str]) -> bool: storage_value = value if metadata.get("sensitive", False) and value: - from app.utils.encryption import encrypt_value, is_encryption_available + from app.utils.encryption import (encrypt_value, + is_encryption_available) if is_encryption_available(): storage_value = encrypt_value(value) logger.debug(f"Encrypted sensitive setting: {key}") else: - logger.warning(f"Storing sensitive setting {key} in plaintext (encryption unavailable)") + logger.warning( + f"Storing sensitive setting {key} in plaintext (encryption unavailable)" + ) + + setting = ( + db.query(ApplicationSettings).filter(ApplicationSettings.key == key).first() + ) + old_storage_value = setting.value if setting else None - setting = db.query(ApplicationSettings).filter(ApplicationSettings.key == key).first() if setting: setting.value = storage_value else: setting = ApplicationSettings(key=key, value=storage_value) db.add(setting) + + # Determine human-readable old value for audit log (decrypt if needed) + old_display_value = None + if old_storage_value is not None: + if metadata.get("sensitive", False): + try: + from app.utils.encryption import decrypt_value + + old_display_value = decrypt_value(old_storage_value) + except Exception: + old_display_value = old_storage_value + else: + old_display_value = old_storage_value + + # Write audit log entry + audit_entry = SettingsAuditLog( + key=key, + old_value=old_display_value, + new_value=value, + changed_by=changed_by, + action="update", + ) + db.add(audit_entry) + db.commit() - logger.info(f"Saved setting {key} to database") + logger.info(f"Saved setting {key} to database (changed_by={changed_by})") return True except SQLAlchemyError as e: logger.error(f"Error saving setting {key} to database: {e}") @@ -963,23 +1000,54 @@ def get_all_settings_from_db(db: Session) -> Dict[str, str]: return {} -def delete_setting_from_db(db: Session, key: str) -> bool: +def delete_setting_from_db(db: Session, key: str, changed_by: str = "system") -> bool: """ Delete a setting from the database. + Records an entry in the settings audit log. + Args: db: Database session key: Setting key to delete + changed_by: Username of the admin performing the change (for audit log) Returns: True if successful, False otherwise """ try: - setting = db.query(ApplicationSettings).filter(ApplicationSettings.key == key).first() + setting = ( + db.query(ApplicationSettings).filter(ApplicationSettings.key == key).first() + ) if setting: + # Capture old value for audit log (decrypt if sensitive) + metadata = get_setting_metadata(key) + old_display_value = None + if setting.value is not None: + if metadata.get("sensitive", False): + try: + from app.utils.encryption import decrypt_value + + old_display_value = decrypt_value(setting.value) + except Exception: + old_display_value = setting.value + else: + old_display_value = setting.value + db.delete(setting) + + audit_entry = SettingsAuditLog( + key=key, + old_value=old_display_value, + new_value=None, + changed_by=changed_by, + action="delete", + ) + db.add(audit_entry) + db.commit() - logger.info(f"Deleted setting {key} from database") + logger.info( + f"Deleted setting {key} from database (changed_by={changed_by})" + ) return True return False except SQLAlchemyError as e: @@ -1061,3 +1129,161 @@ def validate_setting_value(key: str, value: str) -> Tuple[bool, Optional[str]]: return False, "session_secret must be at least 32 characters" return True, None + + +def get_audit_log( + db: Session, limit: int = 100, offset: int = 0 +) -> List[Dict[str, Any]]: + """ + Retrieve the settings audit log, most recent first. + + Sensitive values are masked in the returned list so the log is safe to + display in the admin UI without leaking secrets. + + Args: + db: Database session + limit: Maximum number of entries to return + offset: Number of entries to skip (for pagination) + + Returns: + List of audit log entry dicts ordered by changed_at descending + """ + try: + entries = ( + db.query(SettingsAuditLog) + .order_by(SettingsAuditLog.changed_at.desc()) + .limit(limit) + .offset(offset) + .all() + ) + result = [] + for entry in entries: + meta = get_setting_metadata(entry.key) + is_sensitive = meta.get("sensitive", False) + result.append( + { + "id": entry.id, + "key": entry.key, + "old_value": ( + "[REDACTED]" + if is_sensitive and entry.old_value + else entry.old_value + ), + "new_value": ( + "[REDACTED]" + if is_sensitive and entry.new_value + else entry.new_value + ), + "changed_by": entry.changed_by, + "changed_at": ( + entry.changed_at.isoformat() if entry.changed_at else None + ), + "action": entry.action, + } + ) + return result + except SQLAlchemyError as e: + logger.error(f"Error retrieving audit log: {e}") + return [] + + +def get_setting_history(db: Session, key: str) -> List[Dict[str, Any]]: + """ + Retrieve the change history for a specific setting key, most recent first. + + Sensitive values are masked so the response is safe to surface in the UI. + + Args: + db: Database session + key: Setting key + + Returns: + List of audit log entry dicts for this key + """ + try: + entries = ( + db.query(SettingsAuditLog) + .filter(SettingsAuditLog.key == key) + .order_by(SettingsAuditLog.changed_at.desc()) + .all() + ) + meta = get_setting_metadata(key) + is_sensitive = meta.get("sensitive", False) + result = [] + for entry in entries: + result.append( + { + "id": entry.id, + "key": entry.key, + "old_value": ( + "[REDACTED]" + if is_sensitive and entry.old_value + else entry.old_value + ), + "new_value": ( + "[REDACTED]" + if is_sensitive and entry.new_value + else entry.new_value + ), + "changed_by": entry.changed_by, + "changed_at": ( + entry.changed_at.isoformat() if entry.changed_at else None + ), + "action": entry.action, + } + ) + return result + except SQLAlchemyError as e: + logger.error(f"Error retrieving history for setting {key}: {e}") + return [] + + +def rollback_setting( + db: Session, key: str, history_id: int, changed_by: str = "system" +) -> bool: + """ + Revert a setting to the value recorded in a specific audit log entry. + + The value stored in the chosen history entry's ``new_value`` field is + re-applied as the current database value. If that value is ``None`` + (i.e. the entry recorded a deletion) the setting is removed from the + database entirely, reverting to ENV/defaults. + + A new audit log entry is written to record the rollback operation. + + Args: + db: Database session + key: Setting key to roll back + history_id: ID of the SettingsAuditLog entry whose ``new_value`` + should become the restored value + changed_by: Username performing the rollback (for audit log) + + Returns: + True if successful, False if the history entry was not found or an + error occurred + """ + try: + history_entry = ( + db.query(SettingsAuditLog) + .filter(SettingsAuditLog.id == history_id, SettingsAuditLog.key == key) + .first() + ) + if not history_entry: + logger.warning( + f"Rollback failed: audit log entry {history_id} not found for key '{key}'" + ) + return False + + target_value = history_entry.new_value + + if target_value is None: + # The history entry recorded a deletion – reinstate that by deleting the current db value + return delete_setting_from_db(db, key, changed_by=changed_by) + else: + return save_setting_to_db(db, key, target_value, changed_by=changed_by) + except SQLAlchemyError as e: + logger.error( + f"Error rolling back setting {key} to history entry {history_id}: {e}" + ) + db.rollback() + return False diff --git a/app/utils/settings_sync.py b/app/utils/settings_sync.py new file mode 100644 index 00000000..8ffd03d1 --- /dev/null +++ b/app/utils/settings_sync.py @@ -0,0 +1,86 @@ +""" +Worker settings synchronisation helper. + +When an admin saves a configuration change through the UI, any running Celery +workers still hold the *old* values in their in-process ``settings`` singleton. +This module provides two complementary mechanisms to propagate the change: + +1. **Publish** (API side): :func:`notify_settings_updated` writes a monotonically + increasing timestamp to a Redis key. This is called immediately after every + successful ``save_setting_to_db`` / ``delete_setting_from_db`` operation. + +2. **Subscribe** (worker side): :func:`register_settings_reload_signal` installs + a Celery ``task_prerun`` signal handler. Before each task begins the handler + reads the Redis version key; if it has changed since the last reload it calls + :func:`~app.utils.config_loader.reload_settings_from_db` so the worker picks + up the new values *before* executing the task body. + +The Redis key used is ``docuelevate:settings_version``. Workers cache the last +seen version in a module-level variable to avoid redundant DB round-trips when +nothing has changed. +""" + +import logging +import time + +import redis +from celery.signals import task_prerun + +logger = logging.getLogger(__name__) + +#: Redis key that stores the current settings "version" (epoch timestamp string). +SETTINGS_VERSION_KEY = "docuelevate:settings_version" + +#: Module-level cache: the settings version seen by *this* process on its last reload. +_last_seen_version: str = "" + + +def notify_settings_updated() -> None: + """ + Publish a settings-updated signal by updating the Redis version key. + + Call this after every successful settings write so that all worker + processes know they need to reload their in-memory configuration. + + Errors are caught and logged rather than raised so that a Redis + connectivity issue does not prevent the primary save from succeeding. + """ + try: + from app.config import settings + + r = redis.from_url(settings.redis_url, socket_connect_timeout=2) + version = str(time.time()) + r.set(SETTINGS_VERSION_KEY, version) + logger.debug(f"Settings version bumped to {version}") + except Exception as exc: + logger.warning(f"Could not publish settings update to Redis: {exc}") + + +def register_settings_reload_signal() -> None: + """ + Install a Celery ``task_prerun`` signal handler for worker processes. + + This should be called once during Celery worker initialisation (e.g. from + ``celery_worker.py``). After registration, every task will check the + settings version key in Redis before it starts and reload configuration + from the database if a newer version is detected. + """ + + @task_prerun.connect(weak=False) + def _reload_if_stale(sender, **kwargs) -> None: # type: ignore[misc] + """Reload settings from DB if the Redis version key has changed.""" + global _last_seen_version + try: + from app.config import settings + from app.utils.config_loader import reload_settings_from_db + + r = redis.from_url(settings.redis_url, socket_connect_timeout=2) + current_version = (r.get(SETTINGS_VERSION_KEY) or b"").decode() + if current_version and current_version != _last_seen_version: + reload_settings_from_db(settings) + _last_seen_version = current_version + logger.info(f"Worker settings reloaded (version={current_version})") + except Exception as exc: + logger.debug(f"Settings version check skipped: {exc}") + + logger.info("Settings reload signal handler registered on task_prerun") diff --git a/app/views/settings.py b/app/views/settings.py index 7379d874..3b40a94d 100644 --- a/app/views/settings.py +++ b/app/views/settings.py @@ -12,13 +12,12 @@ from fastapi.responses import RedirectResponse from sqlalchemy.orm import Session from app.utils.config_validator.masking import mask_sensitive_value -from app.utils.settings_service import ( - SETTING_METADATA, - get_all_settings_from_db, - get_setting_metadata, - get_settings_by_category, -) -from app.views.base import APIRouter, get_db, require_login, settings, templates +from app.utils.settings_service import (SETTING_METADATA, + get_all_settings_from_db, + get_setting_metadata, + get_settings_by_category) +from app.views.base import (APIRouter, get_db, require_login, settings, + templates) logger = logging.getLogger(__name__) router = APIRouter() @@ -103,7 +102,9 @@ async def settings_page(request: Request, db: Session = Depends(get_db)): settings_data[category].append( { "key": key, - "display_value": display_value if display_value is not None else "", + "display_value": ( + display_value if display_value is not None else "" + ), "metadata": metadata, "source": source, "source_label": source_label, @@ -112,11 +113,19 @@ async def settings_page(request: Request, db: Session = Depends(get_db)): ) return templates.TemplateResponse( - "settings.html", {"request": request, "settings_data": settings_data, "app_version": settings.version} + "settings.html", + { + "request": request, + "settings_data": settings_data, + "app_version": settings.version, + }, ) except Exception as e: logger.error(f"Error loading settings page: {e}") - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to load settings page") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to load settings page", + ) @router.get("/admin/credentials") @@ -166,7 +175,9 @@ async def credentials_page(request: Request, db: Session = Depends(get_db)): ) total = sum(len(v) for v in categories.values()) - configured_count = sum(1 for creds in categories.values() for c in creds if c["configured"]) + configured_count = sum( + 1 for creds in categories.values() for c in creds if c["configured"] + ) return templates.TemplateResponse( "credentials.html", @@ -181,4 +192,39 @@ async def credentials_page(request: Request, db: Session = Depends(get_db)): ) except Exception as e: logger.error(f"Error loading credentials page: {e}") - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to load credentials page") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to load credentials page", + ) + + +@router.get("/admin/settings/audit-log") +@require_login +@require_admin_access +async def audit_log_page(request: Request, db: Session = Depends(get_db)): + """ + Settings audit log page - admin only. + + Displays a chronological log of all configuration changes made via the + settings UI, including who made the change and what the old/new values + were. Sensitive values are masked. Provides rollback buttons to revert + any setting to a previous value. + """ + from app.utils.settings_service import get_audit_log + + try: + entries = get_audit_log(db, limit=200) + return templates.TemplateResponse( + "audit_log.html", + { + "request": request, + "entries": entries, + "app_version": settings.version, + }, + ) + except Exception as e: + logger.error(f"Error loading audit log page: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to load audit log page", + ) diff --git a/frontend/templates/audit_log.html b/frontend/templates/audit_log.html new file mode 100644 index 00000000..fa16bac6 --- /dev/null +++ b/frontend/templates/audit_log.html @@ -0,0 +1,157 @@ +{% extends "base.html" %} +{% block title %}Settings Audit Log - DocuElevate{% endblock %} + +{% block content %} +
+ +
+
+

Settings Audit Log

+

+ Chronological record of all configuration changes made via the settings UI. + Sensitive values are masked. Use the rollback button to revert any setting to a prior value. +

+
+ + Back to Settings + +
+ + +
+ +
+ + {% if entries %} +
+ + + + + + + + + + + + + + {% for entry in entries %} + + + + + + + + + + {% endfor %} + +
WhenChanged BySetting KeyActionOld ValueNew ValueRollback
{{ entry.changed_at }}{{ entry.changed_by }}{{ entry.key }} + {% if entry.action == 'delete' %} + delete + {% elif entry.action == 'rollback' %} + rollback + {% else %} + update + {% endif %} + + {% if entry.old_value %} + {{ entry.old_value }} + {% else %} + + {% endif %} + + {% if entry.new_value %} + {{ entry.new_value }} + {% else %} + — (deleted) + {% endif %} + + +
+
+ {% else %} +
+ +

No configuration changes recorded yet.

+

Changes you make on the Settings page will appear here.

+
+ {% endif %} +
+ + +{% endblock %} diff --git a/frontend/templates/settings.html b/frontend/templates/settings.html index 61898dab..495193dd 100644 --- a/frontend/templates/settings.html +++ b/frontend/templates/settings.html @@ -13,10 +13,18 @@
-

Application Settings

-

- This is a convenience feature to view and edit application settings through the web interface. -

+
+
+

Application Settings

+

+ This is a convenience feature to view and edit application settings through the web interface. +

+
+ + Audit Log + +
{% endif %}
+ + +
+ +
{% endfor %} @@ -185,6 +209,7 @@ function settingsApp() { originalData: {}, showPassword: {}, saving: false, + savingKey: null, showAlert: false, alertType: 'success', alertTitle: '', @@ -230,6 +255,38 @@ function settingsApp() { this.showAlert = false; }, + async saveSetting(key) { + this.savingKey = key; + this.hideAlert(); + + try { + const value = this.formData[key]; + const response = await fetch(`/api/settings/${key}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ key, value }), + }); + + const result = await response.json(); + + if (response.ok && result.success) { + this.originalData[key] = value; + let message = `Setting '${key}' saved successfully.`; + if (result.restart_required) { + message += ' Please restart the application for this change to take effect.'; + } + this.showSuccessAlert('Setting Saved', message); + } else { + this.showErrorAlert('Save Failed', result.detail || 'Unknown error'); + } + } catch (error) { + console.error('Error saving setting:', error); + this.showErrorAlert('Error', 'Failed to save setting. Please try again.'); + } finally { + this.savingKey = null; + } + }, + async saveSettings() { this.saving = true; this.hideAlert(); diff --git a/tests/test_settings_audit_log.py b/tests/test_settings_audit_log.py new file mode 100644 index 00000000..a28df696 --- /dev/null +++ b/tests/test_settings_audit_log.py @@ -0,0 +1,452 @@ +"""Tests for the settings audit log, rollback, per-option save, and worker sync features.""" + +import asyncio +from unittest.mock import MagicMock, patch + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from app.database import Base +from app.models import SettingsAuditLog + +# --------------------------------------------------------------------------- +# Shared DB fixture +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def db_session(): + engine = create_engine( + "sqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(bind=engine) + Session = sessionmaker(autocommit=False, autoflush=False, bind=engine) + session = Session() + try: + yield session + finally: + session.close() + Base.metadata.drop_all(bind=engine) + + +# =========================================================================== +# A) Audit log written on save +# =========================================================================== + + +@pytest.mark.unit +class TestAuditLogOnSave: + """Audit log entries are created when settings are saved or deleted.""" + + def test_save_creates_audit_entry(self, db_session): + from app.utils.settings_service import save_setting_to_db + + result = save_setting_to_db( + db_session, "workdir", "/new/path", changed_by="alice" + ) + + assert result is True + entry = db_session.query(SettingsAuditLog).filter_by(key="workdir").first() + assert entry is not None + assert entry.action == "update" + assert entry.new_value == "/new/path" + assert entry.changed_by == "alice" + assert entry.old_value is None # was not previously set + + def test_update_records_old_value(self, db_session): + from app.utils.settings_service import save_setting_to_db + + # Set initial value + save_setting_to_db(db_session, "workdir", "/old/path", changed_by="admin") + # Update + save_setting_to_db(db_session, "workdir", "/new/path", changed_by="bob") + + entries = db_session.query(SettingsAuditLog).filter_by(key="workdir").all() + assert len(entries) == 2 + # Second entry should have old_value from first write + update_entry = entries[1] + assert update_entry.old_value == "/old/path" + assert update_entry.new_value == "/new/path" + + def test_delete_creates_audit_entry(self, db_session): + from app.utils.settings_service import (delete_setting_from_db, + save_setting_to_db) + + save_setting_to_db(db_session, "workdir", "/some/path", changed_by="admin") + result = delete_setting_from_db(db_session, "workdir", changed_by="carol") + + assert result is True + delete_entry = ( + db_session.query(SettingsAuditLog) + .filter_by(key="workdir", action="delete") + .first() + ) + assert delete_entry is not None + assert delete_entry.old_value == "/some/path" + assert delete_entry.new_value is None + assert delete_entry.changed_by == "carol" + + def test_delete_nonexistent_returns_false_no_entry(self, db_session): + from app.utils.settings_service import delete_setting_from_db + + result = delete_setting_from_db( + db_session, "nonexistent_key", changed_by="admin" + ) + + assert result is False + assert db_session.query(SettingsAuditLog).count() == 0 + + def test_default_changed_by_is_system(self, db_session): + from app.utils.settings_service import save_setting_to_db + + save_setting_to_db(db_session, "workdir", "/tmp") + + entry = db_session.query(SettingsAuditLog).first() + assert entry.changed_by == "system" + + +# =========================================================================== +# C) Audit log retrieval +# =========================================================================== + + +@pytest.mark.unit +class TestGetAuditLog: + """get_audit_log returns entries, masks sensitive values.""" + + def test_returns_all_entries_most_recent_first(self, db_session): + from app.utils.settings_service import (get_audit_log, + save_setting_to_db) + + save_setting_to_db(db_session, "workdir", "/first", changed_by="u1") + save_setting_to_db(db_session, "workdir", "/second", changed_by="u2") + + log = get_audit_log(db_session, limit=100) + + assert len(log) == 2 + # Most recent first + assert log[0]["new_value"] == "/second" + assert log[1]["new_value"] == "/first" + + def test_sensitive_values_are_masked(self, db_session): + from app.utils.settings_service import (get_audit_log, + save_setting_to_db) + + save_setting_to_db( + db_session, "openai_api_key", "sk-secret123", changed_by="admin" + ) + + log = get_audit_log(db_session) + + entry = next(e for e in log if e["key"] == "openai_api_key") + assert entry["new_value"] == "[REDACTED]" + + def test_required_fields_present(self, db_session): + from app.utils.settings_service import (get_audit_log, + save_setting_to_db) + + save_setting_to_db(db_session, "workdir", "/path", changed_by="alice") + + log = get_audit_log(db_session) + + assert len(log) == 1 + entry = log[0] + for field in ( + "id", + "key", + "old_value", + "new_value", + "changed_by", + "changed_at", + "action", + ): + assert field in entry + + def test_limit_and_offset(self, db_session): + from app.utils.settings_service import (get_audit_log, + save_setting_to_db) + + for i in range(5): + save_setting_to_db(db_session, "workdir", f"/path{i}", changed_by="admin") + + first_page = get_audit_log(db_session, limit=3, offset=0) + second_page = get_audit_log(db_session, limit=3, offset=3) + + assert len(first_page) == 3 + assert len(second_page) == 2 + + +# =========================================================================== +# C) Per-key history +# =========================================================================== + + +@pytest.mark.unit +class TestGetSettingHistory: + """get_setting_history returns only entries for the requested key.""" + + def test_returns_only_matching_key(self, db_session): + from app.utils.settings_service import (get_setting_history, + save_setting_to_db) + + save_setting_to_db(db_session, "workdir", "/wdir", changed_by="admin") + save_setting_to_db(db_session, "debug", "true", changed_by="admin") + + history = get_setting_history(db_session, "workdir") + + assert len(history) == 1 + assert history[0]["key"] == "workdir" + + def test_returns_empty_list_for_unknown_key(self, db_session): + from app.utils.settings_service import get_setting_history + + history = get_setting_history(db_session, "totally_unknown_key") + + assert history == [] + + +# =========================================================================== +# D) Rollback +# =========================================================================== + + +@pytest.mark.unit +class TestRollbackSetting: + """rollback_setting reinstates the value from a given audit log entry.""" + + def test_rollback_to_previous_value(self, db_session): + from app.utils.settings_service import (get_setting_from_db, + rollback_setting, + save_setting_to_db) + + save_setting_to_db( + db_session, "workdir", "/v1", changed_by="admin" + ) # entry id 1 + save_setting_to_db( + db_session, "workdir", "/v2", changed_by="admin" + ) # entry id 2 + + first_entry = ( + db_session.query(SettingsAuditLog).filter_by(key="workdir").first() + ) + # first entry has new_value="/v1" + success = rollback_setting( + db_session, "workdir", first_entry.id, changed_by="rollbacker" + ) + + assert success is True + current = get_setting_from_db(db_session, "workdir") + assert current == "/v1" + + def test_rollback_creates_new_audit_entry(self, db_session): + from app.utils.settings_service import (rollback_setting, + save_setting_to_db) + + save_setting_to_db(db_session, "workdir", "/v1", changed_by="admin") + entry = db_session.query(SettingsAuditLog).filter_by(key="workdir").first() + + initial_count = db_session.query(SettingsAuditLog).count() + rollback_setting(db_session, "workdir", entry.id, changed_by="rollbacker") + + assert db_session.query(SettingsAuditLog).count() == initial_count + 1 + + def test_rollback_wrong_history_id_returns_false(self, db_session): + from app.utils.settings_service import (rollback_setting, + save_setting_to_db) + + save_setting_to_db(db_session, "workdir", "/v1", changed_by="admin") + + result = rollback_setting(db_session, "workdir", 9999, changed_by="admin") + + assert result is False + + def test_rollback_wrong_key_returns_false(self, db_session): + from app.utils.settings_service import (rollback_setting, + save_setting_to_db) + + save_setting_to_db(db_session, "workdir", "/v1", changed_by="admin") + entry = db_session.query(SettingsAuditLog).filter_by(key="workdir").first() + + # Pass wrong key for the history ID + result = rollback_setting(db_session, "debug", entry.id, changed_by="admin") + + assert result is False + + +# =========================================================================== +# B) Worker sync – settings_sync module +# =========================================================================== + + +@pytest.mark.unit +class TestNotifySettingsUpdated: + """notify_settings_updated publishes the settings version key to Redis.""" + + def test_sets_redis_key(self): + from app.utils.settings_sync import (SETTINGS_VERSION_KEY, + notify_settings_updated) + + mock_redis = MagicMock() + mock_redis_instance = MagicMock() + mock_redis.return_value = mock_redis_instance + + with patch("app.utils.settings_sync.redis") as mock_redis_module: + mock_redis_module.from_url.return_value = mock_redis_instance + notify_settings_updated() + + mock_redis_instance.set.assert_called_once() + call_args = mock_redis_instance.set.call_args[0] + assert call_args[0] == SETTINGS_VERSION_KEY + + def test_does_not_raise_on_redis_failure(self): + """notify_settings_updated must not propagate Redis errors.""" + from app.utils.settings_sync import notify_settings_updated + + with patch("app.utils.settings_sync.redis") as mock_redis_module: + mock_redis_module.from_url.side_effect = Exception("Redis down") + # Should not raise + notify_settings_updated() + + +@pytest.mark.unit +class TestRegisterSettingsReloadSignal: + """register_settings_reload_signal installs a task_prerun handler.""" + + def test_registers_without_error(self): + from app.utils.settings_sync import register_settings_reload_signal + + with patch("app.utils.settings_sync.task_prerun") as mock_signal: + mock_signal.connect = MagicMock() + # Call it – the decorator calls task_prerun.connect(weak=False) + register_settings_reload_signal() + # If no exception is raised the registration succeeded + + +# =========================================================================== +# API endpoint – audit log +# =========================================================================== + + +@pytest.mark.integration +class TestAuditLogEndpoint: + """GET /api/settings/audit-log requires admin access.""" + + def test_requires_admin(self, client): + response = client.get("/api/settings/audit-log") + assert response.status_code in [302, 401, 403] + + @patch("app.api.settings.get_audit_log") + def test_returns_entries_for_admin(self, mock_get_log): + from app.api.settings import list_audit_log + + mock_get_log.return_value = [ + { + "id": 1, + "key": "workdir", + "old_value": None, + "new_value": "/tmp", + "changed_by": "admin", + "changed_at": "2024-01-01T00:00:00", + "action": "update", + } + ] + + mock_request = MagicMock() + mock_db = MagicMock() + mock_admin = {"is_admin": True} + + result = asyncio.run(list_audit_log(mock_request, mock_db, mock_admin)) + + assert "entries" in result + assert len(result["entries"]) == 1 + assert result["entries"][0]["key"] == "workdir" + + +@pytest.mark.integration +class TestHistoryEndpoint: + """GET /api/settings/{key}/history requires admin access.""" + + def test_requires_admin(self, client): + response = client.get("/api/settings/workdir/history") + assert response.status_code in [302, 401, 403] + + @patch("app.api.settings.get_setting_history") + def test_returns_history_for_admin(self, mock_get_history): + from app.api.settings import get_key_history + + mock_get_history.return_value = [ + { + "id": 1, + "key": "workdir", + "old_value": None, + "new_value": "/tmp", + "changed_by": "admin", + "changed_at": "2024-01-01T00:00:00", + "action": "update", + } + ] + + mock_request = MagicMock() + mock_db = MagicMock() + mock_admin = {"is_admin": True} + + result = asyncio.run( + get_key_history("workdir", mock_request, mock_db, mock_admin) + ) + + assert result["key"] == "workdir" + assert len(result["history"]) == 1 + + +@pytest.mark.integration +class TestRollbackEndpoint: + """POST /api/settings/{key}/rollback/{history_id} requires admin access.""" + + def test_requires_admin(self, client): + response = client.post("/api/settings/workdir/rollback/1") + assert response.status_code in [302, 401, 403] + + @patch("app.api.settings.notify_settings_updated") + @patch("app.api.settings.rollback_setting") + def test_rollback_success(self, mock_rollback, mock_notify): + from app.api.settings import rollback_setting_to_history + + mock_rollback.return_value = True + mock_request = MagicMock() + mock_request.session = {"user": {"preferred_username": "admin"}} + mock_db = MagicMock() + mock_admin = {"is_admin": True} + + result = asyncio.run( + rollback_setting_to_history("workdir", 1, mock_request, mock_db, mock_admin) + ) + + assert result["success"] is True + mock_notify.assert_called_once() + + @patch("app.api.settings.rollback_setting") + def test_rollback_not_found_raises_404(self, mock_rollback): + import asyncio + + from fastapi import HTTPException + + from app.api.settings import rollback_setting_to_history + + mock_rollback.return_value = False + mock_request = MagicMock() + mock_request.session = {"user": {"preferred_username": "admin"}} + mock_db = MagicMock() + mock_admin = {"is_admin": True} + + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + rollback_setting_to_history( + "workdir", 9999, mock_request, mock_db, mock_admin + ) + ) + + assert exc_info.value.status_code == 404 From 0d5c20f6856e973b6759a1ca0a60ddba7df92a21 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 23 Feb 2026 02:30:56 +0000 Subject: [PATCH 3/8] chore: outline plan for wizard DB persistence, worker sync, and ENV exporter Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- app/api/dropbox.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/app/api/dropbox.py b/app/api/dropbox.py index ea7600aa..826a06bd 100644 --- a/app/api/dropbox.py +++ b/app/api/dropbox.py @@ -7,11 +7,15 @@ import os from typing import Annotated, Optional import requests -from fastapi import APIRouter, Form, HTTPException, Request, status +from fastapi import APIRouter, Depends, Form, HTTPException, Request, status +from sqlalchemy.orm import Session from app.auth import require_login from app.config import settings +from app.database import get_db from app.utils.oauth_helper import exchange_oauth_token +from app.utils.settings_service import save_setting_to_db +from app.utils.settings_sync import notify_settings_updated # Set up logging logger = logging.getLogger(__name__) @@ -63,33 +67,43 @@ async def update_dropbox_settings( app_key: Annotated[Optional[str], Form()] = None, app_secret: Annotated[Optional[str], Form()] = None, folder_path: Annotated[Optional[str], Form()] = None, + db: Session = Depends(get_db), ): """ - Update Dropbox settings in memory + Update Dropbox settings in memory and persist to the database. """ try: - logger.info("Updating Dropbox settings in memory") + logger.info("Updating Dropbox settings in memory and database") - # Update settings in memory + user = request.session.get("user", {}) if hasattr(request, "session") else {} + changed_by = ( + user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard" + ) + + # Update settings in memory and persist to database if refresh_token: settings.dropbox_refresh_token = refresh_token - logger.info("Updated DROPBOX_REFRESH_TOKEN in memory") + save_setting_to_db(db, "dropbox_refresh_token", refresh_token, changed_by=changed_by) + logger.info("Updated DROPBOX_REFRESH_TOKEN in memory and database") if app_key: settings.dropbox_app_key = app_key - logger.info("Updated DROPBOX_APP_KEY in memory") + save_setting_to_db(db, "dropbox_app_key", app_key, changed_by=changed_by) + logger.info("Updated DROPBOX_APP_KEY in memory and database") if app_secret: settings.dropbox_app_secret = app_secret - logger.info("Updated DROPBOX_APP_SECRET in memory") + save_setting_to_db(db, "dropbox_app_secret", app_secret, changed_by=changed_by) + logger.info("Updated DROPBOX_APP_SECRET in memory and database") if folder_path: settings.dropbox_folder = folder_path - logger.info("Updated DROPBOX_FOLDER in memory") + save_setting_to_db(db, "dropbox_folder", folder_path, changed_by=changed_by) + logger.info("Updated DROPBOX_FOLDER in memory and database") - # Test token validity would be here, but we'll skip it for now + notify_settings_updated() - return {"status": "success", "message": "Dropbox settings have been updated in memory"} + return {"status": "success", "message": "Dropbox settings have been updated in memory and saved to database"} except Exception as e: logger.exception(f"Unexpected error updating Dropbox settings: {str(e)}") From ecc8100e24517fee7efc01bf41cc50ebfb873a9d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 23 Feb 2026 02:41:04 +0000 Subject: [PATCH 4/8] feat(settings): persist storage provider settings to DB, add export endpoint, enrich setup wizard - dropbox/google_drive/onedrive save-settings: switch to DB as primary, .env write as best-effort (no longer fails when .env is absent) - onedrive/google_drive update-settings: persist changes to DB alongside in-memory update; call notify_settings_updated() - onedrive test_onedrive_token: persist rotated refresh token to DB - settings_service: add get_settings_for_export() (db / effective modes) - settings API: add GET /api/settings/export-env (admin-only, downloads .env) - wizard: enrich settings with current values (DB > ENV > default) and value_source badges; pass setup_skipped to template; call notify_settings_updated() on save; add /setup/undo-skip route - setup_wizard.html: pre-populate inputs with current_value; show DB/ENV/DEFAULT source badges; skip/undo-skip messaging - settings.html: replace single Audit Log button with Setup Wizard link, Export .env dropdown, and Audit Log button group Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/api/dropbox.py | 197 +++++++++++++------- app/api/google_drive.py | 149 ++++++++++++--- app/api/onedrive.py | 265 +++++++++++++++++++-------- app/api/settings.py | 58 ++++++ app/utils/settings_service.py | 35 ++++ app/views/wizard.py | 70 ++++++- frontend/templates/settings.html | 34 +++- frontend/templates/setup_wizard.html | 23 ++- 8 files changed, 651 insertions(+), 180 deletions(-) diff --git a/app/api/dropbox.py b/app/api/dropbox.py index 826a06bd..57e32d09 100644 --- a/app/api/dropbox.py +++ b/app/api/dropbox.py @@ -49,7 +49,9 @@ async def exchange_dropbox_token( } # Use shared OAuth helper (handles secure logging and error handling) - token_data = exchange_oauth_token(provider_name="Dropbox", token_url=token_url, payload=payload) + token_data = exchange_oauth_token( + provider_name="Dropbox", token_url=token_url, payload=payload + ) # Return just what's needed by the frontend return { @@ -77,13 +79,19 @@ async def update_dropbox_settings( user = request.session.get("user", {}) if hasattr(request, "session") else {} changed_by = ( - user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard" + user.get("preferred_username") + or user.get("username") + or user.get("email") + or user.get("id") + or "wizard" ) # Update settings in memory and persist to database if refresh_token: settings.dropbox_refresh_token = refresh_token - save_setting_to_db(db, "dropbox_refresh_token", refresh_token, changed_by=changed_by) + save_setting_to_db( + db, "dropbox_refresh_token", refresh_token, changed_by=changed_by + ) logger.info("Updated DROPBOX_REFRESH_TOKEN in memory and database") if app_key: @@ -93,7 +101,9 @@ async def update_dropbox_settings( if app_secret: settings.dropbox_app_secret = app_secret - save_setting_to_db(db, "dropbox_app_secret", app_secret, changed_by=changed_by) + save_setting_to_db( + db, "dropbox_app_secret", app_secret, changed_by=changed_by + ) logger.info("Updated DROPBOX_APP_SECRET in memory and database") if folder_path: @@ -103,12 +113,16 @@ async def update_dropbox_settings( notify_settings_updated() - return {"status": "success", "message": "Dropbox settings have been updated in memory and saved to database"} + return { + "status": "success", + "message": "Dropbox settings have been updated in memory and saved to database", + } except Exception as e: logger.exception(f"Unexpected error updating Dropbox settings: {str(e)}") raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to update Dropbox settings: {str(e)}" + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to update Dropbox settings: {str(e)}", ) @@ -121,9 +135,16 @@ async def test_dropbox_token(request: Request): try: logger.info("Testing Dropbox token validity") - if not settings.dropbox_refresh_token or not settings.dropbox_app_key or not settings.dropbox_app_secret: + if ( + not settings.dropbox_refresh_token + or not settings.dropbox_app_key + or not settings.dropbox_app_secret + ): logger.warning("Dropbox credentials not fully configured") - return {"status": "error", "message": "Dropbox credentials are not fully configured"} + return { + "status": "error", + "message": "Dropbox credentials are not fully configured", + } # Check token validity by getting current account info headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"} @@ -146,11 +167,19 @@ async def test_dropbox_token(request: Request): "client_secret": settings.dropbox_app_secret, } - refresh_response = requests.post(refresh_url, data=refresh_data, timeout=settings.http_request_timeout) + refresh_response = requests.post( + refresh_url, data=refresh_data, timeout=settings.http_request_timeout + ) if refresh_response.status_code != 200: - logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}") - return {"status": "error", "message": "Refresh token has expired or is invalid", "needs_reauth": True} + logger.error( + f"Failed to refresh Dropbox token: {refresh_response.text}" + ) + return { + "status": "error", + "message": "Refresh token has expired or is invalid", + "needs_reauth": True, + } token_info = refresh_response.json() access_token = token_info.get("access_token") @@ -164,7 +193,9 @@ async def test_dropbox_token(request: Request): ) if response.status_code != 200: - logger.error(f"Dropbox token test failed: {response.status_code} {response.text}") + logger.error( + f"Dropbox token test failed: {response.status_code} {response.text}" + ) return { "status": "error", "message": f"Token validation failed with status {response.status_code}: {response.text}", @@ -176,7 +207,10 @@ async def test_dropbox_token(request: Request): account_name = account_info.get("name", {}).get("display_name", "Unknown user") # Dropbox refresh tokens don't expire, but we should note that in our response - token_info = {"expires_in_human": "Never expires (perpetual token)", "is_perpetual": True} + token_info = { + "expires_in_human": "Never expires (perpetual token)", + "is_perpetual": True, + } logger.info(f"Successfully connected to Dropbox as {account_email}") @@ -201,65 +235,22 @@ async def save_dropbox_settings( app_key: Annotated[Optional[str], Form()] = None, app_secret: Annotated[Optional[str], Form()] = None, folder_path: Annotated[Optional[str], Form()] = None, + db: Session = Depends(get_db), ): """ - Save Dropbox settings to the .env file + Save Dropbox settings to database (primary) and .env file (best-effort). """ try: - # Get the path to the .env file - env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env") + user = request.session.get("user", {}) if hasattr(request, "session") else {} + changed_by = ( + user.get("preferred_username") + or user.get("username") + or user.get("email") + or user.get("id") + or "wizard" + ) - if not os.path.exists(env_path): - logger.error(f".env file not found at {env_path}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Could not find .env file to update" - ) - - logger.info(f"Updating Dropbox settings in {env_path}") - - # Read the current .env file - with open(env_path, "r") as f: - env_lines = f.readlines() - - # Define settings to update - dropbox_settings = { - "DROPBOX_REFRESH_TOKEN": refresh_token, - } - - # Only update these if provided - if app_key: - dropbox_settings["DROPBOX_APP_KEY"] = app_key - if app_secret: - dropbox_settings["DROPBOX_APP_SECRET"] = app_secret - if folder_path: - dropbox_settings["DROPBOX_FOLDER"] = folder_path - - # Process each line and update or add settings - updated = set() - new_env_lines = [] - for line in env_lines: - stripped_line = line.rstrip() - is_updated = False - for key, value in dropbox_settings.items(): - if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="): - # Uncomment if commented out - check the original stripped line - new_env_lines.append(f"{key}={value}") - updated.add(key) - is_updated = True - break - if not is_updated: - new_env_lines.append(stripped_line) - - # Add any settings that weren't updated (they weren't in the file) - for key, value in dropbox_settings.items(): - if key not in updated: - new_env_lines.append(f"{key}={value}") - - # Write the updated .env file - with open(env_path, "w") as f: - f.write("\n".join(new_env_lines) + "\n") - - # Update the settings in memory + # Update settings in memory if refresh_token: settings.dropbox_refresh_token = refresh_token if app_key: @@ -269,14 +260,78 @@ async def save_dropbox_settings( if folder_path: settings.dropbox_folder = folder_path - logger.info("Successfully updated Dropbox settings") + # Persist to database (primary storage) + if refresh_token: + save_setting_to_db( + db, "dropbox_refresh_token", refresh_token, changed_by=changed_by + ) + if app_key: + save_setting_to_db(db, "dropbox_app_key", app_key, changed_by=changed_by) + if app_secret: + save_setting_to_db( + db, "dropbox_app_secret", app_secret, changed_by=changed_by + ) + if folder_path: + save_setting_to_db(db, "dropbox_folder", folder_path, changed_by=changed_by) + # Best-effort .env file write + try: + env_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env" + ) + if not os.path.exists(env_path): + logger.warning( + f".env file not found at {env_path}, skipping file write" + ) + else: + logger.info(f"Updating Dropbox settings in {env_path}") + + with open(env_path, "r") as f: + env_lines = f.readlines() + + dropbox_settings = {"DROPBOX_REFRESH_TOKEN": refresh_token} + if app_key: + dropbox_settings["DROPBOX_APP_KEY"] = app_key + if app_secret: + dropbox_settings["DROPBOX_APP_SECRET"] = app_secret + if folder_path: + dropbox_settings["DROPBOX_FOLDER"] = folder_path + + updated = set() + new_env_lines = [] + for line in env_lines: + stripped_line = line.rstrip() + is_updated = False + for key, value in dropbox_settings.items(): + if stripped_line.startswith( + f"{key}=" + ) or stripped_line.startswith(f"# {key}="): + new_env_lines.append(f"{key}={value}") + updated.add(key) + is_updated = True + break + if not is_updated: + new_env_lines.append(stripped_line) + + for key, value in dropbox_settings.items(): + if key not in updated: + new_env_lines.append(f"{key}={value}") + + with open(env_path, "w") as f: + f.write("\n".join(new_env_lines) + "\n") + + logger.info("Successfully updated Dropbox settings in .env file") + except Exception as env_err: + logger.warning(f"Failed to write .env file (non-fatal): {env_err}") + + notify_settings_updated() + + logger.info("Successfully saved Dropbox settings") return {"status": "success", "message": "Dropbox settings have been saved"} - except HTTPException: - raise except Exception as e: logger.exception(f"Unexpected error saving Dropbox settings: {str(e)}") raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to save Dropbox settings: {str(e)}" + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to save Dropbox settings: {str(e)}", ) diff --git a/app/api/google_drive.py b/app/api/google_drive.py index b626a470..54a6028e 100644 --- a/app/api/google_drive.py +++ b/app/api/google_drive.py @@ -7,11 +7,15 @@ import os from datetime import datetime from typing import Annotated, Optional -from fastapi import APIRouter, Form, HTTPException, Request, status +from fastapi import APIRouter, Depends, Form, HTTPException, Request, status +from sqlalchemy.orm import Session from app.auth import require_login from app.config import settings +from app.database import get_db from app.utils.oauth_helper import exchange_oauth_token +from app.utils.settings_service import save_setting_to_db +from app.utils.settings_sync import notify_settings_updated # Set up logging logger = logging.getLogger(__name__) @@ -45,7 +49,9 @@ async def exchange_google_drive_token( } # Use shared OAuth helper (handles secure logging and error handling) - token_data = exchange_oauth_token(provider_name="Google Drive", token_url=token_url, payload=payload) + token_data = exchange_oauth_token( + provider_name="Google Drive", token_url=token_url, payload=payload + ) # Return just what's needed by the frontend return { @@ -64,38 +70,73 @@ async def update_google_drive_settings( client_secret: Annotated[Optional[str], Form()] = None, folder_id: Annotated[Optional[str], Form()] = None, use_oauth: Annotated[str, Form()] = "true", + db: Session = Depends(get_db), ): """ - Update Google Drive settings in memory + Update Google Drive settings in memory and persist to database """ try: - logger.info("Updating Google Drive settings in memory") + logger.info("Updating Google Drive settings in memory and database") + + user = request.session.get("user", {}) if hasattr(request, "session") else {} + changed_by = ( + user.get("preferred_username") + or user.get("username") + or user.get("email") + or user.get("id") + or "wizard" + ) # Convert use_oauth string to boolean use_oauth_bool = use_oauth.lower() in ("true", "1", "yes", "y", "t") - # Update settings in memory + # Update settings in memory and persist to database if refresh_token: settings.google_drive_refresh_token = refresh_token - logger.info("Updated GOOGLE_DRIVE_REFRESH_TOKEN in memory") + save_setting_to_db( + db, "google_drive_refresh_token", refresh_token, changed_by=changed_by + ) + logger.info("Updated GOOGLE_DRIVE_REFRESH_TOKEN in memory and database") if client_id: settings.google_drive_client_id = client_id - logger.info("Updated GOOGLE_DRIVE_CLIENT_ID in memory") + save_setting_to_db( + db, "google_drive_client_id", client_id, changed_by=changed_by + ) + logger.info("Updated GOOGLE_DRIVE_CLIENT_ID in memory and database") if client_secret: settings.google_drive_client_secret = client_secret - logger.info("Updated GOOGLE_DRIVE_CLIENT_SECRET in memory") + save_setting_to_db( + db, "google_drive_client_secret", client_secret, changed_by=changed_by + ) + logger.info("Updated GOOGLE_DRIVE_CLIENT_SECRET in memory and database") if folder_id: settings.google_drive_folder_id = folder_id - logger.info("Updated GOOGLE_DRIVE_FOLDER_ID in memory") + save_setting_to_db( + db, "google_drive_folder_id", folder_id, changed_by=changed_by + ) + logger.info("Updated GOOGLE_DRIVE_FOLDER_ID in memory and database") # Set the OAuth flag settings.google_drive_use_oauth = use_oauth_bool - logger.info(f"Updated GOOGLE_DRIVE_USE_OAUTH in memory to {use_oauth_bool}") + save_setting_to_db( + db, + "google_drive_use_oauth", + str(use_oauth_bool).lower(), + changed_by=changed_by, + ) + logger.info( + f"Updated GOOGLE_DRIVE_USE_OAUTH in memory and database to {use_oauth_bool}" + ) - return {"status": "success", "message": "Google Drive settings have been updated in memory"} + notify_settings_updated() + + return { + "status": "success", + "message": "Google Drive settings have been updated in memory and database", + } except Exception as e: logger.exception(f"Unexpected error updating Google Drive settings: {str(e)}") @@ -113,7 +154,8 @@ async def test_google_drive_token(request: Request): Tests both OAuth and service account approaches based on configuration. """ try: - from app.tasks.upload_to_google_drive import get_drive_service_oauth, get_google_drive_service + from app.tasks.upload_to_google_drive import (get_drive_service_oauth, + get_google_drive_service) logger.info("Testing Google Drive token validity") @@ -125,7 +167,10 @@ async def test_google_drive_token(request: Request): and settings.google_drive_refresh_token ): logger.warning("Google Drive OAuth credentials not fully configured") - return {"status": "error", "message": "Google Drive OAuth credentials are not fully configured"} + return { + "status": "error", + "message": "Google Drive OAuth credentials are not fully configured", + } try: # Test OAuth connection @@ -187,8 +232,13 @@ async def test_google_drive_token(request: Request): else: # Test service account connection if not settings.google_drive_credentials_json: - logger.warning("Google Drive service account credentials not configured") - return {"status": "error", "message": "Google Drive service account credentials are not configured"} + logger.warning( + "Google Drive service account credentials not configured" + ) + return { + "status": "error", + "message": "Google Drive service account credentials are not configured", + } try: service = get_google_drive_service() @@ -203,7 +253,9 @@ async def test_google_drive_token(request: Request): else: user_display = user_email - logger.info(f"Successfully connected to Google Drive using service account as {user_display}") + logger.info( + f"Successfully connected to Google Drive using service account as {user_display}" + ) return { "status": "success", @@ -214,7 +266,10 @@ async def test_google_drive_token(request: Request): except Exception as e: error_msg = str(e) logger.error(f"Google Drive service account test failed: {error_msg}") - return {"status": "error", "message": f"Service account validation failed: {error_msg}"} + return { + "status": "error", + "message": f"Service account validation failed: {error_msg}", + } except Exception as e: logger.exception("Unexpected error testing Google Drive token") @@ -246,7 +301,10 @@ async def get_google_drive_token_info(request: Request): and settings.google_drive_refresh_token ): logger.warning("Google Drive OAuth credentials not fully configured") - return {"status": "error", "message": "Google Drive OAuth credentials are not fully configured"} + return { + "status": "error", + "message": "Google Drive OAuth credentials are not fully configured", + } try: # Get credentials and access token @@ -333,17 +391,29 @@ async def save_dropbox_settings( client_secret: Annotated[Optional[str], Form()] = None, folder_id: Annotated[Optional[str], Form()] = None, use_oauth: Annotated[str, Form()] = "true", + db: Session = Depends(get_db), ): """ - Save Google Drive settings to the .env file + Save Google Drive settings to the .env file (best-effort) and persist to database. """ try: # Get the path to the .env file - env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env") + env_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env" + ) # Convert use_oauth string to boolean use_oauth_bool = use_oauth.lower() in ("true", "1", "yes", "y", "t") + user = request.session.get("user", {}) if hasattr(request, "session") else {} + changed_by = ( + user.get("preferred_username") + or user.get("username") + or user.get("email") + or user.get("id") + or "wizard" + ) + # Define settings to update drive_settings = {"GOOGLE_DRIVE_USE_OAUTH": str(use_oauth_bool).lower()} @@ -376,7 +446,9 @@ async def save_dropbox_settings( stripped_line = line.rstrip() is_updated = False for key, value in drive_settings.items(): - if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="): + if stripped_line.startswith( + f"{key}=" + ) or stripped_line.startswith(f"# {key}="): # Uncomment if commented out - check the original stripped line new_env_lines.append(f"{key}={value}") updated.add(key) @@ -396,7 +468,9 @@ async def save_dropbox_settings( logger.info("Successfully updated Google Drive settings in .env file") except Exception as e: - logger.warning(f"Failed to update .env file: {str(e)}, but will continue with in-memory update") + logger.warning( + f"Failed to update .env file: {str(e)}, but will continue with in-memory update" + ) else: logger.warning( f".env file not found at {env_path}, skipping file update but continuing with in-memory update" @@ -415,7 +489,33 @@ async def save_dropbox_settings( # Set OAuth flag settings.google_drive_use_oauth = use_oauth_bool - logger.info("Successfully updated Google Drive settings in memory") + # Persist to database + save_setting_to_db( + db, + "google_drive_use_oauth", + str(use_oauth_bool).lower(), + changed_by=changed_by, + ) + if refresh_token: + save_setting_to_db( + db, "google_drive_refresh_token", refresh_token, changed_by=changed_by + ) + if client_id: + save_setting_to_db( + db, "google_drive_client_id", client_id, changed_by=changed_by + ) + if client_secret: + save_setting_to_db( + db, "google_drive_client_secret", client_secret, changed_by=changed_by + ) + if folder_id: + save_setting_to_db( + db, "google_drive_folder_id", folder_id, changed_by=changed_by + ) + + notify_settings_updated() + + logger.info("Successfully updated Google Drive settings in memory and database") return { "status": "success", @@ -426,5 +526,6 @@ async def save_dropbox_settings( except Exception as e: logger.exception(f"Unexpected error saving Google Drive settings: {str(e)}") raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to save Google Drive settings: {str(e)}" + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to save Google Drive settings: {str(e)}", ) diff --git a/app/api/onedrive.py b/app/api/onedrive.py index 894a1a44..0329b406 100644 --- a/app/api/onedrive.py +++ b/app/api/onedrive.py @@ -8,11 +8,15 @@ from datetime import datetime, timedelta from typing import Annotated, Optional import requests -from fastapi import APIRouter, Form, HTTPException, Request, status +from fastapi import APIRouter, Depends, Form, HTTPException, Request, status +from sqlalchemy.orm import Session from app.auth import require_login from app.config import settings +from app.database import get_db from app.utils.oauth_helper import exchange_oauth_token +from app.utils.settings_service import save_setting_to_db +from app.utils.settings_sync import notify_settings_updated # Set up logging logger = logging.getLogger(__name__) @@ -47,10 +51,15 @@ async def exchange_onedrive_token( } # Use shared OAuth helper (handles secure logging and error handling) - token_data = exchange_oauth_token(provider_name="OneDrive", token_url=token_url, payload=payload) + token_data = exchange_oauth_token( + provider_name="OneDrive", token_url=token_url, payload=payload + ) # Return just what's needed by the frontend - return {"refresh_token": token_data["refresh_token"], "expires_in": token_data.get("expires_in", 3600)} + return { + "refresh_token": token_data["refresh_token"], + "expires_in": token_data.get("expires_in", 3600), + } @router.get("/onedrive/test-token") @@ -68,7 +77,10 @@ async def test_onedrive_token(request: Request): or not settings.onedrive_client_secret ): logger.warning("OneDrive credentials not fully configured") - return {"status": "error", "message": "OneDrive credentials are not fully configured"} + return { + "status": "error", + "message": "OneDrive credentials are not fully configured", + } # Refresh token to get a new access token and expiration info tenant_id = settings.onedrive_tenant_id or "common" @@ -82,27 +94,39 @@ async def test_onedrive_token(request: Request): "scope": "offline_access Files.ReadWrite", } - response = requests.post(token_url, data=refresh_data, timeout=settings.http_request_timeout) + response = requests.post( + token_url, data=refresh_data, timeout=settings.http_request_timeout + ) if response.status_code != 200: logger.error(f"Failed to refresh OneDrive token: {response.text}") - return {"status": "error", "message": "Refresh token has expired or is invalid", "needs_reauth": True} + return { + "status": "error", + "message": "Refresh token has expired or is invalid", + "needs_reauth": True, + } token_data = response.json() access_token = token_data.get("access_token") - expires_in = token_data.get("expires_in", 3600) # Default to 1 hour if not specified + expires_in = token_data.get( + "expires_in", 3600 + ) # Default to 1 hour if not specified # Check if we got a new refresh token (Microsoft sometimes issues a new one) new_refresh_token = token_data.get("refresh_token") if new_refresh_token and new_refresh_token != settings.onedrive_refresh_token: - logger.info("Received new refresh token from Microsoft - will update configuration") + logger.info( + "Received new refresh token from Microsoft - will update configuration" + ) # Update refresh token in memory settings.onedrive_refresh_token = new_refresh_token # Also try to update .env file if it exists try: - env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env") + env_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env" + ) if os.path.exists(env_path): with open(env_path, "r") as f: env_lines = f.readlines() @@ -112,13 +136,17 @@ async def test_onedrive_token(request: Request): for line in env_lines: if line.startswith("ONEDRIVE_REFRESH_TOKEN="): - updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n") + updated_lines.append( + f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n" + ) updated = True else: updated_lines.append(line) if not updated: - updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n") + updated_lines.append( + f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n" + ) with open(env_path, "w") as f: f.writelines(updated_lines) @@ -128,14 +156,38 @@ async def test_onedrive_token(request: Request): except Exception as e: logger.warning(f"Failed to update refresh token in .env file: {e}") + # Persist the rotated refresh token to the database + try: + from app.database import SessionLocal + + _db = SessionLocal() + try: + save_setting_to_db( + _db, + "onedrive_refresh_token", + new_refresh_token, + changed_by="onedrive_token_rotation", + ) + notify_settings_updated() + finally: + _db.close() + except Exception as _e: + logger.warning( + f"Failed to persist rotated OneDrive refresh token to database: {_e}" + ) + # Test the access token by getting user information user_info_url = "https://graph.microsoft.com/v1.0/me" headers = {"Authorization": f"Bearer {access_token}"} - user_response = requests.get(user_info_url, headers=headers, timeout=settings.http_request_timeout) + user_response = requests.get( + user_info_url, headers=headers, timeout=settings.http_request_timeout + ) if user_response.status_code != 200: - logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}") + logger.error( + f"OneDrive token test failed: {user_response.status_code} {user_response.text}" + ) return { "status": "error", "message": f"Token validation failed with status {user_response.status_code}: {user_response.text}", @@ -203,65 +255,72 @@ async def save_onedrive_settings( client_secret: Annotated[Optional[str], Form()] = None, tenant_id: Annotated[str, Form()] = "common", folder_path: Annotated[Optional[str], Form()] = None, + db: Session = Depends(get_db), ): """ - Save OneDrive settings to the .env file + Saves to database (primary) and .env file (best-effort). """ try: - # Get the path to the .env file - env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env") + user = request.session.get("user", {}) if hasattr(request, "session") else {} + changed_by = ( + user.get("preferred_username") + or user.get("username") + or user.get("email") + or user.get("id") + or "wizard" + ) - if not os.path.exists(env_path): - logger.error(f".env file not found at {env_path}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Could not find .env file to update" + # Best-effort .env file write + try: + env_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env" ) + if not os.path.exists(env_path): + logger.warning( + f".env file not found at {env_path}, skipping file write" + ) + else: + logger.info(f"Updating OneDrive settings in {env_path}") - logger.info(f"Updating OneDrive settings in {env_path}") + with open(env_path, "r") as f: + env_lines = f.readlines() - # Read the current .env file - with open(env_path, "r") as f: - env_lines = f.readlines() + onedrive_settings = {"ONEDRIVE_REFRESH_TOKEN": refresh_token} + if client_id: + onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id + if client_secret: + onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret + if tenant_id: + onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id + if folder_path: + onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path - # Define settings to update - onedrive_settings = { - "ONEDRIVE_REFRESH_TOKEN": refresh_token, - } + updated = set() + new_env_lines = [] + for line in env_lines: + stripped_line = line.rstrip() + is_updated = False + for key, value in onedrive_settings.items(): + if stripped_line.startswith( + f"{key}=" + ) or stripped_line.startswith(f"# {key}="): + new_env_lines.append(f"{key}={value}") + updated.add(key) + is_updated = True + break + if not is_updated: + new_env_lines.append(stripped_line) - # Only update these if provided - if client_id: - onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id - if client_secret: - onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret - if tenant_id: - onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id - if folder_path: - onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path + for key, value in onedrive_settings.items(): + if key not in updated: + new_env_lines.append(f"{key}={value}") - # Process each line and update or add settings - updated = set() - new_env_lines = [] - for line in env_lines: - stripped_line = line.rstrip() - is_updated = False - for key, value in onedrive_settings.items(): - if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="): - # Uncomment if commented out - check the original stripped line - new_env_lines.append(f"{key}={value}") - updated.add(key) - is_updated = True - break - if not is_updated: - new_env_lines.append(stripped_line) + with open(env_path, "w") as f: + f.write("\n".join(new_env_lines) + "\n") - # Add any settings that weren't updated (they weren't in the file) - for key, value in onedrive_settings.items(): - if key not in updated: - new_env_lines.append(f"{key}={value}") - - # Write the updated .env file - with open(env_path, "w") as f: - f.write("\n".join(new_env_lines) + "\n") + logger.info("Successfully updated OneDrive settings in .env file") + except Exception as env_err: + logger.warning(f"Failed to write .env file (non-fatal): {env_err}") # Update the settings in memory if refresh_token: @@ -275,16 +334,38 @@ async def save_onedrive_settings( if folder_path: settings.onedrive_folder_path = folder_path - logger.info("Successfully updated OneDrive settings") + # Persist to database (primary) + if refresh_token: + save_setting_to_db( + db, "onedrive_refresh_token", refresh_token, changed_by=changed_by + ) + if client_id: + save_setting_to_db( + db, "onedrive_client_id", client_id, changed_by=changed_by + ) + if client_secret: + save_setting_to_db( + db, "onedrive_client_secret", client_secret, changed_by=changed_by + ) + if tenant_id: + save_setting_to_db( + db, "onedrive_tenant_id", tenant_id, changed_by=changed_by + ) + if folder_path: + save_setting_to_db( + db, "onedrive_folder_path", folder_path, changed_by=changed_by + ) + notify_settings_updated() + + logger.info("Successfully saved OneDrive settings") return {"status": "success", "message": "OneDrive settings have been saved"} - except HTTPException: - raise except Exception as e: logger.exception(f"Unexpected error saving OneDrive settings: {str(e)}") raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to save OneDrive settings: {str(e)}" + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to save OneDrive settings: {str(e)}", ) @@ -297,33 +378,60 @@ async def update_onedrive_settings( client_secret: Annotated[Optional[str], Form()] = None, tenant_id: Annotated[str, Form()] = "common", folder_path: Annotated[Optional[str], Form()] = None, + db: Session = Depends(get_db), ): """ - Update OneDrive settings in memory (without modifying .env file) + Update OneDrive settings in memory and persist to database """ try: - logger.info("Updating OneDrive settings in memory") + logger.info("Updating OneDrive settings in memory and database") - # Update settings in memory + user = request.session.get("user", {}) if hasattr(request, "session") else {} + changed_by = ( + user.get("preferred_username") + or user.get("username") + or user.get("email") + or user.get("id") + or "wizard" + ) + + # Update settings in memory and persist to database if refresh_token: settings.onedrive_refresh_token = refresh_token - logger.info("Updated ONEDRIVE_REFRESH_TOKEN in memory") + save_setting_to_db( + db, "onedrive_refresh_token", refresh_token, changed_by=changed_by + ) + logger.info("Updated ONEDRIVE_REFRESH_TOKEN in memory and database") if client_id: settings.onedrive_client_id = client_id - logger.info("Updated ONEDRIVE_CLIENT_ID in memory") + save_setting_to_db( + db, "onedrive_client_id", client_id, changed_by=changed_by + ) + logger.info("Updated ONEDRIVE_CLIENT_ID in memory and database") if client_secret: settings.onedrive_client_secret = client_secret - logger.info("Updated ONEDRIVE_CLIENT_SECRET in memory") + save_setting_to_db( + db, "onedrive_client_secret", client_secret, changed_by=changed_by + ) + logger.info("Updated ONEDRIVE_CLIENT_SECRET in memory and database") if tenant_id: settings.onedrive_tenant_id = tenant_id - logger.info("Updated ONEDRIVE_TENANT_ID in memory") + save_setting_to_db( + db, "onedrive_tenant_id", tenant_id, changed_by=changed_by + ) + logger.info("Updated ONEDRIVE_TENANT_ID in memory and database") if folder_path: settings.onedrive_folder_path = folder_path - logger.info("Updated ONEDRIVE_FOLDER_PATH in memory") + save_setting_to_db( + db, "onedrive_folder_path", folder_path, changed_by=changed_by + ) + logger.info("Updated ONEDRIVE_FOLDER_PATH in memory and database") + + notify_settings_updated() # Test the token to make sure it works try: @@ -333,14 +441,21 @@ async def update_onedrive_settings( logger.info("Successfully tested OneDrive token") except Exception as e: logger.error(f"Token test failed after updating settings: {str(e)}") - return {"status": "warning", "message": "Settings updated but token test failed: " + str(e)} + return { + "status": "warning", + "message": "Settings updated but token test failed: " + str(e), + } - return {"status": "success", "message": "OneDrive settings have been updated in memory"} + return { + "status": "success", + "message": "OneDrive settings have been updated in memory and database", + } except Exception as e: logger.exception(f"Unexpected error updating OneDrive settings: {str(e)}") raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to update OneDrive settings: {str(e)}" + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to update OneDrive settings: {str(e)}", ) diff --git a/app/api/settings.py b/app/api/settings.py index dd82bbb4..a22ac742 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -450,3 +450,61 @@ async def rollback_setting_to_history( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to roll back setting: {key}", ) + + +@router.get("/export-env") +async def export_env_settings( + request: Request, + db: DbSession, + admin: AdminUser, + source: str = "db", +): + """ + Export current settings as a ``.env`` file. + + Query params: + - ``source=db`` (default) – only settings explicitly saved to the database. + - ``source=effective`` – full runtime configuration (DB > ENV > defaults) for + every key defined in SETTING_METADATA. + + Returns a downloadable plain-text file suitable for bootstrapping another + installation. All values — including sensitive ones — are included; only + admins can access this endpoint. + """ + from fastapi.responses import Response as FastAPIResponse + + from app.utils.settings_service import get_settings_for_export + + if source not in ("db", "effective"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="source must be 'db' or 'effective'", + ) + + try: + export_data = get_settings_for_export(db, source=source) + lines = [ + "# DocuElevate configuration export", + f"# Source: {source}", + "# Generated by DocuElevate Settings Export", + "# WARNING: This file contains sensitive values. Handle with care.", + "", + ] + for env_key, value in export_data.items(): + lines.append(f"{env_key}={value}") + lines.append("") # trailing newline + content = "\n".join(lines) + + return FastAPIResponse( + content=content, + media_type="text/plain", + headers={ + "Content-Disposition": f'attachment; filename="docuelevate-{source}.env"' + }, + ) + except Exception as e: + logger.error(f"Error exporting settings: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to export settings", + ) diff --git a/app/utils/settings_service.py b/app/utils/settings_service.py index 3b6fc85a..b821bcee 100644 --- a/app/utils/settings_service.py +++ b/app/utils/settings_service.py @@ -1287,3 +1287,38 @@ def rollback_setting( ) db.rollback() return False + + +def get_settings_for_export(db: Session, source: str = "db") -> Dict[str, str]: + """ + Collect settings for export as environment variables. + + Args: + db: Database session + source: ``"db"`` to export only database-persisted settings (default); + ``"effective"`` to export the full current runtime configuration + (DB overrides ENV overrides application defaults) for every key + listed in SETTING_METADATA. + + Returns: + Ordered dict mapping uppercase ENV variable names to their string values. + Sensitive values are included (the caller is responsible for access control). + """ + if source == "effective": + from app.config import settings as app_settings + + db_settings = get_all_settings_from_db(db) + result = {} + for key in sorted(SETTING_METADATA.keys()): + # DB wins, then live settings object (ENV/default) + if key in db_settings and db_settings[key] is not None: + value = db_settings[key] + else: + value = getattr(app_settings, key, None) + if value is not None: + result[key.upper()] = str(value) + return result + else: + # DB only + db_settings = get_all_settings_from_db(db) + return {k.upper(): v for k, v in sorted(db_settings.items()) if v is not None} diff --git a/app/views/wizard.py b/app/views/wizard.py index 27696c8c..3105e7c4 100644 --- a/app/views/wizard.py +++ b/app/views/wizard.py @@ -10,6 +10,7 @@ from fastapi.responses import RedirectResponse from sqlalchemy.orm import Session from app.utils.settings_service import save_setting_to_db +from app.utils.settings_sync import notify_settings_updated from app.utils.setup_wizard import get_wizard_steps from app.views.base import APIRouter, get_db, templates @@ -18,7 +19,7 @@ router = APIRouter() @router.get("/setup") -async def setup_wizard(request: Request, step: int = 1): +async def setup_wizard(request: Request, step: int = 1, db: Session = Depends(get_db)): """ Setup wizard for first-time configuration. @@ -39,7 +40,38 @@ async def setup_wizard(request: Request, step: int = 1): current_settings = wizard_steps.get(step, []) # Get step category (all settings in a step should have same category) - step_category = current_settings[0].get("wizard_category", "Configuration") if current_settings else "Configuration" + step_category = ( + current_settings[0].get("wizard_category", "Configuration") + if current_settings + else "Configuration" + ) + + # Enrich settings with current live values + from app.config import settings as app_settings + from app.utils.settings_service import get_setting_from_db + + enriched_settings = [] + for s in current_settings: + key = s["key"] + db_val = get_setting_from_db(db, key) + env_val = getattr(app_settings, key, None) + # Determine current_value and source + if db_val is not None: + current_value = db_val + value_source = "db" + elif env_val is not None and str(env_val).strip(): + current_value = str(env_val) + value_source = "env" + elif s.get("default") is not None: + current_value = s["default"] + value_source = "default" + else: + current_value = "" + value_source = "none" + enriched_settings.append( + {**s, "current_value": current_value, "value_source": value_source} + ) + current_settings = enriched_settings return templates.TemplateResponse( "setup_wizard.html", @@ -50,12 +82,15 @@ async def setup_wizard(request: Request, step: int = 1): "settings": current_settings, "step_category": step_category, "progress_percent": int((step / max_step) * 100), + "setup_skipped": bool(get_setting_from_db(db, "_setup_wizard_skipped")), }, ) @router.post("/setup") -async def setup_wizard_save(request: Request, step: int = Form(...), db: Session = Depends(get_db)): +async def setup_wizard_save( + request: Request, step: int = Form(...), db: Session = Depends(get_db) +): """ Save settings from the current wizard step. """ @@ -87,6 +122,9 @@ async def setup_wizard_save(request: Request, step: int = Form(...), db: Session logger.info(f"Setup wizard step {step}: Saved {saved_count} settings") + if saved_count > 0: + notify_settings_updated() + # Determine next step max_step = max(wizard_steps.keys()) next_step = step + 1 @@ -100,7 +138,9 @@ async def setup_wizard_save(request: Request, step: int = Form(...), db: Session except Exception as e: logger.error(f"Error saving wizard settings: {e}") - return RedirectResponse(url=f"/setup?step={step}&error=save_failed", status_code=303) + return RedirectResponse( + url=f"/setup?step={step}&error=save_failed", status_code=303 + ) @router.get("/setup/skip") @@ -122,3 +162,25 @@ async def setup_wizard_skip(request: Request): except Exception as e: logger.error(f"Error skipping setup wizard: {e}") return RedirectResponse(url="/", status_code=303) + + +@router.get("/setup/undo-skip") +async def setup_wizard_undo_skip(request: Request, db: Session = Depends(get_db)): + """ + Undo a previously skipped setup wizard. + + Removes the skip marker from the database so the wizard will be + presented again on next visit to the home page. Redirects to + step 1 of the wizard immediately. + """ + try: + from app.utils.settings_service import delete_setting_from_db + + delete_setting_from_db( + db, "_setup_wizard_skipped", changed_by="wizard_undo_skip" + ) + logger.info("Setup wizard skip marker removed; redirecting to wizard") + return RedirectResponse(url="/setup?step=1", status_code=303) + except Exception as e: + logger.error(f"Error undoing setup wizard skip: {e}") + return RedirectResponse(url="/settings", status_code=303) diff --git a/frontend/templates/settings.html b/frontend/templates/settings.html index 495193dd..36afbd72 100644 --- a/frontend/templates/settings.html +++ b/frontend/templates/settings.html @@ -20,10 +20,36 @@ This is a convenience feature to view and edit application settings through the web interface.

- - Audit Log - +
+ + Setup Wizard + +
+ + +
+ + Audit Log + +