gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: fix all flake8 linter errors across app/ directory

Browse Source 
- Run Black formatter and isort on all app/ files
- Remove unused imports (F401) across multiple files
- Add # noqa: F401 for intentional re-exports in celery_worker.py,
  tasks/__init__.py, utils.py, frontend.py, views/base.py
- Fix f-strings without placeholders (F541) in azure.py, notification.py,
  check_credentials.py, upload_to_onedrive.py, settings.py
- Fix bare except (E722) in upload_to_sftp.py
- Fix block comment format (E265) in models.py
- Move imports to top of file to fix E402 in celery_app.py, celery_worker.py
- Fix line-too-long (E501) by wrapping strings in multiple files
- Remove unused variable (F841) in upload_to_nextcloud.py

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-08 17:42:33 +00:00
parent 7827b97e06
commit d08040ac4a
73 changed files with 2200 additions and 2185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/utils/__init__.py
+1 -1
View File
@@ -7,4 +7,4 @@ from app.utils.file_operations import hash_file
from app.utils.logging import log_task_progress
# Export all the functions that should be available when importing from app.utils
__all__ = ['hash_file', 'log_task_progress']
__all__ = ["hash_file", "log_task_progress"]
app/utils/config_loader.py
+22 -21
View File
@@ -9,6 +9,7 @@ This module provides functionality to:
import logging
from typing import Any, Optional, Union
from sqlalchemy.orm import Session
from app.models import ApplicationSettings
@@ -19,27 +20,27 @@ logger = logging.getLogger(__name__)
def load_settings_from_db(settings_obj, db_session: Session) -> None:
"""
Load settings from database and apply them to the settings object.
Database settings take precedence over environment variables and defaults.
This function should be called after database initialization.
Args:
settings_obj: The Settings instance to update
db_session: Database session to use for loading settings
"""
try:
db_settings = db_session.query(ApplicationSettings).all()
if not db_settings:
logger.info("No database settings found, using environment/defaults")
return
# Apply database settings to the settings object
updated_count = 0
for db_setting in db_settings:
key = db_setting.key
value = db_setting.value
# Check if the setting exists in the Settings class
if hasattr(settings_obj, key):
# Get the field info to determine the type
@@ -47,17 +48,17 @@ def load_settings_from_db(settings_obj, db_session: Session) -> None:
if field_info:
# Convert value to the appropriate type
converted_value = convert_setting_value(value, field_info.annotation)
# Set the attribute
setattr(settings_obj, key, converted_value)
updated_count += 1
logger.debug(f"Applied database setting: {key}")
if updated_count > 0:
logger.info(f"Loaded {updated_count} settings from database")
else:
logger.info("No applicable database settings found")
except Exception as e:
logger.error(f"Error loading settings from database: {e}")
# Don't fail application startup if database settings can't be loaded
@@ -67,27 +68,27 @@ def load_settings_from_db(settings_obj, db_session: Session) -> None:
def convert_setting_value(value: Optional[str], field_type: Any) -> Any:
"""
Convert a string value from database to the appropriate type.
Args:
value: String value from database
field_type: Target type from Pydantic field annotation
Returns:
Converted value in the appropriate type
"""
if value is None:
return None
# Handle Optional types
origin = getattr(field_type, '__origin__', None)
origin = getattr(field_type, "__origin__", None)
if origin is Union:
# Get the non-None type from Union (for Optional)
args = getattr(field_type, '__args__', ())
args = getattr(field_type, "__args__", ())
field_type = next((arg for arg in args if arg is not type(None)), str)
# Convert based on type
if field_type == bool:
return value.lower() in ('true', '1', 'yes', 'y', 't')
return value.lower() in ("true", "1", "yes", "y", "t")
elif field_type == int:
try:
return int(value)
@@ -100,10 +101,10 @@ def convert_setting_value(value: Optional[str], field_type: Any) -> Any:
except ValueError:
logger.warning(f"Failed to convert '{value}' to float, returning 0.0")
return 0.0
elif field_type == list or getattr(field_type, '__origin__', None) == list:
elif field_type == list or getattr(field_type, "__origin__", None) == list:
# Handle list types - assume comma-separated values
if isinstance(value, str):
return [item.strip() for item in value.split(',') if item.strip()]
return [item.strip() for item in value.split(",") if item.strip()]
return value
else:
# Default to string
@@ -113,19 +114,19 @@ def convert_setting_value(value: Optional[str], field_type: Any) -> Any:
def reload_settings_from_db(settings_obj) -> bool:
"""
Reload settings from database.
This is useful after settings have been updated through the UI.
Note: Some settings require application restart to take effect.
Args:
settings_obj: The Settings instance to update
Returns:
True if reload was successful, False otherwise
"""
try:
from app.database import SessionLocal
db = SessionLocal()
try:
load_settings_from_db(settings_obj, db)
app/utils/config_validator.py
+16 -20
View File
@@ -4,29 +4,25 @@ Configuration validation for the application.
This file serves as a backward-compatible interface to the config_validator package.
"""
# Import and re-export all functions from the new package
from app.utils.config_validator.validators import (
validate_email_config,
validate_storage_configs,
validate_notification_config,
check_all_configs
)
from app.utils.config_validator.masking import mask_sensitive_value
from app.utils.config_validator.providers import get_provider_status
from app.utils.config_validator.settings_display import (
get_settings_for_display,
dump_all_settings
from app.utils.config_validator.settings_display import dump_all_settings, get_settings_for_display
# Import and re-export all functions from the new package
from app.utils.config_validator.validators import (
check_all_configs,
validate_email_config,
validate_notification_config,
validate_storage_configs,
)
__all__ = [
'validate_email_config',
'validate_storage_configs',
'validate_notification_config',
'mask_sensitive_value',
'get_provider_status',
'get_settings_for_display',
'dump_all_settings',
'check_all_configs'
"validate_email_config",
"validate_storage_configs",
"validate_notification_config",
"mask_sensitive_value",
"get_provider_status",
"get_settings_for_display",
"dump_all_settings",
"check_all_configs",
]
app/utils/config_validator/__init__.py
+16 -19
View File
@@ -2,28 +2,25 @@
Configuration validation package for the application.
"""
from app.utils.config_validator.validators import (
validate_email_config,
validate_storage_configs,
validate_notification_config,
validate_auth_config,
check_all_configs
)
from app.utils.config_validator.masking import mask_sensitive_value
from app.utils.config_validator.providers import get_provider_status
from app.utils.config_validator.settings_display import (
get_settings_for_display,
dump_all_settings
from app.utils.config_validator.settings_display import dump_all_settings, get_settings_for_display
from app.utils.config_validator.validators import (
check_all_configs,
validate_auth_config,
validate_email_config,
validate_notification_config,
validate_storage_configs,
)
__all__ = [
'validate_email_config',
'validate_storage_configs',
'validate_notification_config',
'validate_auth_config',
'mask_sensitive_value',
'get_provider_status',
'get_settings_for_display',
'dump_all_settings',
'check_all_configs'
"validate_email_config",
"validate_storage_configs",
"validate_notification_config",
"validate_auth_config",
"mask_sensitive_value",
"get_provider_status",
"get_settings_for_display",
"dump_all_settings",
"check_all_configs",
]
app/utils/config_validator/masking.py
+1
View File
@@ -2,6 +2,7 @@
Module for masking sensitive information in configuration values
"""
def mask_sensitive_value(value):
"""
Masks sensitive values like API keys in logs and output
app/utils/config_validator/providers.py
+200 -171
View File
@@ -5,294 +5,323 @@ Module for handling provider status information
from app.config import settings
from app.utils.config_validator.masking import mask_sensitive_value
def get_provider_status():
"""
Returns status information for all configured providers
"""
providers = {}
# Add Authentication configuration
auth_enabled = getattr(settings, 'auth_enabled', False)
using_oidc = bool(getattr(settings, 'authentik_client_id', None) and
getattr(settings, 'authentik_client_secret', None) and
getattr(settings, 'authentik_config_url', None))
auth_enabled = getattr(settings, "auth_enabled", False)
using_oidc = bool(
getattr(settings, "authentik_client_id", None)
and getattr(settings, "authentik_client_secret", None)
and getattr(settings, "authentik_config_url", None)
)
auth_method = "OIDC" if using_oidc else "Basic Auth" if auth_enabled else "None"
providers["Authentication"] = {
"name": "Authentication",
"name": "Authentication",
"icon": "fa-solid fa-lock",
"configured": bool(auth_enabled and
(getattr(settings, 'admin_username', None) or
using_oidc)),
"configured": bool(auth_enabled and (getattr(settings, "admin_username", None) or using_oidc)),
"enabled": auth_enabled,
"description": "Access control and user authentication",
"details": {
"method": auth_method,
"provider_name": getattr(settings, 'oauth_provider_name', 'Not set') if using_oidc else "N/A",
"session_security": "Configured" if getattr(settings, 'session_secret', None) else "Not configured"
}
"provider_name": getattr(settings, "oauth_provider_name", "Not set") if using_oidc else "N/A",
"session_security": "Configured" if getattr(settings, "session_secret", None) else "Not configured",
},
}
# Add Notification configuration - Make sure this provider is near the top of the list
providers["Notifications"] = {
"name": "Notifications",
"name": "Notifications",
"icon": "fa-solid fa-bell",
"configured": bool(getattr(settings, 'notification_urls', None)),
"configured": bool(getattr(settings, "notification_urls", None)),
"enabled": True,
"description": "Send system notifications via various services",
"details": {
"services": str(len(getattr(settings, 'notification_urls', []))) + " service(s) configured" if getattr(settings, 'notification_urls', None) else "Not configured",
"task_failure": getattr(settings, 'notify_on_task_failure', True),
"credential_failure": getattr(settings, 'notify_on_credential_failure', True),
"startup": getattr(settings, 'notify_on_startup', True),
"shutdown": getattr(settings, 'notify_on_shutdown', False)
"services": (
str(len(getattr(settings, "notification_urls", []))) + " service(s) configured"
if getattr(settings, "notification_urls", None)
else "Not configured"
),
"task_failure": getattr(settings, "notify_on_task_failure", True),
"credential_failure": getattr(settings, "notify_on_credential_failure", True),
"startup": getattr(settings, "notify_on_startup", True),
"shutdown": getattr(settings, "notify_on_shutdown", False),
},
"testable": True,
"test_endpoint": "/api/diagnostic/test-notification"
"test_endpoint": "/api/diagnostic/test-notification",
}
# Add AI services first
providers["OpenAI"] = {
"name": "OpenAI",
"name": "OpenAI",
"icon": "fa-brands fa-openai",
"configured": bool(getattr(settings, 'openai_api_key', None) and
str(getattr(settings, 'openai_api_key', '')).startswith('sk-')),
"configured": bool(
getattr(settings, "openai_api_key", None) and str(getattr(settings, "openai_api_key", "")).startswith("sk-")
),
"enabled": True,
"description": "AI-powered document analysis and metadata extraction",
"details": {
"api_key": mask_sensitive_value(getattr(settings, 'openai_api_key', None)),
"base_url": getattr(settings, 'openai_base_url', 'https://api.openai.com/v1'),
"model": getattr(settings, 'openai_model', 'gpt-4')
}
"api_key": mask_sensitive_value(getattr(settings, "openai_api_key", None)),
"base_url": getattr(settings, "openai_base_url", "https://api.openai.com/v1"),
"model": getattr(settings, "openai_model", "gpt-4"),
},
}
providers["Azure AI"] = {
"name": "Azure AI",
"name": "Azure AI",
"icon": "fa-solid fa-robot",
"configured": bool(getattr(settings, 'azure_ai_key', None) and
getattr(settings, 'azure_endpoint', None)),
"configured": bool(getattr(settings, "azure_ai_key", None) and getattr(settings, "azure_endpoint", None)),
"enabled": True,
"description": "Microsoft Azure Document Intelligence",
"details": {
"api_key": mask_sensitive_value(getattr(settings, 'azure_ai_key', None)),
"endpoint": getattr(settings, 'azure_endpoint', 'Not set'),
"region": getattr(settings, 'azure_region', 'Not set')
}
"api_key": mask_sensitive_value(getattr(settings, "azure_ai_key", None)),
"endpoint": getattr(settings, "azure_endpoint", "Not set"),
"region": getattr(settings, "azure_region", "Not set"),
},
}
# Add Dropbox configuration - alphabetically ordered providers
providers["Dropbox"] = {
"name": "Dropbox",
"name": "Dropbox",
"icon": "fa-brands fa-dropbox",
"configured": bool(getattr(settings, 'dropbox_app_key', None) and
getattr(settings, 'dropbox_app_secret', None) and
getattr(settings, 'dropbox_refresh_token', None)),
"configured": bool(
getattr(settings, "dropbox_app_key", None)
and getattr(settings, "dropbox_app_secret", None)
and getattr(settings, "dropbox_refresh_token", None)
),
"enabled": True,
"description": "Upload files to Dropbox cloud storage",
"details": {
"folder": getattr(settings, 'dropbox_folder', 'Not set'),
"app_key": getattr(settings, 'dropbox_app_key', 'Not set'),
"app_secret": mask_sensitive_value(getattr(settings, 'dropbox_app_secret', None)),
"refresh_token": mask_sensitive_value(getattr(settings, 'dropbox_refresh_token', None))
}
"folder": getattr(settings, "dropbox_folder", "Not set"),
"app_key": getattr(settings, "dropbox_app_key", "Not set"),
"app_secret": mask_sensitive_value(getattr(settings, "dropbox_app_secret", None)),
"refresh_token": mask_sensitive_value(getattr(settings, "dropbox_refresh_token", None)),
},
}
# Add Email configuration
providers["Email"] = {
"name": "Email",
"name": "Email",
"icon": "fa-solid fa-envelope",
"configured": bool(getattr(settings, 'email_host', None) and
getattr(settings, 'email_default_recipient', None)),
"configured": bool(
getattr(settings, "email_host", None) and getattr(settings, "email_default_recipient", None)
),
"enabled": True,
"description": "Send documents via email",
"details": {
"host": getattr(settings, 'email_host', 'Not set'),
"port": getattr(settings, 'email_port', 'Not set'),
"username": getattr(settings, 'email_username', 'Not set'),
"password": mask_sensitive_value(getattr(settings, 'email_password', None)),
"use_tls": getattr(settings, 'email_use_tls', 'Not set'),
"sender": getattr(settings, 'email_sender', 'Not set'),
"default_recipient": getattr(settings, 'email_default_recipient', 'Not set')
}
"host": getattr(settings, "email_host", "Not set"),
"port": getattr(settings, "email_port", "Not set"),
"username": getattr(settings, "email_username", "Not set"),
"password": mask_sensitive_value(getattr(settings, "email_password", None)),
"use_tls": getattr(settings, "email_use_tls", "Not set"),
"sender": getattr(settings, "email_sender", "Not set"),
"default_recipient": getattr(settings, "email_default_recipient", "Not set"),
},
}
# Add FTP configuration to providers
providers["FTP Storage"] = {
"name": "FTP Storage",
"name": "FTP Storage",
"icon": "fa-solid fa-server",
"configured": bool(getattr(settings, 'ftp_host', None) and
getattr(settings, 'ftp_username', None) and
getattr(settings, 'ftp_password', None)),
"configured": bool(
getattr(settings, "ftp_host", None)
and getattr(settings, "ftp_username", None)
and getattr(settings, "ftp_password", None)
),
"enabled": True,
"description": "Upload files to FTP server",
"details": {
"host": getattr(settings, 'ftp_host', 'Not set'),
"port": getattr(settings, 'ftp_port', 'Not set'),
"username": getattr(settings, 'ftp_username', 'Not set'),
"password": mask_sensitive_value(getattr(settings, 'ftp_password', None)),
"folder": getattr(settings, 'ftp_folder', 'Not set'),
"tls": getattr(settings, 'ftp_use_tls', True),
"allow_plaintext": getattr(settings, 'ftp_allow_plaintext', True)
}
"host": getattr(settings, "ftp_host", "Not set"),
"port": getattr(settings, "ftp_port", "Not set"),
"username": getattr(settings, "ftp_username", "Not set"),
"password": mask_sensitive_value(getattr(settings, "ftp_password", None)),
"folder": getattr(settings, "ftp_folder", "Not set"),
"tls": getattr(settings, "ftp_use_tls", True),
"allow_plaintext": getattr(settings, "ftp_allow_plaintext", True),
},
}
# Check Google Drive configuration
gdrive_oauth_configured = bool(getattr(settings, 'google_drive_client_id', None) and
getattr(settings, 'google_drive_client_secret', None) and
getattr(settings, 'google_drive_refresh_token', None))
gdrive_sa_configured = bool(getattr(settings, 'google_drive_credentials_json', None))
gdrive_oauth_configured = bool(
getattr(settings, "google_drive_client_id", None)
and getattr(settings, "google_drive_client_secret", None)
and getattr(settings, "google_drive_refresh_token", None)
)
gdrive_sa_configured = bool(getattr(settings, "google_drive_credentials_json", None))
# Determine if using OAuth or service account
use_oauth = getattr(settings, 'google_drive_use_oauth', False)
use_oauth = getattr(settings, "google_drive_use_oauth", False)
is_configured = (use_oauth and gdrive_oauth_configured) or (not use_oauth and gdrive_sa_configured)
providers["Google Drive"] = {
"name": "Google Drive",
"name": "Google Drive",
"icon": "fa-brands fa-google-drive",
"configured": is_configured and bool(getattr(settings, 'google_drive_folder_id', None)),
"configured": is_configured and bool(getattr(settings, "google_drive_folder_id", None)),
"enabled": True,
"description": "Store documents in Google Drive",
"details": {
"auth_type": "OAuth" if use_oauth else "Service Account",
"client_id": getattr(settings, 'google_drive_client_id', 'Not set') if use_oauth else 'N/A',
"client_secret": mask_sensitive_value(getattr(settings, 'google_drive_client_secret', None)) if use_oauth else 'N/A',
"refresh_token": mask_sensitive_value(getattr(settings, 'google_drive_refresh_token', None)) if use_oauth else 'N/A',
"credentials_json": mask_sensitive_value(getattr(settings, 'google_drive_credentials_json', None)) if not use_oauth else 'N/A',
"folder_id": getattr(settings, 'google_drive_folder_id', 'Not set'),
"delegate": getattr(settings, 'google_drive_delegate_to', 'Not set') if not use_oauth else 'N/A'
}
"client_id": getattr(settings, "google_drive_client_id", "Not set") if use_oauth else "N/A",
"client_secret": (
mask_sensitive_value(getattr(settings, "google_drive_client_secret", None)) if use_oauth else "N/A"
),
"refresh_token": (
mask_sensitive_value(getattr(settings, "google_drive_refresh_token", None)) if use_oauth else "N/A"
),
"credentials_json": (
mask_sensitive_value(getattr(settings, "google_drive_credentials_json", None))
if not use_oauth
else "N/A"
),
"folder_id": getattr(settings, "google_drive_folder_id", "Not set"),
"delegate": getattr(settings, "google_drive_delegate_to", "Not set") if not use_oauth else "N/A",
},
}
# Check NextCloud configuration
nextcloud_url = getattr(settings, 'nextcloud_upload_url', 'Not set')
nextcloud_url = getattr(settings, "nextcloud_upload_url", "Not set")
# Extract base URL from WebDAV URL (remove the /remote.php part and everything after it)
nextcloud_base_url = nextcloud_url
if nextcloud_url != 'Not set' and nextcloud_url is not None and '/remote.php' in nextcloud_url:
nextcloud_base_url = nextcloud_url.split('/remote.php')[0]
if nextcloud_url != "Not set" and nextcloud_url is not None and "/remote.php" in nextcloud_url:
nextcloud_base_url = nextcloud_url.split("/remote.php")[0]
providers["NextCloud"] = {
"name": "NextCloud",
"name": "NextCloud",
"icon": "fa-solid fa-cloud",
"configured": bool(getattr(settings, 'nextcloud_upload_url', None) and
getattr(settings, 'nextcloud_username', None) and
getattr(settings, 'nextcloud_password', None)),
"configured": bool(
getattr(settings, "nextcloud_upload_url", None)
and getattr(settings, "nextcloud_username", None)
and getattr(settings, "nextcloud_password", None)
),
"enabled": True,
"description": "Store documents in NextCloud",
"details": {
"url": getattr(settings, 'nextcloud_upload_url', 'Not set'),
"url": getattr(settings, "nextcloud_upload_url", "Not set"),
"base_url": nextcloud_base_url,
"username": getattr(settings, 'nextcloud_username', 'Not set'),
"password": mask_sensitive_value(getattr(settings, 'nextcloud_password', None)),
"folder": getattr(settings, 'nextcloud_folder', 'Not set')
}
"username": getattr(settings, "nextcloud_username", "Not set"),
"password": mask_sensitive_value(getattr(settings, "nextcloud_password", None)),
"folder": getattr(settings, "nextcloud_folder", "Not set"),
},
}
# Check OneDrive configuration
providers["OneDrive"] = {
"name": "OneDrive",
"name": "OneDrive",
"icon": "fa-brands fa-microsoft",
"configured": bool(getattr(settings, 'onedrive_client_id', None) and
getattr(settings, 'onedrive_client_secret', None) and
getattr(settings, 'onedrive_refresh_token', None)),
"configured": bool(
getattr(settings, "onedrive_client_id", None)
and getattr(settings, "onedrive_client_secret", None)
and getattr(settings, "onedrive_refresh_token", None)
),
"enabled": True,
"description": "Store documents in Microsoft OneDrive",
"details": {
"client_id": getattr(settings, 'onedrive_client_id', 'Not set'),
"client_secret": mask_sensitive_value(getattr(settings, 'onedrive_client_secret', None)),
"tenant_id": getattr(settings, 'onedrive_tenant_id', 'Not set'),
"refresh_token": mask_sensitive_value(getattr(settings, 'onedrive_refresh_token', None)),
"folder": getattr(settings, 'onedrive_folder_path', 'Not set')
}
"client_id": getattr(settings, "onedrive_client_id", "Not set"),
"client_secret": mask_sensitive_value(getattr(settings, "onedrive_client_secret", None)),
"tenant_id": getattr(settings, "onedrive_tenant_id", "Not set"),
"refresh_token": mask_sensitive_value(getattr(settings, "onedrive_refresh_token", None)),
"folder": getattr(settings, "onedrive_folder_path", "Not set"),
},
}
# Check Paperless configuration
providers["Paperless-ngx"] = {
"name": "Paperless-ngx",
"name": "Paperless-ngx",
"icon": "fa-solid fa-file-lines",
"configured": bool(getattr(settings, 'paperless_host', None) and
getattr(settings, 'paperless_ngx_api_token', None)),
"configured": bool(
getattr(settings, "paperless_host", None) and getattr(settings, "paperless_ngx_api_token", None)
),
"enabled": True,
"description": "Document management system for digital archives",
"details": {
"host": getattr(settings, 'paperless_host', 'Not set'),
"api_token": mask_sensitive_value(getattr(settings, 'paperless_ngx_api_token', None))
}
"host": getattr(settings, "paperless_host", "Not set"),
"api_token": mask_sensitive_value(getattr(settings, "paperless_ngx_api_token", None)),
},
}
# Check S3 configuration
providers["S3 Storage"] = {
"name": "S3 Storage",
"name": "S3 Storage",
"icon": "fa-brands fa-aws",
"configured": bool(getattr(settings, 's3_bucket_name', None) and
getattr(settings, 'aws_access_key_id', None) and
getattr(settings, 'aws_secret_access_key', None)),
"configured": bool(
getattr(settings, "s3_bucket_name", None)
and getattr(settings, "aws_access_key_id", None)
and getattr(settings, "aws_secret_access_key", None)
),
"enabled": True,
"description": "Store documents in S3-compatible object storage",
"details": {
"bucket": getattr(settings, 's3_bucket_name', 'Not set'),
"region": getattr(settings, 'aws_region', 'Not set'),
"access_key_id": getattr(settings, 'aws_access_key_id', 'Not set'),
"secret_access_key": mask_sensitive_value(getattr(settings, 'aws_secret_access_key', None)),
"folder_prefix": getattr(settings, 's3_folder_prefix', 'Not set'),
"storage_class": getattr(settings, 's3_storage_class', 'Not set'),
"acl": getattr(settings, 's3_acl', 'Not set')
}
"bucket": getattr(settings, "s3_bucket_name", "Not set"),
"region": getattr(settings, "aws_region", "Not set"),
"access_key_id": getattr(settings, "aws_access_key_id", "Not set"),
"secret_access_key": mask_sensitive_value(getattr(settings, "aws_secret_access_key", None)),
"folder_prefix": getattr(settings, "s3_folder_prefix", "Not set"),
"storage_class": getattr(settings, "s3_storage_class", "Not set"),
"acl": getattr(settings, "s3_acl", "Not set"),
},
}
# Check SFTP configuration
providers["SFTP Storage"] = {
"name": "SFTP Storage",
"name": "SFTP Storage",
"icon": "fa-solid fa-lock",
"configured": bool(getattr(settings, 'sftp_host', None) and
getattr(settings, 'sftp_username', None) and
(getattr(settings, 'sftp_password', None) or
getattr(settings, 'sftp_private_key', None))),
"configured": bool(
getattr(settings, "sftp_host", None)
and getattr(settings, "sftp_username", None)
and (getattr(settings, "sftp_password", None) or getattr(settings, "sftp_private_key", None))
),
"enabled": True,
"description": "Upload files to SFTP server",
"details": {
"host": getattr(settings, 'sftp_host', 'Not set'),
"port": getattr(settings, 'sftp_port', 'Not set'),
"username": getattr(settings, 'sftp_username', 'Not set'),
"password": mask_sensitive_value(getattr(settings, 'sftp_password', None)),
"private_key": getattr(settings, 'sftp_private_key', 'Not set'),
"private_key_passphrase": mask_sensitive_value(getattr(settings, 'sftp_private_key_passphrase', None)),
"folder": getattr(settings, 'sftp_folder', 'Not set')
}
"host": getattr(settings, "sftp_host", "Not set"),
"port": getattr(settings, "sftp_port", "Not set"),
"username": getattr(settings, "sftp_username", "Not set"),
"password": mask_sensitive_value(getattr(settings, "sftp_password", None)),
"private_key": getattr(settings, "sftp_private_key", "Not set"),
"private_key_passphrase": mask_sensitive_value(getattr(settings, "sftp_private_key_passphrase", None)),
"folder": getattr(settings, "sftp_folder", "Not set"),
},
}
# Add Uptime Kuma configuration
providers["Uptime Kuma"] = {
"name": "Uptime Kuma",
"name": "Uptime Kuma",
"icon": "fa-solid fa-heart-pulse",
"configured": bool(getattr(settings, 'uptime_kuma_url', None)),
"configured": bool(getattr(settings, "uptime_kuma_url", None)),
"enabled": True,
"description": "Server monitoring and status page",
"details": {
"url": getattr(settings, 'uptime_kuma_url', 'Not set'),
"ping_interval": getattr(settings, 'uptime_kuma_ping_interval', 'Not set')
}
"url": getattr(settings, "uptime_kuma_url", "Not set"),
"ping_interval": getattr(settings, "uptime_kuma_ping_interval", "Not set"),
},
}
# Check WebDAV configuration
providers["WebDAV"] = {
"name": "WebDAV",
"name": "WebDAV",
"icon": "fa-solid fa-globe",
"configured": bool(getattr(settings, 'webdav_url', None) and
getattr(settings, 'webdav_username', None) and
getattr(settings, 'webdav_password', None)),
"configured": bool(
getattr(settings, "webdav_url", None)
and getattr(settings, "webdav_username", None)
and getattr(settings, "webdav_password", None)
),
"enabled": True,
"description": "Store documents on WebDAV servers",
"details": {
"url": getattr(settings, 'webdav_url', 'Not set'),
"username": getattr(settings, 'webdav_username', 'Not set'),
"password": mask_sensitive_value(getattr(settings, 'webdav_password', None)),
"folder": getattr(settings, 'webdav_folder', 'Not set'),
"verify_ssl": getattr(settings, 'webdav_verify_ssl', 'Not set')
}
"url": getattr(settings, "webdav_url", "Not set"),
"username": getattr(settings, "webdav_username", "Not set"),
"password": mask_sensitive_value(getattr(settings, "webdav_password", None)),
"folder": getattr(settings, "webdav_folder", "Not set"),
"verify_ssl": getattr(settings, "webdav_verify_ssl", "Not set"),
},
}
return providers
app/utils/config_validator/settings_display.py
+82 -92
View File
@@ -3,31 +3,47 @@ Module for displaying and organizing settings information
"""
import logging
from app.config import settings
from app.utils.config_validator.masking import mask_sensitive_value
logger = logging.getLogger(__name__)
def dump_all_settings():
"""Log all settings values for diagnostic purposes"""
logger.info("--- DUMPING ALL SETTINGS FOR DIAGNOSTIC PURPOSES ---")
for key in dir(settings):
if not key.startswith('_') and not callable(getattr(settings, key)):
if not key.startswith("_") and not callable(getattr(settings, key)):
value = getattr(settings, key)
# Mask sensitive values in logs
if key.lower().find('password') >= 0 or key.lower().find('secret') >= 0 or key.lower().find('token') >= 0 or key.lower().find('key') >= 0:
if (
key.lower().find("password") >= 0
or key.lower().find("secret") >= 0
or key.lower().find("token") >= 0
or key.lower().find("key") >= 0
):
if value:
if isinstance(value, str) and len(value) > 10:
visible_start = max(1, len(value) // 3)
visible_end = max(1, len(value) // 4)
value = f"{value[:visible_start]}{'*' * (len(value) - visible_start - visible_end)}{value[-visible_end:]}"
value = (
f"{value[:visible_start]}"
f"{'*' * (len(value) - visible_start - visible_end)}"
f"{value[-visible_end:]}"
)
else:
value = f"{value[:2]}{'*' * (len(value) - 4)}{value[-2:]}" if isinstance(value, str) and len(value) > 4 else "****"
value = (
f"{value[:2]}{'*' * (len(value) - 4)}{value[-2:]}"
if isinstance(value, str) and len(value) > 4
else "****"
)
# Special handling for notification URLs
if key == 'notification_urls' and value:
if key == "notification_urls" and value:
try:
from app.utils.notification import _mask_sensitive_url
if isinstance(value, list):
masked_urls = [_mask_sensitive_url(url) for url in value]
logger.info(f"{key}: {masked_urls}")
@@ -36,44 +52,37 @@ def dump_all_settings():
continue # Skip the default logging
except (ImportError, AttributeError):
pass # Fall back to default logging if _mask_sensitive_url is not available
logger.info(f"{key}: {value}")
logger.info("--- END OF SETTINGS DUMP ---")
def get_settings_for_display(show_values=False):
"""
Group settings into logical categories and check if they are configured.
Returns a dictionary with categories as keys and lists of setting items as values.
Returns a dictionary with categories as keys and lists of setting items as values.
Each setting item is a dict with name, value, and is_configured.
If show_values is False, sensitive values are masked.
"""
# First include system info with version in result
result = {
"System Info": [
{
"name": "App Version",
"value": settings.version,
"is_configured": True
},
{
"name": "Build Date",
"value": settings.build_date,
"is_configured": True
}
{"name": "App Version", "value": settings.version, "is_configured": True},
{"name": "Build Date", "value": settings.build_date, "is_configured": True},
]
}
# Define categories and their settings
categories = {
"Core": [
"debug", # Explicitly include debug setting
"debug", # Explicitly include debug setting
"external_hostname",
"workdir",
"database_url",
"redis_url",
"gotenberg_url",
"allow_file_delete" # Added allow_file_delete to Core settings
"allow_file_delete", # Added allow_file_delete to Core settings
],
"Authentication": [
"auth_enabled",
@@ -83,7 +92,7 @@ def get_settings_for_display(show_values=False):
"authentik_client_id",
"authentik_client_secret",
"authentik_config_url",
"oauth_provider_name"
"oauth_provider_name",
],
"Email": [
"email_host",
@@ -92,7 +101,7 @@ def get_settings_for_display(show_values=False):
"email_password",
"email_use_tls",
"email_sender",
"email_default_recipient"
"email_default_recipient",
],
"IMAP": [
"imap1_host",
@@ -108,47 +117,28 @@ def get_settings_for_display(show_values=False):
"imap2_password",
"imap2_ssl",
"imap2_poll_interval_minutes",
"imap2_delete_after_process"
],
"Dropbox": [
"dropbox_app_key",
"dropbox_app_secret",
"dropbox_folder",
"dropbox_refresh_token"
],
"NextCloud": [
"nextcloud_upload_url",
"nextcloud_username",
"nextcloud_password",
"nextcloud_folder"
],
"Paperless": [
"paperless_host",
"paperless_ngx_api_token"
"imap2_delete_after_process",
],
"Dropbox": ["dropbox_app_key", "dropbox_app_secret", "dropbox_folder", "dropbox_refresh_token"],
"NextCloud": ["nextcloud_upload_url", "nextcloud_username", "nextcloud_password", "nextcloud_folder"],
"Paperless": ["paperless_host", "paperless_ngx_api_token"],
"Google Drive": [
"google_drive_use_oauth",
"google_drive_client_id",
"google_drive_client_secret",
"google_drive_client_secret",
"google_drive_refresh_token",
"google_drive_credentials_json",
"google_drive_folder_id",
"google_drive_delegate_to"
"google_drive_delegate_to",
],
"OneDrive": [
"onedrive_client_id",
"onedrive_client_secret",
"onedrive_tenant_id",
"onedrive_refresh_token",
"onedrive_folder_path"
],
"WebDAV": [
"webdav_url",
"webdav_username",
"webdav_password",
"webdav_folder",
"webdav_verify_ssl"
"onedrive_folder_path",
],
"WebDAV": ["webdav_url", "webdav_username", "webdav_password", "webdav_folder", "webdav_verify_ssl"],
"SFTP": [
"sftp_host",
"sftp_port",
@@ -156,7 +146,7 @@ def get_settings_for_display(show_values=False):
"sftp_password",
"sftp_folder",
"sftp_private_key",
"sftp_private_key_passphrase"
"sftp_private_key_passphrase",
],
"FTP": [
"ftp_host",
@@ -165,7 +155,7 @@ def get_settings_for_display(show_values=False):
"ftp_password",
"ftp_folder",
"ftp_use_tls",
"ftp_allow_plaintext"
"ftp_allow_plaintext",
],
"S3/AWS": [
"aws_access_key_id",
@@ -174,7 +164,7 @@ def get_settings_for_display(show_values=False):
"s3_bucket_name",
"s3_folder_prefix",
"s3_storage_class",
"s3_acl"
"s3_acl",
],
"AI Services": [
"openai_api_key",
@@ -182,84 +172,84 @@ def get_settings_for_display(show_values=False):
"openai_model",
"azure_ai_key",
"azure_endpoint",
"azure_region"
],
"Monitoring": [
"uptime_kuma_url",
"uptime_kuma_ping_interval"
"azure_region",
],
"Monitoring": ["uptime_kuma_url", "uptime_kuma_ping_interval"],
"Notifications": [
"notification_urls",
"notify_on_task_failure",
"notify_on_credential_failure",
"notify_on_credential_failure",
"notify_on_startup",
"notify_on_shutdown"
]
"notify_on_shutdown",
],
}
# Handle any settings that don't fit into the predefined categories
all_settings = set([key for key in dir(settings)
if not key.startswith('_') and
not callable(getattr(settings, key)) and
key not in ["model_computed_fields", "model_config",
"model_extra", "model_fields",
"model_fields_set"]])
all_settings = set(
[
key
for key in dir(settings)
if not key.startswith("_")
and not callable(getattr(settings, key))
and key not in ["model_computed_fields", "model_config", "model_extra", "model_fields", "model_fields_set"]
]
)
# Ensure 'version' is excluded since we display it separately
all_settings.discard("version")
categorized_settings = set()
for cat_settings in categories.values():
categorized_settings.update(cat_settings)
uncategorized = all_settings - categorized_settings
if uncategorized:
categories["Other"] = list(uncategorized)
# Build the result
for category, setting_keys in categories.items():
items = []
for key in setting_keys:
if hasattr(settings, key):
value = getattr(settings, key)
# List of patterns that indicate sensitive values
sensitive_patterns = [
'password', 'secret', 'token', 'api_key', 'private_key',
'credentials', 'access_key', 'ai_key'
"password",
"secret",
"token",
"api_key",
"private_key",
"credentials",
"access_key",
"ai_key",
]
# Check if this is a sensitive value that should be masked
is_sensitive = any(
pattern in key.lower() for pattern in sensitive_patterns
)
is_sensitive = any(pattern in key.lower() for pattern in sensitive_patterns)
# Special handling for "auth" to avoid matching prefixes like "authentik"
if not is_sensitive and "auth" in key.lower():
# Only mark as sensitive if "auth" is a standalone word or at the end
# This avoids matching "authentik" as sensitive
parts = key.lower().split('_')
parts = key.lower().split("_")
is_sensitive = any(part == "auth" for part in parts) or key.lower().endswith("auth")
# Mask sensitive values regardless of debug mode
# Other values are only hidden if debug mode is off AND show_values is False
if (is_sensitive or not show_values) and value:
if is_sensitive:
value = mask_sensitive_value(value)
# Check if the setting is configured (has a non-None value)
# For boolean settings, consider them configured even if False
is_configured = value is not None
if is_configured and isinstance(value, str):
is_configured = len(value) > 0
items.append({
"name": key,
"value": value,
"is_configured": is_configured
})
items.append({"name": key, "value": value, "is_configured": is_configured})
if items: # Only add categories that have items
result[category] = items
return result
app/utils/config_validator/validators.py
+108 -98
View File
@@ -1,24 +1,26 @@
#!/usr/bin/env python3
import logging
import os
import socket
import logging
from app.config import settings
logger = logging.getLogger(__name__)
def validate_email_config():
"""Validates email configuration settings"""
issues = []
# Check for required email settings
if not getattr(settings, 'email_host', None):
if not getattr(settings, "email_host", None):
issues.append("EMAIL_HOST is not configured")
if not getattr(settings, 'email_port', None):
if not getattr(settings, "email_port", None):
issues.append("EMAIL_PORT is not configured")
# Test SMTP server connectivity if host is configured
if getattr(settings, 'email_host', None) and getattr(settings, 'email_port', None):
if getattr(settings, "email_host", None) and getattr(settings, "email_port", None):
try:
# Attempt to resolve the hostname
socket.gethostbyname(settings.email_host)
@@ -26,199 +28,212 @@ def validate_email_config():
issues.append(f"Cannot resolve email host: {settings.email_host}")
# Check for authentication settings
if not getattr(settings, 'email_username', None):
if not getattr(settings, "email_username", None):
issues.append("EMAIL_USERNAME is not configured")
if not getattr(settings, 'email_password', None):
if not getattr(settings, "email_password", None):
issues.append("EMAIL_PASSWORD is not configured")
return issues
def validate_auth_config():
"""Validates authentication configuration settings"""
issues = []
# If auth is enabled, check for required settings
if getattr(settings, 'auth_enabled', False):
if getattr(settings, "auth_enabled", False):
# Check for session secret
if not getattr(settings, 'session_secret', None):
if not getattr(settings, "session_secret", None):
issues.append("SESSION_SECRET is not configured but AUTH_ENABLED is True")
elif len(getattr(settings, 'session_secret', '')) < 32:
elif len(getattr(settings, "session_secret", "")) < 32:
issues.append("SESSION_SECRET must be at least 32 characters long")
# Check if using simple authentication or OIDC
using_simple_auth = bool(getattr(settings, 'admin_username', None) and
getattr(settings, 'admin_password', None))
using_oidc = bool(getattr(settings, 'authentik_client_id', None) and
getattr(settings, 'authentik_client_secret', None) and
getattr(settings, 'authentik_config_url', None))
using_simple_auth = bool(
getattr(settings, "admin_username", None) and getattr(settings, "admin_password", None)
)
using_oidc = bool(
getattr(settings, "authentik_client_id", None)
and getattr(settings, "authentik_client_secret", None)
and getattr(settings, "authentik_config_url", None)
)
if not using_simple_auth and not using_oidc:
issues.append("Neither simple authentication nor OIDC are properly configured")
# If using OIDC, check for provider name
if using_oidc and not getattr(settings, 'oauth_provider_name', None):
if using_oidc and not getattr(settings, "oauth_provider_name", None):
issues.append("OAUTH_PROVIDER_NAME is not configured but OIDC is enabled")
return issues
def validate_storage_configs():
"""Validates configuration for all storage providers"""
issues = {}
# Validate Dropbox config
dropbox_issues = []
if not (getattr(settings, 'dropbox_app_key', None) and
getattr(settings, 'dropbox_app_secret', None) and
getattr(settings, 'dropbox_refresh_token', None)):
if not (
getattr(settings, "dropbox_app_key", None)
and getattr(settings, "dropbox_app_secret", None)
and getattr(settings, "dropbox_refresh_token", None)
):
dropbox_issues.append("Dropbox credentials are not fully configured")
issues['dropbox'] = dropbox_issues
issues["dropbox"] = dropbox_issues
# Validate Nextcloud config
nextcloud_issues = []
if not (getattr(settings, 'nextcloud_upload_url', None) and
getattr(settings, 'nextcloud_username', None) and
getattr(settings, 'nextcloud_password', None)):
if not (
getattr(settings, "nextcloud_upload_url", None)
and getattr(settings, "nextcloud_username", None)
and getattr(settings, "nextcloud_password", None)
):
nextcloud_issues.append("Nextcloud credentials are not fully configured")
issues['nextcloud'] = nextcloud_issues
issues["nextcloud"] = nextcloud_issues
# Validate SFTP config
sftp_issues = []
if not getattr(settings, 'sftp_host', None):
if not getattr(settings, "sftp_host", None):
sftp_issues.append("SFTP_HOST is not configured")
sftp_key_path = getattr(settings, 'sftp_private_key', None)
sftp_key_path = getattr(settings, "sftp_private_key", None)
if sftp_key_path and not os.path.exists(sftp_key_path):
sftp_issues.append(f"SFTP_KEY_PATH file not found: {sftp_key_path}")
if not sftp_key_path and not getattr(settings, 'sftp_password', None):
if not sftp_key_path and not getattr(settings, "sftp_password", None):
sftp_issues.append("Neither SFTP_KEY_PATH nor SFTP_PASSWORD is configured")
issues['sftp'] = sftp_issues
issues["sftp"] = sftp_issues
# Validate Email sending
email_issues = []
if not getattr(settings, 'email_host', None):
if not getattr(settings, "email_host", None):
email_issues.append("EMAIL_HOST is not configured")
if not getattr(settings, 'email_default_recipient', None):
if not getattr(settings, "email_default_recipient", None):
email_issues.append("EMAIL_DEFAULT_RECIPIENT is not configured")
issues['email'] = email_issues
issues["email"] = email_issues
# Validate S3
s3_issues = []
if not getattr(settings, 's3_bucket_name', None):
if not getattr(settings, "s3_bucket_name", None):
s3_issues.append("S3_BUCKET_NAME is not configured")
if not (getattr(settings, 'aws_access_key_id', None) and
getattr(settings, 'aws_secret_access_key', None)):
if not (getattr(settings, "aws_access_key_id", None) and getattr(settings, "aws_secret_access_key", None)):
s3_issues.append("AWS credentials are not configured")
issues['s3'] = s3_issues
issues["s3"] = s3_issues
# Validate FTP
ftp_issues = []
if not getattr(settings, 'ftp_host', None):
if not getattr(settings, "ftp_host", None):
ftp_issues.append("FTP_HOST is not configured")
if not getattr(settings, 'ftp_username', None):
if not getattr(settings, "ftp_username", None):
ftp_issues.append("FTP_USERNAME is not configured")
if not getattr(settings, 'ftp_password', None):
if not getattr(settings, "ftp_password", None):
ftp_issues.append("FTP_PASSWORD is not configured")
issues['ftp'] = ftp_issues
issues["ftp"] = ftp_issues
# Validate WebDAV
webdav_issues = []
if not getattr(settings, 'webdav_url', None):
if not getattr(settings, "webdav_url", None):
webdav_issues.append("WEBDAV_URL is not configured")
if not getattr(settings, 'webdav_username', None):
if not getattr(settings, "webdav_username", None):
webdav_issues.append("WEBDAV_USERNAME is not configured")
if not getattr(settings, 'webdav_password', None):
if not getattr(settings, "webdav_password", None):
webdav_issues.append("WEBDAV_PASSWORD is not configured")
issues['webdav'] = webdav_issues
issues["webdav"] = webdav_issues
# Validate Google Drive
gdrive_issues = []
if not getattr(settings, 'google_drive_credentials_json', None):
if not getattr(settings, "google_drive_credentials_json", None):
gdrive_issues.append("GOOGLE_DRIVE_CREDENTIALS_JSON is not configured")
if not getattr(settings, 'google_drive_folder_id', None):
if not getattr(settings, "google_drive_folder_id", None):
gdrive_issues.append("GOOGLE_DRIVE_FOLDER_ID is not configured")
issues['google_drive'] = gdrive_issues
issues["google_drive"] = gdrive_issues
# Validate Paperless
paperless_issues = []
if not getattr(settings, 'paperless_host', None):
if not getattr(settings, "paperless_host", None):
paperless_issues.append("PAPERLESS_HOST is not configured")
if not getattr(settings, 'paperless_ngx_api_token', None):
if not getattr(settings, "paperless_ngx_api_token", None):
paperless_issues.append("PAPERLESS_NGX_API_TOKEN is not configured")
issues['paperless'] = paperless_issues
issues["paperless"] = paperless_issues
# Validate OneDrive
onedrive_issues = []
if not (getattr(settings, 'onedrive_client_id', None) and
getattr(settings, 'onedrive_client_secret', None) and
getattr(settings, 'onedrive_refresh_token', None)):
if not (
getattr(settings, "onedrive_client_id", None)
and getattr(settings, "onedrive_client_secret", None)
and getattr(settings, "onedrive_refresh_token", None)
):
onedrive_issues.append("OneDrive credentials are not fully configured")
issues['onedrive'] = onedrive_issues
issues["onedrive"] = onedrive_issues
# Validate Uptime Kuma
uptime_kuma_issues = []
if not getattr(settings, 'uptime_kuma_url', None):
if not getattr(settings, "uptime_kuma_url", None):
uptime_kuma_issues.append("UPTIME_KUMA_URL is not configured")
issues['uptime_kuma'] = uptime_kuma_issues
issues["uptime_kuma"] = uptime_kuma_issues
return issues
def validate_notification_config():
"""Check notification configuration"""
issues = []
# Check if any notification URLs are configured
if not getattr(settings, 'notification_urls', None):
if not getattr(settings, "notification_urls", None):
issues.append("No notification URLs configured")
else:
try:
# Try initializing Apprise to validate URLs
import apprise
a = apprise.Apprise()
for url in settings.notification_urls:
try:
if not a.add(url):
issues.append(f"Invalid notification URL format: {url}")
except Exception as e:
issues.append(f"Error with notification URL: {str(e)}")
except ImportError:
issues.append("Apprise module not installed")
if not issues:
logger.info("Notification configuration valid")
else:
logger.warning(f"Notification configuration issues: {', '.join(issues)}")
return issues
def check_all_configs():
"""Run all configuration validations and log results"""
from app.utils.config_validator.settings_display import dump_all_settings
logger.info("Validating application configuration...")
# Check if debug is enabled and dump all settings if it is
if hasattr(settings, 'debug') and settings.debug:
if hasattr(settings, "debug") and settings.debug:
dump_all_settings()
# Check auth config
auth_issues = validate_auth_config()
if auth_issues:
logger.warning(f"Authentication configuration issues: {', '.join(auth_issues)}")
else:
logger.info("Authentication configuration OK")
# Check email config
email_issues = validate_email_config()
if email_issues:
logger.warning(f"Email configuration issues: {', '.join(email_issues)}")
else:
logger.info("Email configuration OK")
# Check storage configs
storage_issues = validate_storage_configs()
for provider, issues in storage_issues.items():
@@ -226,18 +241,13 @@ def check_all_configs():
logger.warning(f"{provider.capitalize()} configuration issues: {', '.join(issues)}")
else:
logger.info(f"{provider.capitalize()} configuration OK")
# Check notification configuration
notification_issues = validate_notification_config()
if notification_issues:
logger.warning(f"Notification configuration issues: {', '.join(notification_issues)}")
else:
logger.info("Notification configuration OK")
# Return all identified issues
return {
'auth': auth_issues,
'email': email_issues,
'storage': storage_issues,
'notification': notification_issues
}
return {"auth": auth_issues, "email": email_issues, "storage": storage_issues, "notification": notification_issues}
app/utils/encryption.py
+29 -28
View File
@@ -5,9 +5,9 @@ Uses Fernet symmetric encryption with a key derived from SESSION_SECRET.
This provides encryption at rest for sensitive configuration values.
"""
import logging
import base64
import hashlib
import logging
from typing import Optional
logger = logging.getLogger(__name__)
@@ -19,33 +19,34 @@ _cipher_suite = None
def _get_cipher_suite():
"""
Get or create the Fernet cipher suite for encryption/decryption.
The encryption key is derived from SESSION_SECRET to ensure:
1. Settings are encrypted at rest in the database
2. The same key is used across app restarts
3. No additional secret management needed
Returns:
Fernet cipher suite instance
"""
global _cipher_suite
if _cipher_suite is None:
try:
from cryptography.fernet import Fernet
from app.config import settings
# Derive a Fernet-compatible key from SESSION_SECRET
# Fernet requires a 32-byte base64-encoded key
secret = settings.session_secret.encode('utf-8')
secret = settings.session_secret.encode("utf-8")
# Use SHA256 to get exactly 32 bytes, then base64 encode
key_bytes = hashlib.sha256(secret).digest()
fernet_key = base64.urlsafe_b64encode(key_bytes)
_cipher_suite = Fernet(fernet_key)
logger.debug("Encryption cipher suite initialized")
except ImportError:
logger.warning(
"cryptography library not installed. "
@@ -56,34 +57,34 @@ def _get_cipher_suite():
except Exception as e:
logger.error(f"Failed to initialize encryption: {e}")
_cipher_suite = None
return _cipher_suite
def encrypt_value(plaintext: Optional[str]) -> Optional[str]:
"""
Encrypt a plaintext value for storage in the database.
Args:
plaintext: The value to encrypt (or None)
Returns:
Encrypted value as base64 string, or plaintext if encryption unavailable
"""
if plaintext is None or plaintext == "":
return plaintext
cipher = _get_cipher_suite()
if cipher is None:
# Encryption not available, store in plaintext with warning
logger.warning("Storing sensitive value in plaintext (encryption unavailable)")
return plaintext
try:
encrypted_bytes = cipher.encrypt(plaintext.encode('utf-8'))
encrypted_bytes = cipher.encrypt(plaintext.encode("utf-8"))
# Prefix with "enc:" to identify encrypted values
return "enc:" + encrypted_bytes.decode('utf-8')
return "enc:" + encrypted_bytes.decode("utf-8")
except Exception as e:
logger.error(f"Encryption failed: {e}")
# Fall back to plaintext
@@ -93,32 +94,32 @@ def encrypt_value(plaintext: Optional[str]) -> Optional[str]:
def decrypt_value(ciphertext: Optional[str]) -> Optional[str]:
"""
Decrypt a value from the database.
Args:
ciphertext: The encrypted value (or plaintext if not encrypted)
Returns:
Decrypted plaintext value
"""
if ciphertext is None or ciphertext == "":
return ciphertext
# Check if value is encrypted (has "enc:" prefix)
if not ciphertext.startswith("enc:"):
# Not encrypted, return as-is
return ciphertext
cipher = _get_cipher_suite()
if cipher is None:
logger.error("Cannot decrypt value: encryption not available")
return "[ENCRYPTED - Cannot decrypt]"
try:
# Remove "enc:" prefix and decrypt
encrypted_bytes = ciphertext[4:].encode('utf-8')
encrypted_bytes = ciphertext[4:].encode("utf-8")
plaintext_bytes = cipher.decrypt(encrypted_bytes)
return plaintext_bytes.decode('utf-8')
return plaintext_bytes.decode("utf-8")
except Exception as e:
logger.error(f"Decryption failed: {e}")
return "[DECRYPTION FAILED]"
@@ -127,10 +128,10 @@ def decrypt_value(ciphertext: Optional[str]) -> Optional[str]:
def is_encrypted(value: Optional[str]) -> bool:
"""
Check if a value is encrypted.
Args:
value: The value to check
Returns:
True if the value is encrypted, False otherwise
"""
@@ -140,7 +141,7 @@ def is_encrypted(value: Optional[str]) -> bool:
def is_encryption_available() -> bool:
"""
Check if encryption is available.
Returns:
True if cryptography library is installed and encryption is working
"""
app/utils/file_operations.py
+1
View File
@@ -1,5 +1,6 @@
import hashlib
def hash_file(filepath, chunk_size=65536):
"""
Returns the SHA-256 hash of the file at 'filepath'.
app/utils/file_status.py
+29 -33
View File
@@ -1,89 +1,90 @@
"""
Utility functions for file processing status determination.
"""
from typing import Dict, List
from sqlalchemy.orm import Session
from app.models import ProcessingLog
def get_file_processing_status(db: Session, file_id: int) -> Dict:
"""
Get the processing status for a file by checking its processing logs.
Args:
db: Database session
file_id: ID of the file
Returns:
dict with status, last_step, and has_errors
"""
# Get all logs for this file
logs = db.query(ProcessingLog).filter(
ProcessingLog.file_id == file_id
).order_by(ProcessingLog.timestamp.desc()).all()
logs = (
db.query(ProcessingLog).filter(ProcessingLog.file_id == file_id).order_by(ProcessingLog.timestamp.desc()).all()
)
return _compute_status_from_logs(logs)
def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, Dict]:
"""
Get processing status for multiple files efficiently.
Args:
db: Database session
file_ids: List of file IDs
Returns:
dict mapping file_id to status dict
"""
# Get all logs for these files in one query
logs = db.query(ProcessingLog).filter(
ProcessingLog.file_id.in_(file_ids)
).order_by(ProcessingLog.file_id, ProcessingLog.timestamp.desc()).all()
logs = (
db.query(ProcessingLog)
.filter(ProcessingLog.file_id.in_(file_ids))
.order_by(ProcessingLog.file_id, ProcessingLog.timestamp.desc())
.all()
)
# Group logs by file_id
logs_by_file = {}
for log in logs:
if log.file_id not in logs_by_file:
logs_by_file[log.file_id] = []
logs_by_file[log.file_id].append(log)
# Compute status for each file
result = {}
for file_id in file_ids:
file_logs = logs_by_file.get(file_id, [])
result[file_id] = _compute_status_from_logs(file_logs)
return result
def _compute_status_from_logs(logs: List[ProcessingLog]) -> Dict:
"""
Compute processing status from a list of processing logs.
Args:
logs: List of ProcessingLog objects (should be ordered by timestamp desc)
Returns:
dict with status, last_step, has_errors, and total_steps
"""
if not logs:
return {
"status": "pending",
"last_step": None,
"has_errors": False,
"total_steps": 0
}
return {"status": "pending", "last_step": None, "has_errors": False, "total_steps": 0}
# Check for failures
has_errors = any(log.status == "failure" for log in logs)
# Check if any in progress
in_progress = any(log.status == "in_progress" for log in logs)
# Get the latest log
latest_log = logs[0]
# Determine overall status
if has_errors:
status = "failed"
@@ -93,10 +94,5 @@ def _compute_status_from_logs(logs: List[ProcessingLog]) -> Dict:
status = "completed"
else:
status = "pending"
return {
"status": status,
"last_step": latest_log.step_name,
"has_errors": has_errors,
"total_steps": len(logs)
}
return {"status": status, "last_step": latest_log.step_name, "has_errors": has_errors, "total_steps": len(logs)}
app/utils/filename_utils.py
+36 -33
View File
@@ -1,55 +1,56 @@
import logging
import os
import re
import uuid
import logging
from datetime import datetime
from pathlib import Path
logger = logging.getLogger(__name__)
def get_unique_filename(original_path, check_exists_func=None):
"""
Generates a unique filename by appending a timestamp or counter when a collision occurs.
Args:
original_path (str): The original file path
check_exists_func (callable): Function that checks if file exists in target system.
Takes a path string and returns True if exists, False otherwise.
If None, will use local filesystem check.
Returns:
str: A unique filename that doesn't collide with existing files
"""
if check_exists_func is None:
check_exists_func = os.path.exists
path = Path(original_path)
directory = str(path.parent)
filename = path.name
name, ext = os.path.splitext(filename)
# If file doesn't exist, return the original
if not check_exists_func(original_path):
return original_path
# Try timestamp-based suffix first (more user-friendly)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
new_filename = f"{name}_{timestamp}{ext}"
new_path = os.path.join(directory, new_filename)
if not check_exists_func(new_path):
logger.info(f"Renamed '{filename}' to '{new_filename}' to avoid collision")
return new_path
# If timestamp-based name also exists, try random UUID
uuid_str = str(uuid.uuid4())[:8] # Use first 8 chars of UUID for brevity
new_filename = f"{name}_{uuid_str}{ext}"
new_path = os.path.join(directory, new_filename)
if not check_exists_func(new_path):
logger.info(f"Renamed '{filename}' to '{new_filename}' using UUID to avoid collision")
return new_path
# If that still exists (very unlikely), use incremental numbering
counter = 1
while counter < 1000: # Limit to avoid infinite loop
@@ -59,77 +60,79 @@ def get_unique_filename(original_path, check_exists_func=None):
logger.info(f"Renamed '{filename}' to '{new_filename}' using counter to avoid collision")
return new_path
counter += 1
# If we got here, something is weird - just use a full UUID
new_filename = f"{name}_{str(uuid.uuid4())}{ext}"
new_path = os.path.join(directory, new_filename)
logger.warning(f"Had to use full UUID to rename '{filename}' to '{new_filename}'")
return new_path
def sanitize_filename(filename):
"""
Sanitize a filename to ensure it's valid across different file systems.
Args:
filename (str): The filename to sanitize
Returns:
str: A sanitized filename
"""
# Replace characters that are problematic in various filesystems
# Keep only alphanumeric, dash, underscore, period, and space
sanitized = re.sub(r'[^\w\-\. ]', '_', filename)
sanitized = re.sub(r"[^\w\-\. ]", "_", filename)
# Replace multiple spaces/underscores with single ones
sanitized = re.sub(r'__+', '_', sanitized)
sanitized = re.sub(r' +', ' ', sanitized)
sanitized = re.sub(r"__+", "_", sanitized)
sanitized = re.sub(r" +", " ", sanitized)
# Trim leading/trailing spaces and periods which cause issues in Windows
sanitized = sanitized.strip('. ')
sanitized = sanitized.strip(". ")
# Ensure the filename isn't empty after sanitization
if not sanitized or sanitized == '.':
if not sanitized or sanitized == ".":
sanitized = f"document_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
return sanitized
def extract_remote_path(file_path, base_dir, remote_base=""):
"""
Extract a remote path for a file by preserving its directory structure
relative to the base directory, but with a new remote base path.
Modified to skip 'processed' directory in the remote path.
"""
# Normalize paths for consistent handling across platforms
file_path = os.path.normpath(file_path)
base_dir = os.path.normpath(base_dir)
# Get relative path from base directory
if file_path.startswith(base_dir):
rel_path = os.path.relpath(file_path, base_dir)
else:
# If not a subdirectory of base_dir, just use the filename
rel_path = os.path.basename(file_path)
# Skip 'processed' directory if it's in the path
path_parts = rel_path.split(os.sep)
if 'processed' in path_parts:
if "processed" in path_parts:
# Remove 'processed' from the path
path_parts.remove('processed')
path_parts.remove("processed")
rel_path = os.path.join(*path_parts)
# Combine with remote base path
if remote_base:
if remote_base.startswith('/'):
if remote_base.startswith("/"):
# Handle absolute path for services like Dropbox
remote_path = os.path.join(remote_base[1:], rel_path)
else:
remote_path = os.path.join(remote_base, rel_path)
else:
remote_path = rel_path
# Convert to forward slashes for compatibility with most cloud services
remote_path = remote_path.replace(os.sep, '/')
remote_path = remote_path.replace(os.sep, "/")
return remote_path
app/utils/logging.py
+1
View File
@@ -1,6 +1,7 @@
from app.database import SessionLocal
from app.models import ProcessingLog
def log_task_progress(task_id, step_name, status, message=None, file_id=None):
"""
Logs the progress of a Celery task to the database.
app/utils/notification.py
+55 -69
View File
@@ -1,6 +1,7 @@
import apprise
import logging
from typing import List, Optional, Dict, Any, Union
from typing import Any, Dict, List, Optional
import apprise
from app.config import settings
@@ -9,13 +10,14 @@ logger = logging.getLogger(__name__)
# Global Apprise instance
_apprise = None
def init_apprise() -> apprise.Apprise:
"""Initialize the Apprise instance with configured notification services"""
global _apprise
if _apprise is None:
_apprise = apprise.Apprise()
# Add all configured notification services
if settings.notification_urls:
for url in settings.notification_urls:
@@ -26,31 +28,34 @@ def init_apprise() -> apprise.Apprise:
logger.error(f"Failed to add notification service: {str(e)}")
else:
logger.warning("No notification services configured")
return _apprise
def _mask_sensitive_url(url: str) -> str:
"""Mask sensitive parts of notification URLs for logging"""
# Simple masking for common URL formats with credentials
import re
# Match patterns like user:pass@host or token in URL parameters
masked = re.sub(r'://([^:]+):([^@]+)@', r'://\1:****@', url)
masked = re.sub(r'(discord://)[^/]+/[^/]+', r'\1webhook_id/****', masked)
masked = re.sub(r'(tgram://)[^/]+/[^/]+', r'\1bot_token/****', masked)
masked = re.sub(r'([?&](token|key|api_key|password|secret)=)([^&]+)', r'\1****', masked)
masked = re.sub(r"://([^:]+):([^@]+)@", r"://\1:****@", url)
masked = re.sub(r"(discord://)[^/]+/[^/]+", r"\1webhook_id/****", masked)
masked = re.sub(r"(tgram://)[^/]+/[^/]+", r"\1bot_token/****", masked)
masked = re.sub(r"([?&](token|key|api_key|password|secret)=)([^&]+)", r"\1****", masked)
return masked
def send_notification(
title: str,
message: str,
title: str,
message: str,
notification_type: str = "info",
tags: Optional[List[str]] = None,
attachments: Optional[List[str]] = None,
data: Optional[Dict[str, Any]] = None
data: Optional[Dict[str, Any]] = None,
) -> bool:
"""
Send a notification through all configured channels
Args:
title: The notification title
message: The notification body message
@@ -58,17 +63,17 @@ def send_notification(
tags: Optional list of tags for filtering notifications
attachments: Optional list of file paths to attach
data: Optional additional data for the notification
Returns:
bool: True if notification was sent successfully to at least one service
"""
if not settings.notification_urls:
logger.debug(f"Notification not sent (no services configured): {title}")
return False
try:
apprise_obj = init_apprise()
# Set notification type
notify_type = apprise.NotifyType.INFO
if notification_type == "success":
@@ -77,25 +82,20 @@ def send_notification(
notify_type = apprise.NotifyType.WARNING
elif notification_type in ("failure", "error", "failed"):
notify_type = apprise.NotifyType.FAILURE
# Send the notification to each service individually for better error reporting
if not apprise_obj.servers: # Access servers as an attribute, not a method
logger.warning("No notification servers available despite having URLs configured")
return False
total_services = len(apprise_obj.servers)
successful_services = 0
for server in apprise_obj.servers: # Iterate through the list directly
try:
service_name = str(server).split("://")[0] if "://" in str(server) else str(server)
service_result = server.notify(
title=title,
body=message,
notify_type=notify_type,
attach=attachments
)
service_result = server.notify(title=title, body=message, notify_type=notify_type, attach=attachments)
if service_result:
successful_services += 1
logger.debug(f"Notification sent via {service_name}")
@@ -103,25 +103,26 @@ def send_notification(
logger.warning(f"Failed to send notification via {service_name}")
except Exception as e:
logger.error(f"Error sending notification via {str(server)}: {str(e)}")
overall_result = successful_services > 0
if overall_result:
logger.debug(f"Notification sent: '{title}' (successful: {successful_services}/{total_services})")
else:
logger.warning(f"Failed to send notification to ALL services: '{title}' (0/{total_services})")
return overall_result
except Exception as e:
logger.exception(f"Error sending notification: {e}")
return False
def notify_celery_failure(task_name: str, task_id: str, exc: Exception, args: list, kwargs: dict) -> bool:
"""Send a notification about a failed Celery task"""
if not settings.notify_on_task_failure:
return False
title = f"Task Failed: {task_name}"
message = f"""
Task {task_name} ({task_id}) failed with error:
@@ -131,17 +132,15 @@ Arguments: {args}
Keyword arguments: {kwargs}
"""
return send_notification(
title=title,
message=message,
notification_type="failure",
tags=["celery", "failure", task_name]
title=title, message=message, notification_type="failure", tags=["celery", "failure", task_name]
)
def notify_credential_failure(service_name: str, error: str) -> bool:
"""Send a notification about a credential failure"""
if not settings.notify_on_credential_failure:
return False
title = f"Credential Failure: {service_name}"
message = f"""
The credentials for {service_name} have failed:
@@ -150,57 +149,47 @@ The credentials for {service_name} have failed:
Please check and update the credentials in the system settings.
"""
return send_notification(
title=title,
message=message,
notification_type="warning",
tags=["credentials", "warning", service_name]
title=title, message=message, notification_type="warning", tags=["credentials", "warning", service_name]
)
def notify_startup() -> bool:
"""Send a notification that the application has started"""
if not settings.notify_on_startup:
return False
title = f"DocuElevate Started"
title = "DocuElevate Started"
message = f"DocuElevate has been started successfully on {settings.external_hostname}"
return send_notification(
title=title,
message=message,
notification_type="success",
tags=["system", "startup"]
)
return send_notification(title=title, message=message, notification_type="success", tags=["system", "startup"])
def notify_shutdown() -> bool:
"""Send a notification that the application is shutting down"""
if not settings.notify_on_shutdown:
return False
title = f"DocuElevate Shutting Down"
title = "DocuElevate Shutting Down"
message = f"DocuElevate on {settings.external_hostname} is shutting down"
return send_notification(
title=title,
message=message,
notification_type="info",
tags=["system", "shutdown"]
)
return send_notification(title=title, message=message, notification_type="info", tags=["system", "shutdown"])
def notify_file_processed(filename: str, file_size: int, metadata: dict, destinations: list) -> bool:
"""Send a notification that a file has been successfully processed"""
if not settings.notify_on_file_processed:
return False
# Format file size for display
size_mb = file_size / (1024 * 1024)
size_str = f"{size_mb:.2f} MB" if size_mb >= 1 else f"{file_size / 1024:.2f} KB"
# Extract key metadata fields
doc_type = metadata.get('document_type', 'Unknown')
tags = metadata.get('tags', [])
tags_str = ', '.join(tags) if tags else 'None'
doc_type = metadata.get("document_type", "Unknown")
tags = metadata.get("tags", [])
tags_str = ", ".join(tags) if tags else "None"
# Format destinations
destinations_str = ', '.join(destinations) if destinations else 'None configured'
destinations_str = ", ".join(destinations) if destinations else "None configured"
title = f"File Processed: {filename}"
message = f"""
File: {filename}
@@ -211,10 +200,7 @@ Destinations: {destinations_str}
The file has been successfully processed and is being uploaded to all configured destinations.
"""
return send_notification(
title=title,
message=message.strip(),
notification_type="success",
tags=["document", "processed", "success"]
title=title, message=message.strip(), notification_type="success", tags=["document", "processed", "success"]
)
app/utils/oauth_helper.py
+2 -1
View File
@@ -4,7 +4,8 @@ Shared across multiple OAuth providers to reduce code duplication.
"""
import logging
from typing import Dict, Any, Optional
from typing import Any, Dict, Optional
import requests
from fastapi import HTTPException, status
app/utils/settings_service.py
+43 -55
View File
@@ -9,8 +9,9 @@ This module provides functionality to:
import logging
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.orm import Session
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import Session
from app.models import ApplicationSettings
@@ -67,7 +68,6 @@ SETTING_METADATA = {
"required": True,
"restart_required": True,
},
# Authentication Settings
"auth_enabled": {
"category": "Authentication",
@@ -133,7 +133,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
# AI Services
"openai_api_key": {
"category": "AI Services",
@@ -183,7 +182,6 @@ SETTING_METADATA = {
"required": True,
"restart_required": False,
},
# Storage Providers - Dropbox
"dropbox_app_key": {
"category": "Storage Providers",
@@ -217,7 +215,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - Nextcloud
"nextcloud_upload_url": {
"category": "Storage Providers",
@@ -251,7 +248,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - Paperless-ngx
"paperless_ngx_api_token": {
"category": "Storage Providers",
@@ -269,7 +265,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - Google Drive
"google_drive_credentials_json": {
"category": "Storage Providers",
@@ -327,7 +322,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - OneDrive
"onedrive_client_id": {
"category": "Storage Providers",
@@ -369,7 +363,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - WebDAV
"webdav_url": {
"category": "Storage Providers",
@@ -411,7 +404,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - FTP
"ftp_host": {
"category": "Storage Providers",
@@ -469,7 +461,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - SFTP
"sftp_host": {
"category": "Storage Providers",
@@ -535,7 +526,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - AWS S3
"aws_access_key_id": {
"category": "Storage Providers",
@@ -593,7 +583,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Email Settings
"email_host": {
"category": "Email",
@@ -651,7 +640,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# IMAP Settings - Account 1
"imap1_host": {
"category": "IMAP",
@@ -709,7 +697,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# IMAP Settings - Account 2
"imap2_host": {
"category": "IMAP",
@@ -767,7 +754,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Monitoring - Uptime Kuma
"uptime_kuma_url": {
"category": "Monitoring",
@@ -785,7 +771,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Processing Settings
"http_request_timeout": {
"category": "Processing",
@@ -811,7 +796,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Notifications Settings
"notification_urls": {
"category": "Notifications",
@@ -861,7 +845,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Feature Flags
"allow_file_delete": {
"category": "Feature Flags",
@@ -877,13 +860,13 @@ SETTING_METADATA = {
def get_setting_from_db(db: Session, key: str) -> Optional[str]:
"""
Retrieve a setting value from the database.
Automatically decrypts sensitive values if encryption is enabled.
Args:
db: Database session
key: Setting key to retrieve
Returns:
Setting value as string (decrypted if necessary), or None if not found
"""
@@ -891,13 +874,14 @@ def get_setting_from_db(db: Session, key: str) -> Optional[str]:
setting = db.query(ApplicationSettings).filter(ApplicationSettings.key == key).first()
if not setting:
return None
# Check if this setting is sensitive and should be decrypted
metadata = get_setting_metadata(key)
if metadata.get("sensitive", False):
from app.utils.encryption import decrypt_value
return decrypt_value(setting.value)
return setting.value
except SQLAlchemyError as e:
logger.error(f"Error retrieving setting {key} from database: {e}")
@@ -907,14 +891,14 @@ def get_setting_from_db(db: Session, key: str) -> Optional[str]:
def save_setting_to_db(db: Session, key: str, value: Optional[str]) -> bool:
"""
Save or update a setting in the database.
Automatically encrypts sensitive values if encryption is enabled.
Args:
db: Database session
key: Setting key
value: Setting value (as string)
Returns:
True if successful, False otherwise
"""
@@ -922,16 +906,16 @@ def save_setting_to_db(db: Session, key: str, value: Optional[str]) -> bool:
# Check if this setting is sensitive and should be encrypted
metadata = get_setting_metadata(key)
storage_value = value
if metadata.get("sensitive", False) and value:
from app.utils.encryption import encrypt_value, is_encryption_available
if is_encryption_available():
storage_value = encrypt_value(value)
logger.debug(f"Encrypted sensitive setting: {key}")
else:
logger.warning(f"Storing sensitive setting {key} in plaintext (encryption unavailable)")
setting = db.query(ApplicationSettings).filter(ApplicationSettings.key == key).first()
if setting:
setting.value = storage_value
@@ -950,28 +934,29 @@ def save_setting_to_db(db: Session, key: str, value: Optional[str]) -> bool:
def get_all_settings_from_db(db: Session) -> Dict[str, str]:
"""
Retrieve all settings from the database.
Automatically decrypts sensitive values if encryption is enabled.
Args:
db: Database session
Returns:
Dictionary of setting key-value pairs (decrypted)
"""
try:
settings = db.query(ApplicationSettings).all()
result = {}
for setting in settings:
# Check if this setting is sensitive and should be decrypted
metadata = get_setting_metadata(setting.key)
if metadata.get("sensitive", False):
from app.utils.encryption import decrypt_value
result[setting.key] = decrypt_value(setting.value)
else:
result[setting.key] = setting.value
return result
except SQLAlchemyError as e:
logger.error(f"Error retrieving all settings from database: {e}")
@@ -981,11 +966,11 @@ def get_all_settings_from_db(db: Session) -> Dict[str, str]:
def delete_setting_from_db(db: Session, key: str) -> bool:
"""
Delete a setting from the database.
Args:
db: Database session
key: Setting key to delete
Returns:
True if successful, False otherwise
"""
@@ -1006,27 +991,30 @@ def delete_setting_from_db(db: Session, key: str) -> bool:
def get_setting_metadata(key: str) -> Dict[str, Any]:
"""
Get metadata for a specific setting.
Args:
key: Setting key
Returns:
Dictionary containing setting metadata
"""
return SETTING_METADATA.get(key, {
"category": "Other",
"description": f"Setting: {key}",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
})
return SETTING_METADATA.get(
key,
{
"category": "Other",
"description": f"Setting: {key}",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
)
def get_settings_by_category() -> Dict[str, List[str]]:
"""
Get settings organized by category.
Returns:
Dictionary mapping category names to lists of setting keys
"""
@@ -1042,34 +1030,34 @@ def get_settings_by_category() -> Dict[str, List[str]]:
def validate_setting_value(key: str, value: str) -> Tuple[bool, Optional[str]]:
"""
Validate a setting value based on its metadata.
Args:
key: Setting key
value: Setting value to validate
Returns:
Tuple of (is_valid, error_message)
"""
metadata = get_setting_metadata(key)
setting_type = metadata.get("type", "string")
# Check required fields
if metadata.get("required", False) and not value:
return False, f"{key} is required"
# Type-specific validation
if setting_type == "boolean":
if value.lower() not in ["true", "false", "1", "0", "yes", "no"]:
return False, f"{key} must be a boolean value (true/false)"
elif setting_type == "integer":
try:
int(value)
except ValueError:
return False, f"{key} must be an integer"
# Special validation for specific keys
if key == "session_secret" and value and len(value) < 32:
return False, "session_secret must be at least 32 characters"
return True, None
app/utils/setup_wizard.py
+30 -28
View File
@@ -5,7 +5,8 @@ Detects if the system needs initial setup and provides required settings list.
"""
import logging
from typing import List, Dict, Any
from typing import Any, Dict, List
from app.config import settings
logger = logging.getLogger(__name__)
@@ -14,7 +15,7 @@ logger = logging.getLogger(__name__)
def get_required_settings() -> List[Dict[str, Any]]:
"""
Get list of settings that are absolutely required for the system to operate.
Returns:
List of required setting definitions with metadata
"""
@@ -27,7 +28,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": "sqlite:///./app/database.db",
"wizard_step": 1,
"wizard_category": "Core Infrastructure"
"wizard_category": "Core Infrastructure",
},
{
"key": "redis_url",
@@ -37,7 +38,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": "redis://localhost:6379/0",
"wizard_step": 1,
"wizard_category": "Core Infrastructure"
"wizard_category": "Core Infrastructure",
},
{
"key": "workdir",
@@ -47,7 +48,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": "/workdir",
"wizard_step": 1,
"wizard_category": "Core Infrastructure"
"wizard_category": "Core Infrastructure",
},
{
"key": "gotenberg_url",
@@ -57,7 +58,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": "http://gotenberg:3000",
"wizard_step": 1,
"wizard_category": "Core Infrastructure"
"wizard_category": "Core Infrastructure",
},
{
"key": "session_secret",
@@ -67,7 +68,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": True,
"default": None, # Should be generated
"wizard_step": 2,
"wizard_category": "Security"
"wizard_category": "Security",
},
{
"key": "admin_username",
@@ -77,7 +78,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": "admin",
"wizard_step": 2,
"wizard_category": "Security"
"wizard_category": "Security",
},
{
"key": "admin_password",
@@ -87,7 +88,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": True,
"default": None, # Must be set
"wizard_step": 2,
"wizard_category": "Security"
"wizard_category": "Security",
},
{
"key": "openai_api_key",
@@ -97,7 +98,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": True,
"default": None,
"wizard_step": 3,
"wizard_category": "AI Services"
"wizard_category": "AI Services",
},
{
"key": "azure_ai_key",
@@ -107,7 +108,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": True,
"default": None,
"wizard_step": 3,
"wizard_category": "AI Services"
"wizard_category": "AI Services",
},
{
"key": "azure_region",
@@ -117,7 +118,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": "eastus",
"wizard_step": 3,
"wizard_category": "AI Services"
"wizard_category": "AI Services",
},
{
"key": "azure_endpoint",
@@ -127,7 +128,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": None,
"wizard_step": 3,
"wizard_category": "AI Services"
"wizard_category": "AI Services",
},
]
@@ -135,9 +136,9 @@ def get_required_settings() -> List[Dict[str, Any]]:
def is_setup_required() -> bool:
"""
Check if the system requires initial setup.
Returns True if any critical required settings are missing or have placeholder values.
Returns:
True if setup wizard should be shown, False otherwise
"""
@@ -149,16 +150,16 @@ def is_setup_required() -> bool:
("openai_api_key", [None, "", "<OPENAI_API_KEY>", "test-key"]),
("azure_ai_key", [None, "", "<AZURE_AI_KEY>", "test-key"]),
]
for setting_key, invalid_values in critical_settings:
value = getattr(settings, setting_key, None)
if value in invalid_values:
logger.warning(f"Setup required: {setting_key} has placeholder or missing value")
return True
# All critical settings are configured
return False
except Exception as e:
logger.error(f"Error checking if setup required: {e}")
# If we can't check, assume setup is not required (fail open)
@@ -168,45 +169,46 @@ def is_setup_required() -> bool:
def get_missing_required_settings() -> List[str]:
"""
Get list of required settings that are missing or have placeholder values.
Returns:
List of setting keys that need to be configured
"""
missing = []
for required_setting in get_required_settings():
key = required_setting["key"]
value = getattr(settings, key, None)
# Check if value is missing or is a placeholder
placeholder_values = [
None, "",
None,
"",
f"<{key.upper()}>",
"test-key",
"your_secure_password",
"changeme",
"INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS"
"INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS",
]
if value in placeholder_values:
missing.append(key)
return missing
def get_wizard_steps() -> Dict[int, List[Dict[str, Any]]]:
"""
Get setup wizard steps organized by step number.
Returns:
Dictionary mapping step number to list of settings in that step
"""
steps = {}
for setting in get_required_settings():
step_num = setting.get("wizard_step", 1)
if step_num not in steps:
steps[step_num] = []
steps[step_num].append(setting)
return steps