gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Uptime Kuma integration with periodic ping task and configuration options

Browse Source
This commit is contained in:
Christian Krakau-Louis
2025-04-01 22:44:46 +02:00
parent 4ba34e2cbc
commit ceaed1a081
19 changed files with 921 additions and 525 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api.py
+4 -2
View File
@@ -86,12 +86,12 @@ def list_files_api(request: Request, db: Session = Depends(get_db)):
# API endpoints
@router.get("/diagnostic/settings")
@require_login
async def diagnostic_settings(current_user: dict = Depends(get_current_user)):
async def diagnostic_settings(request: Request, current_user: dict = Depends(get_current_user)):
"""
API endpoint to dump settings to the log and view basic config information
This endpoint doesn't expose sensitive information like passwords or tokens
"""
from app.utils.config_validator import dump_all_settings
from app.utils.config_validator import dump_all_settings, get_settings_for_display
# Dump full settings to log for admin to see
dump_all_settings()
@@ -107,6 +107,8 @@ async def diagnostic_settings(current_user: dict = Depends(get_current_user)):
"sftp": bool(getattr(settings, 'sftp_host', None)),
"paperless": bool(getattr(settings, 'paperless_host', None)),
"google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)),
"uptime_kuma": bool(getattr(settings, 'uptime_kuma_url', None)),
"auth": bool(getattr(settings, 'authentik_config_url', None)),
},
"imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)),
}
app/celery_worker.py
+11 -1
View File
@@ -30,6 +30,7 @@ from app.tasks.upload_to_email import upload_to_email
from app.tasks.imap_tasks import pull_all_inboxes
from app.tasks.send_to_all import send_to_all_destinations
from app.tasks.uptime_kuma_tasks import ping_uptime_kuma
celery.conf.task_routes = {
"app.tasks.*": {"queue": "default"},
@@ -47,4 +48,13 @@ celery.conf.beat_schedule = {
"task": "app.tasks.imap_tasks.pull_all_inboxes",
"schedule": crontab(minute="*/1"), # every 1 minute
},
}
# Add Uptime Kuma ping task if configured
"ping-uptime-kuma": {
"task": "app.tasks.uptime_kuma_tasks.ping_uptime_kuma",
"schedule": crontab(minute=f"*/{settings.uptime_kuma_ping_interval}"),
"options": {"expires": 55}, # Ensure tasks don't pile up
} if settings.uptime_kuma_url else None,
}
# Remove None entries from beat_schedule
celery.conf.beat_schedule = {k: v for k, v in celery.conf.beat_schedule.items() if v is not None}
app/config.py
+33 -1
View File
@@ -1,7 +1,8 @@
#!/usr/bin/env python3
from pydantic_settings import BaseSettings
from typing import Optional
from typing import Optional, List, Dict, Any
import os
class Settings(BaseSettings):
database_url: str
@@ -10,6 +11,7 @@ class Settings(BaseSettings):
openai_base_url: str = "https://api.openai.com/v1" # Default to OpenAI's endpoint
openai_model: str = "gpt-4o-mini" # Default model
workdir: str
debug: bool = False # Default to False
# Making Dropbox optional
dropbox_app_key: Optional[str] = None
@@ -110,7 +112,37 @@ class Settings(BaseSettings):
s3_storage_class: Optional[str] = "STANDARD" # Default storage class
s3_acl: Optional[str] = "private" # Default ACL
# Uptime Kuma settings
uptime_kuma_url: Optional[str] = None
uptime_kuma_ping_interval: int = 5 # Default ping interval in minutes
# Get version from file or environment
@property
def version(self) -> str:
# First try to get version from environment
env_version = os.environ.get("APP_VERSION")
if env_version:
return env_version
# Then try to get version from VERSION file
version_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "VERSION")
if os.path.exists(version_file):
with open(version_file, "r") as f:
return f.read().strip()
# Default version if not found
return "0.1.0-dev"
class Config:
env_file = ".env"
# Convert string representations of booleans to actual booleans
@classmethod
def parse_env_var(cls, field_name: str, raw_val: str) -> Any:
if field_name.endswith('_enabled') or field_name == 'debug':
if raw_val.lower() in ('false', '0', 'no', 'n', 'f'):
return False
if raw_val.lower() in ('true', '1', 'yes', 'y', 't'):
return True
return raw_val
settings = Settings()
app/frontend.py
+4 -4
View File
@@ -76,10 +76,10 @@ async def status_dashboard(request: Request):
async def env_debug(request: Request):
"""
Debug endpoint to view environment variables and settings
Only shows values when DEBUG is True
Uses actual debug setting from config
"""
# Default DEBUG to True for this route
debug_enabled = True
# Use the actual debug setting from configuration
debug_enabled = settings.debug
# Get settings data
from app.utils.config_validator import get_settings_for_display
@@ -91,7 +91,7 @@ async def env_debug(request: Request):
"request": request,
"settings": settings_data,
"debug_enabled": debug_enabled,
"app_version": getattr(settings, 'version', 'Unknown')
"app_version": settings.version
}
)
app/tasks/upload_to_sftp.py
+4
View File
@@ -68,6 +68,10 @@ def upload_to_sftp(file_path: str):
remote_base = settings.sftp_folder or ""
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
# Ensure the remote path starts with a slash if the base folder does
if remote_base.startswith('/') and not remote_path.startswith('/'):
remote_path = '/' + remote_path
# Function to check if file exists in SFTP server
def check_exists_in_sftp(path):
try:
app/tasks/uptime_kuma_tasks.py
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env python3
import logging
import requests
from celery import shared_task
from app.config import settings
logger = logging.getLogger(__name__)
@shared_task
def ping_uptime_kuma():
"""
Periodic Celery task that pings the configured Uptime Kuma URL
to report that the document processor service is running.
If no URL is configured, the task does nothing.
"""
if not settings.uptime_kuma_url:
logger.debug("Uptime Kuma URL not configured, skipping ping")
return
try:
logger.info(f"Pinging Uptime Kuma at {settings.uptime_kuma_url}")
response = requests.get(settings.uptime_kuma_url, timeout=10)
response.raise_for_status()
logger.info(f"Successfully pinged Uptime Kuma: {response.status_code}")
return True
except requests.exceptions.RequestException as e:
logger.error(f"Failed to ping Uptime Kuma: {e}")
return False
app/utils/config_validator.py
+301 -149
View File
@@ -129,170 +129,322 @@ def validate_storage_configs():
onedrive_issues.append("OneDrive credentials are not fully configured")
issues['onedrive'] = onedrive_issues
# Validate Uptime Kuma
uptime_kuma_issues = []
if not getattr(settings, 'uptime_kuma_url', None):
uptime_kuma_issues.append("UPTIME_KUMA_URL is not configured")
issues['uptime_kuma'] = uptime_kuma_issues
return issues
def get_provider_status():
"""Get the status of each provider for the dashboard"""
providers = {
"Email": {
"configured": bool(getattr(settings, 'email_host', None) and
getattr(settings, 'email_username', None) and
getattr(settings, 'email_password', None)),
"icon": "mail",
"url": getattr(settings, 'email_host', None) or "",
"description": f"Send to {getattr(settings, 'email_default_recipient', 'Not configured')}"
},
"Dropbox": {
"configured": bool(getattr(settings, 'dropbox_app_key', None) and
getattr(settings, 'dropbox_app_secret', None) and
getattr(settings, 'dropbox_refresh_token', None)),
"icon": "dropbox",
"url": "https://dropbox.com",
"description": f"Upload to folder: {getattr(settings, 'dropbox_folder', 'Root')}"
},
"Nextcloud": {
"configured": bool(getattr(settings, 'nextcloud_upload_url', None) and
getattr(settings, 'nextcloud_username', None)),
"icon": "cloud",
"url": getattr(settings, 'nextcloud_upload_url', "").split('/remote.php')[0] if getattr(settings, 'nextcloud_upload_url', None) else "",
"description": f"Upload to folder: {getattr(settings, 'nextcloud_folder', 'Root')}"
},
"SFTP": {
"configured": bool(getattr(settings, 'sftp_host', None) and
getattr(settings, 'sftp_username', None) and
(getattr(settings, 'sftp_password', None) or getattr(settings, 'sftp_private_key', None))),
"icon": "server",
"url": f"sftp://{getattr(settings, 'sftp_host', '')}:{getattr(settings, 'sftp_port', 22)}",
"description": f"Upload to {getattr(settings, 'sftp_host', 'Not configured')}:{getattr(settings, 'sftp_folder', '/')}"
},
"Paperless": {
"configured": bool(getattr(settings, 'paperless_host', None) and
getattr(settings, 'paperless_ngx_api_token', None)),
"icon": "file-text",
"url": getattr(settings, 'paperless_host', ""),
"description": "Document management system"
},
"S3": {
"configured": bool(getattr(settings, 's3_bucket_name', None) and
getattr(settings, 'aws_access_key_id', None)),
"icon": "database",
"url": f"https://s3.console.aws.amazon.com/s3/buckets/{getattr(settings, 's3_bucket_name', '')}",
"description": f"Bucket: {getattr(settings, 's3_bucket_name', 'Not configured')}"
},
"FTP": {
"configured": bool(getattr(settings, 'ftp_host', None) and
getattr(settings, 'ftp_username', None)),
"icon": "hard-drive",
"url": f"ftp://{getattr(settings, 'ftp_host', '')}:{getattr(settings, 'ftp_port', 21)}",
"description": f"Upload to {getattr(settings, 'ftp_host', 'Not configured')}:{getattr(settings, 'ftp_folder', '/')}"
},
"WebDAV": {
"configured": bool(getattr(settings, 'webdav_url', None) and
getattr(settings, 'webdav_username', None)),
"icon": "globe",
"url": getattr(settings, 'webdav_url', ""),
"description": f"Upload to {getattr(settings, 'webdav_folder', '/')}"
},
"Google Drive": {
"configured": bool(getattr(settings, 'google_drive_credentials_json', None)),
"icon": "google",
"url": "https://drive.google.com",
"description": f"Folder ID: {getattr(settings, 'google_drive_folder_id', 'Not configured')}"
},
"OneDrive": {
"configured": bool(getattr(settings, 'onedrive_client_id', None) and
getattr(settings, 'onedrive_refresh_token', None)),
"icon": "microsoft",
"url": "https://onedrive.live.com",
"description": f"Upload to folder: {getattr(settings, 'onedrive_folder_path', 'Not configured')}"
"""Returns status information for all configured providers"""
providers = {}
# Check Dropbox configuration
providers["Dropbox"] = {
"name": "Dropbox",
"configured": bool(getattr(settings, 'dropbox_refresh_token', None)),
"enabled": True,
"details": {
"folder": getattr(settings, 'dropbox_folder', 'Not set')
}
}
# Check Paperless configuration
providers["Paperless-ngx"] = {
"name": "Paperless-ngx",
"configured": bool(getattr(settings, 'paperless_host', None) and
getattr(settings, 'paperless_ngx_api_token', None)),
"enabled": True,
"details": {
"host": getattr(settings, 'paperless_host', 'Not set')
}
}
# Check NextCloud configuration
providers["NextCloud"] = {
"name": "NextCloud",
"configured": bool(getattr(settings, 'nextcloud_upload_url', None) and
getattr(settings, 'nextcloud_username', None) and
getattr(settings, 'nextcloud_password', None)),
"enabled": True,
"details": {
"url": getattr(settings, 'nextcloud_upload_url', 'Not set'),
"folder": getattr(settings, 'nextcloud_folder', 'Not set')
}
}
# Check SFTP configuration
providers["SFTP Storage"] = {
"name": "SFTP Storage",
"configured": bool(getattr(settings, 'sftp_host', None) and
getattr(settings, 'sftp_username', None) and
(getattr(settings, 'sftp_password', None) or
getattr(settings, 'sftp_private_key', None))),
"enabled": True,
"details": {
"host": getattr(settings, 'sftp_host', 'Not set'),
"folder": getattr(settings, 'sftp_folder', 'Not set')
}
}
# Check S3 configuration
providers["S3 Storage"] = {
"name": "S3 Storage",
"configured": bool(getattr(settings, 's3_bucket_name', None) and
getattr(settings, 'aws_access_key_id', None) and
getattr(settings, 'aws_secret_access_key', None)),
"enabled": True,
"details": {
"bucket": getattr(settings, 's3_bucket_name', 'Not set'),
"region": getattr(settings, 'aws_region', 'Not set')
}
}
# Check Google Drive configuration
providers["Google Drive"] = {
"name": "Google Drive",
"configured": bool(getattr(settings, 'google_drive_credentials_json', None) and
getattr(settings, 'google_drive_folder_id', None)),
"enabled": True,
"details": {
"folder_id": getattr(settings, 'google_drive_folder_id', 'Not set'),
"delegate": getattr(settings, 'google_drive_delegate_to', 'Not set')
}
}
# Check OneDrive configuration
providers["OneDrive"] = {
"name": "OneDrive",
"configured": bool(getattr(settings, 'onedrive_client_id', None) and
getattr(settings, 'onedrive_client_secret', None) and
getattr(settings, 'onedrive_refresh_token', None)),
"enabled": True,
"details": {
"folder": getattr(settings, 'onedrive_folder_path', 'Not set')
}
}
# Check WebDAV configuration
providers["WebDAV"] = {
"name": "WebDAV",
"configured": bool(getattr(settings, 'webdav_url', None) and
getattr(settings, 'webdav_username', None) and
getattr(settings, 'webdav_password', None)),
"enabled": True,
"details": {
"url": getattr(settings, 'webdav_url', 'Not set'),
"folder": getattr(settings, 'webdav_folder', 'Not set')
}
}
return providers
def dump_all_settings():
"""Dump all settings to the log for debugging"""
logger.info("================ SETTINGS DUMP ================")
# Get all attributes from settings object
attributes = inspect.getmembers(settings, lambda a: not inspect.isroutine(a))
settings_dict = {a[0]: a[1] for a in attributes
if not a[0].startswith('_') and not callable(a[1])}
# Sort keys for better readability
for key in sorted(settings_dict.keys()):
value = settings_dict[key]
# Hide sensitive values
if any(sensitive in key.lower() for sensitive in ['password', 'secret', 'token', 'key']):
if value:
value = "******** [HIDDEN FOR SECURITY]"
logger.info(f" {key} = {value}")
# Also log all environment variables
logger.info("----------- ENVIRONMENT VARIABLES -----------")
env_vars_to_log = {}
for key in sorted(os.environ.keys()):
value = os.environ[key]
# Hide sensitive values
if any(sensitive in key.lower() for sensitive in ['password', 'secret', 'token', 'key']):
if value:
value = "******** [HIDDEN FOR SECURITY]"
env_vars_to_log[key] = value
for key in sorted(env_vars_to_log.keys()):
logger.info(f" {key} = {env_vars_to_log[key]}")
logger.info("=============================================")
"""Log all settings values for diagnostic purposes"""
logger.info("--- DUMPING ALL SETTINGS FOR DIAGNOSTIC PURPOSES ---")
for key in dir(settings):
if not key.startswith('_') and not callable(getattr(settings, key)):
value = getattr(settings, key)
# Mask sensitive values in logs
if key.lower().find('password') >= 0 or key.lower().find('secret') >= 0 or key.lower().find('token') >= 0 or key.lower().find('key') >= 0:
if value:
value = "********"
logger.info(f"{key}: {value}")
logger.info("--- END OF SETTINGS DUMP ---")
def get_settings_for_display(show_values=False):
"""Get all settings organized by category for display in UI"""
# Get all attributes from settings object
attributes = inspect.getmembers(settings, lambda a: not inspect.isroutine(a))
settings_dict = {a[0]: a[1] for a in attributes
if not a[0].startswith('_') and not callable(a[1])}
"""
Group settings into logical categories and check if they are configured.
Returns a dictionary with categories as keys and lists of setting items as values.
Each setting item is a dict with name, value, and is_configured.
# Categorize settings
categories = {
"Core": [],
"Email": [],
"IMAP": [],
"Storage": [],
"Authentication": [],
"Integration": [],
"Other": []
If show_values is False, sensitive values are masked.
"""
# First include system info with version in result
result = {
"System Info": [
{
"name": "App Version",
"value": settings.version,
"is_configured": True
}
]
}
# Sort keys for better readability
for key in sorted(settings_dict.keys()):
value = settings_dict[key]
# Mask sensitive values if show_values is False
display_value = value
if not show_values or any(sensitive in key.lower() for sensitive in ['password', 'secret', 'token', 'key']):
if value:
display_value = "******** [HIDDEN]"
else:
display_value = None
# Categorize by key prefix
setting_item = {"name": key, "value": display_value, "is_configured": value is not None and value != ""}
if key.startswith(('email_', 'smtp_')):
categories["Email"].append(setting_item)
elif key.startswith('imap'):
categories["IMAP"].append(setting_item)
elif key.startswith(('s3_', 'aws_', 'dropbox_', 'nextcloud_', 'sftp_', 'ftp_', 'google_drive_')):
categories["Storage"].append(setting_item)
elif key.startswith(('auth_', 'jwt_', 'oauth_')):
categories["Authentication"].append(setting_item)
elif key.startswith(('paperless_', 'tesseract_', 'azure_')):
categories["Integration"].append(setting_item)
elif key in ('workdir', 'external_hostname', 'debug', 'version', 'env', 'log_level'):
categories["Core"].append(setting_item)
else:
categories["Other"].append(setting_item)
# Define categories and their settings
categories = {
"Core": [
"debug", # Explicitly include debug setting
"external_hostname",
"workdir",
"database_url",
"redis_url",
"gotenberg_url"
],
"Authentication": [
"auth_enabled",
"authentik_client_id",
"authentik_client_secret",
"authentik_config_url"
],
"Email": [
"email_host",
"email_port",
"email_username",
"email_password",
"email_use_tls",
"email_sender",
"email_default_recipient"
],
"IMAP": [
"imap1_host",
"imap1_port",
"imap1_username",
"imap1_password",
"imap1_ssl",
"imap1_poll_interval_minutes",
"imap1_delete_after_process",
"imap2_host",
"imap2_port",
"imap2_username",
"imap2_password",
"imap2_ssl",
"imap2_poll_interval_minutes",
"imap2_delete_after_process"
],
"Dropbox": [
"dropbox_app_key",
"dropbox_app_secret",
"dropbox_folder",
"dropbox_refresh_token"
],
"NextCloud": [
"nextcloud_upload_url",
"nextcloud_username",
"nextcloud_password",
"nextcloud_folder"
],
"Paperless": [
"paperless_host",
"paperless_ngx_api_token"
],
"Google Drive": [
"google_drive_credentials_json",
"google_drive_folder_id",
"google_drive_delegate_to"
],
"OneDrive": [
"onedrive_client_id",
"onedrive_client_secret",
"onedrive_tenant_id",
"onedrive_refresh_token",
"onedrive_folder_path"
],
"WebDAV": [
"webdav_url",
"webdav_username",
"webdav_password",
"webdav_folder",
"webdav_verify_ssl"
],
"SFTP": [
"sftp_host",
"sftp_port",
"sftp_username",
"sftp_password",
"sftp_folder",
"sftp_private_key",
"sftp_private_key_passphrase"
],
"FTP": [
"ftp_host",
"ftp_port",
"ftp_username",
"ftp_password",
"ftp_folder"
],
"S3/AWS": [
"aws_access_key_id",
"aws_secret_access_key",
"aws_region",
"s3_bucket_name",
"s3_folder_prefix",
"s3_storage_class",
"s3_acl"
],
"AI Services": [
"openai_api_key",
"openai_base_url",
"openai_model",
"azure_ai_key",
"azure_endpoint",
"azure_region"
],
"Monitoring": [
"uptime_kuma_url",
"uptime_kuma_ping_interval"
]
}
# Remove empty categories
return {k: v for k, v in categories.items() if v}
# Handle any settings that don't fit into the predefined categories
all_settings = set([key for key in dir(settings)
if not key.startswith('_') and
not callable(getattr(settings, key)) and
key not in ["model_computed_fields", "model_config",
"model_extra", "model_fields",
"model_fields_set"]])
# Ensure 'version' is excluded since we display it separately
all_settings.discard("version")
categorized_settings = set()
for cat_settings in categories.values():
categorized_settings.update(cat_settings)
uncategorized = all_settings - categorized_settings
if uncategorized:
categories["Other"] = list(uncategorized)
# Build the result
for category, setting_keys in categories.items():
items = []
for key in setting_keys:
if hasattr(settings, key):
value = getattr(settings, key)
# List of patterns that indicate sensitive values
sensitive_patterns = [
'password', 'secret', 'token', 'api_key', 'private_key',
'credentials', 'access_key', 'auth'
]
# Check if this is a sensitive value that should be masked
is_sensitive = any(pattern in key.lower() for pattern in sensitive_patterns)
# Mask sensitive values regardless of debug mode
# Other values are only hidden if debug mode is off AND show_values is False
if (is_sensitive or not show_values) and value:
if is_sensitive:
value = "********"
# Check if the setting is configured (has a non-None value)
# For boolean settings, consider them configured even if False
is_configured = value is not None
if is_configured and isinstance(value, str):
is_configured = len(value) > 0
items.append({
"name": key,
"value": value,
"is_configured": is_configured
})
if items: # Only add categories that have items
result[category] = items
return result
def check_all_configs():
"""Run all configuration validations and log results"""
app/utils/filename_utils.py
+31 -27
View File
@@ -94,38 +94,42 @@ def sanitize_filename(filename):
return sanitized
def extract_remote_path(local_path, base_dir, remote_base=None):
def extract_remote_path(file_path, base_dir, remote_base=""):
"""
Extracts the appropriate remote path based on a local path structure.
Extract a remote path for a file by preserving its directory structure
relative to the base directory, but with a new remote base path.
Args:
local_path (str): The local file path
base_dir (str): The local base directory to remove from path
remote_base (str, optional): Remote base directory to prepend
Returns:
str: The calculated remote path
Modified to skip 'processed' directory in the remote path.
"""
# Convert both paths to use forward slashes for consistency
local_path = local_path.replace('\\', '/')
base_dir = base_dir.replace('\\', '/')
# Normalize paths for consistent handling across platforms
file_path = os.path.normpath(file_path)
base_dir = os.path.normpath(base_dir)
# Make sure base_dir ends with a slash
if not base_dir.endswith('/'):
base_dir += '/'
# Remove the base directory from the local path
if local_path.startswith(base_dir):
relative_path = local_path[len(base_dir):]
# Get relative path from base directory
if file_path.startswith(base_dir):
rel_path = os.path.relpath(file_path, base_dir)
else:
# If local_path is not within base_dir, just use the filename
relative_path = os.path.basename(local_path)
# If not a subdirectory of base_dir, just use the filename
rel_path = os.path.basename(file_path)
# Prepend the remote base if provided
# Skip 'processed' directory if it's in the path
path_parts = rel_path.split(os.sep)
if 'processed' in path_parts:
# Remove 'processed' from the path
path_parts.remove('processed')
rel_path = os.path.join(*path_parts)
# Combine with remote base path
if remote_base:
# Ensure remote_base ends with slash
if not remote_base.endswith('/'):
remote_base += '/'
return remote_base + relative_path
if remote_base.startswith('/'):
# Handle absolute path for services like Dropbox
remote_path = os.path.join(remote_base[1:], rel_path)
else:
remote_path = os.path.join(remote_base, rel_path)
else:
remote_path = rel_path
return relative_path
# Convert to forward slashes for compatibility with most cloud services
remote_path = remote_path.replace(os.sep, '/')
return remote_path