gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #31 from christianlouis/uptime-monitor

Browse Source 
Add Uptime Kuma integration with periodic ping task and configuration options
This commit is contained in:
Christian Krakau-Louis
2025-04-01 22:54:30 +02:00
committed by GitHub
parent 4ba34e2cbc ceaed1a081
commit 8769982cea
19 changed files with 921 additions and 525 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.demo
+4
View File
@@ -113,3 +113,7 @@ SFTP_PASSWORD=your_secure_sftp_password
# SFTP_PRIVATE_KEY=/path/to/private_key.pem # SFTP_PRIVATE_KEY=/path/to/private_key.pem
# SFTP_PRIVATE_KEY_PASSPHRASE=optional_passphrase # SFTP_PRIVATE_KEY_PASSPHRASE=optional_passphrase
SFTP_FOLDER=/Documents/Uploads SFTP_FOLDER=/Documents/Uploads
# Uptime Kuma
UPTIME_KUMA_URL=https://status.example.com/api/push/abcdef123456?status=up
UPTIME_KUMA_PING_INTERVAL=5
VERSION
+1
View File
@@ -0,0 +1 @@
1.0.0
app/api.py
+4 -2
View File
@@ -86,12 +86,12 @@ def list_files_api(request: Request, db: Session = Depends(get_db)):
# API endpoints # API endpoints
@router.get("/diagnostic/settings") @router.get("/diagnostic/settings")
@require_login @require_login
async def diagnostic_settings(current_user: dict = Depends(get_current_user)): async def diagnostic_settings(request: Request, current_user: dict = Depends(get_current_user)):
""" """
API endpoint to dump settings to the log and view basic config information API endpoint to dump settings to the log and view basic config information
This endpoint doesn't expose sensitive information like passwords or tokens This endpoint doesn't expose sensitive information like passwords or tokens
""" """
from app.utils.config_validator import dump_all_settings from app.utils.config_validator import dump_all_settings, get_settings_for_display
# Dump full settings to log for admin to see # Dump full settings to log for admin to see
dump_all_settings() dump_all_settings()
@@ -107,6 +107,8 @@ async def diagnostic_settings(current_user: dict = Depends(get_current_user)):
"sftp": bool(getattr(settings, 'sftp_host', None)), "sftp": bool(getattr(settings, 'sftp_host', None)),
"paperless": bool(getattr(settings, 'paperless_host', None)), "paperless": bool(getattr(settings, 'paperless_host', None)),
"google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)), "google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)),
"uptime_kuma": bool(getattr(settings, 'uptime_kuma_url', None)),
"auth": bool(getattr(settings, 'authentik_config_url', None)),
}, },
"imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)), "imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)),
} }
app/celery_worker.py
+10
View File
@@ -30,6 +30,7 @@ from app.tasks.upload_to_email import upload_to_email
from app.tasks.imap_tasks import pull_all_inboxes from app.tasks.imap_tasks import pull_all_inboxes
from app.tasks.send_to_all import send_to_all_destinations from app.tasks.send_to_all import send_to_all_destinations
from app.tasks.uptime_kuma_tasks import ping_uptime_kuma
celery.conf.task_routes = { celery.conf.task_routes = {
"app.tasks.*": {"queue": "default"}, "app.tasks.*": {"queue": "default"},
@@ -47,4 +48,13 @@ celery.conf.beat_schedule = {
"task": "app.tasks.imap_tasks.pull_all_inboxes", "task": "app.tasks.imap_tasks.pull_all_inboxes",
"schedule": crontab(minute="*/1"), # every 1 minute "schedule": crontab(minute="*/1"), # every 1 minute
}, },
# Add Uptime Kuma ping task if configured
"ping-uptime-kuma": {
"task": "app.tasks.uptime_kuma_tasks.ping_uptime_kuma",
"schedule": crontab(minute=f"*/{settings.uptime_kuma_ping_interval}"),
"options": {"expires": 55}, # Ensure tasks don't pile up
} if settings.uptime_kuma_url else None,
} }
# Remove None entries from beat_schedule
celery.conf.beat_schedule = {k: v for k, v in celery.conf.beat_schedule.items() if v is not None}
app/config.py
+33 -1
View File
@@ -1,7 +1,8 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from pydantic_settings import BaseSettings from pydantic_settings import BaseSettings
from typing import Optional from typing import Optional, List, Dict, Any
import os
class Settings(BaseSettings): class Settings(BaseSettings):
database_url: str database_url: str
@@ -10,6 +11,7 @@ class Settings(BaseSettings):
openai_base_url: str = "https://api.openai.com/v1" # Default to OpenAI's endpoint openai_base_url: str = "https://api.openai.com/v1" # Default to OpenAI's endpoint
openai_model: str = "gpt-4o-mini" # Default model openai_model: str = "gpt-4o-mini" # Default model
workdir: str workdir: str
debug: bool = False # Default to False
# Making Dropbox optional # Making Dropbox optional
dropbox_app_key: Optional[str] = None dropbox_app_key: Optional[str] = None
@@ -110,7 +112,37 @@ class Settings(BaseSettings):
s3_storage_class: Optional[str] = "STANDARD" # Default storage class s3_storage_class: Optional[str] = "STANDARD" # Default storage class
s3_acl: Optional[str] = "private" # Default ACL s3_acl: Optional[str] = "private" # Default ACL
# Uptime Kuma settings
uptime_kuma_url: Optional[str] = None
uptime_kuma_ping_interval: int = 5 # Default ping interval in minutes
# Get version from file or environment
@property
def version(self) -> str:
# First try to get version from environment
env_version = os.environ.get("APP_VERSION")
if env_version:
return env_version
# Then try to get version from VERSION file
version_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "VERSION")
if os.path.exists(version_file):
with open(version_file, "r") as f:
return f.read().strip()
# Default version if not found
return "0.1.0-dev"
class Config: class Config:
env_file = ".env" env_file = ".env"
# Convert string representations of booleans to actual booleans
@classmethod
def parse_env_var(cls, field_name: str, raw_val: str) -> Any:
if field_name.endswith('_enabled') or field_name == 'debug':
if raw_val.lower() in ('false', '0', 'no', 'n', 'f'):
return False
if raw_val.lower() in ('true', '1', 'yes', 'y', 't'):
return True
return raw_val
settings = Settings() settings = Settings()
app/frontend.py
+4 -4
View File
@@ -76,10 +76,10 @@ async def status_dashboard(request: Request):
async def env_debug(request: Request): async def env_debug(request: Request):
""" """
Debug endpoint to view environment variables and settings Debug endpoint to view environment variables and settings
Only shows values when DEBUG is True Uses actual debug setting from config
""" """
# Default DEBUG to True for this route # Use the actual debug setting from configuration
debug_enabled = True debug_enabled = settings.debug
# Get settings data # Get settings data
from app.utils.config_validator import get_settings_for_display from app.utils.config_validator import get_settings_for_display
@@ -91,7 +91,7 @@ async def env_debug(request: Request):
"request": request, "request": request,
"settings": settings_data, "settings": settings_data,
"debug_enabled": debug_enabled, "debug_enabled": debug_enabled,
"app_version": getattr(settings, 'version', 'Unknown') "app_version": settings.version
} }
) )
app/tasks/upload_to_sftp.py
+4
View File
@@ -68,6 +68,10 @@ def upload_to_sftp(file_path: str):
remote_base = settings.sftp_folder or "" remote_base = settings.sftp_folder or ""
remote_path = extract_remote_path(file_path, settings.workdir, remote_base) remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
# Ensure the remote path starts with a slash if the base folder does
if remote_base.startswith('/') and not remote_path.startswith('/'):
remote_path = '/' + remote_path
# Function to check if file exists in SFTP server # Function to check if file exists in SFTP server
def check_exists_in_sftp(path): def check_exists_in_sftp(path):
try: try:
app/tasks/uptime_kuma_tasks.py
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env python3
import logging
import requests
from celery import shared_task
from app.config import settings
logger = logging.getLogger(__name__)
@shared_task
def ping_uptime_kuma():
"""
Periodic Celery task that pings the configured Uptime Kuma URL
to report that the document processor service is running.
If no URL is configured, the task does nothing.
"""
if not settings.uptime_kuma_url:
logger.debug("Uptime Kuma URL not configured, skipping ping")
return
try:
logger.info(f"Pinging Uptime Kuma at {settings.uptime_kuma_url}")
response = requests.get(settings.uptime_kuma_url, timeout=10)
response.raise_for_status()
logger.info(f"Successfully pinged Uptime Kuma: {response.status_code}")
return True
except requests.exceptions.RequestException as e:
logger.error(f"Failed to ping Uptime Kuma: {e}")
return False
app/utils/config_validator.py
+299 -147
View File
@@ -129,170 +129,322 @@ def validate_storage_configs():
onedrive_issues.append("OneDrive credentials are not fully configured") onedrive_issues.append("OneDrive credentials are not fully configured")
issues['onedrive'] = onedrive_issues issues['onedrive'] = onedrive_issues
# Validate Uptime Kuma
uptime_kuma_issues = []
if not getattr(settings, 'uptime_kuma_url', None):
uptime_kuma_issues.append("UPTIME_KUMA_URL is not configured")
issues['uptime_kuma'] = uptime_kuma_issues
return issues return issues
def get_provider_status(): def get_provider_status():
"""Get the status of each provider for the dashboard""" """Returns status information for all configured providers"""
providers = { providers = {}
"Email": {
"configured": bool(getattr(settings, 'email_host', None) and # Check Dropbox configuration
getattr(settings, 'email_username', None) and providers["Dropbox"] = {
getattr(settings, 'email_password', None)), "name": "Dropbox",
"icon": "mail", "configured": bool(getattr(settings, 'dropbox_refresh_token', None)),
"url": getattr(settings, 'email_host', None) or "", "enabled": True,
"description": f"Send to {getattr(settings, 'email_default_recipient', 'Not configured')}" "details": {
}, "folder": getattr(settings, 'dropbox_folder', 'Not set')
"Dropbox": { }
"configured": bool(getattr(settings, 'dropbox_app_key', None) and }
getattr(settings, 'dropbox_app_secret', None) and
getattr(settings, 'dropbox_refresh_token', None)), # Check Paperless configuration
"icon": "dropbox", providers["Paperless-ngx"] = {
"url": "https://dropbox.com", "name": "Paperless-ngx",
"description": f"Upload to folder: {getattr(settings, 'dropbox_folder', 'Root')}" "configured": bool(getattr(settings, 'paperless_host', None) and
}, getattr(settings, 'paperless_ngx_api_token', None)),
"Nextcloud": { "enabled": True,
"configured": bool(getattr(settings, 'nextcloud_upload_url', None) and "details": {
getattr(settings, 'nextcloud_username', None)), "host": getattr(settings, 'paperless_host', 'Not set')
"icon": "cloud", }
"url": getattr(settings, 'nextcloud_upload_url', "").split('/remote.php')[0] if getattr(settings, 'nextcloud_upload_url', None) else "", }
"description": f"Upload to folder: {getattr(settings, 'nextcloud_folder', 'Root')}"
}, # Check NextCloud configuration
"SFTP": { providers["NextCloud"] = {
"configured": bool(getattr(settings, 'sftp_host', None) and "name": "NextCloud",
getattr(settings, 'sftp_username', None) and "configured": bool(getattr(settings, 'nextcloud_upload_url', None) and
(getattr(settings, 'sftp_password', None) or getattr(settings, 'sftp_private_key', None))), getattr(settings, 'nextcloud_username', None) and
"icon": "server", getattr(settings, 'nextcloud_password', None)),
"url": f"sftp://{getattr(settings, 'sftp_host', '')}:{getattr(settings, 'sftp_port', 22)}", "enabled": True,
"description": f"Upload to {getattr(settings, 'sftp_host', 'Not configured')}:{getattr(settings, 'sftp_folder', '/')}" "details": {
}, "url": getattr(settings, 'nextcloud_upload_url', 'Not set'),
"Paperless": { "folder": getattr(settings, 'nextcloud_folder', 'Not set')
"configured": bool(getattr(settings, 'paperless_host', None) and }
getattr(settings, 'paperless_ngx_api_token', None)), }
"icon": "file-text",
"url": getattr(settings, 'paperless_host', ""), # Check SFTP configuration
"description": "Document management system" providers["SFTP Storage"] = {
}, "name": "SFTP Storage",
"S3": { "configured": bool(getattr(settings, 'sftp_host', None) and
"configured": bool(getattr(settings, 's3_bucket_name', None) and getattr(settings, 'sftp_username', None) and
getattr(settings, 'aws_access_key_id', None)), (getattr(settings, 'sftp_password', None) or
"icon": "database", getattr(settings, 'sftp_private_key', None))),
"url": f"https://s3.console.aws.amazon.com/s3/buckets/{getattr(settings, 's3_bucket_name', '')}", "enabled": True,
"description": f"Bucket: {getattr(settings, 's3_bucket_name', 'Not configured')}" "details": {
}, "host": getattr(settings, 'sftp_host', 'Not set'),
"FTP": { "folder": getattr(settings, 'sftp_folder', 'Not set')
"configured": bool(getattr(settings, 'ftp_host', None) and }
getattr(settings, 'ftp_username', None)), }
"icon": "hard-drive",
"url": f"ftp://{getattr(settings, 'ftp_host', '')}:{getattr(settings, 'ftp_port', 21)}", # Check S3 configuration
"description": f"Upload to {getattr(settings, 'ftp_host', 'Not configured')}:{getattr(settings, 'ftp_folder', '/')}" providers["S3 Storage"] = {
}, "name": "S3 Storage",
"WebDAV": { "configured": bool(getattr(settings, 's3_bucket_name', None) and
"configured": bool(getattr(settings, 'webdav_url', None) and getattr(settings, 'aws_access_key_id', None) and
getattr(settings, 'webdav_username', None)), getattr(settings, 'aws_secret_access_key', None)),
"icon": "globe", "enabled": True,
"url": getattr(settings, 'webdav_url', ""), "details": {
"description": f"Upload to {getattr(settings, 'webdav_folder', '/')}" "bucket": getattr(settings, 's3_bucket_name', 'Not set'),
}, "region": getattr(settings, 'aws_region', 'Not set')
"Google Drive": { }
"configured": bool(getattr(settings, 'google_drive_credentials_json', None)), }
"icon": "google",
"url": "https://drive.google.com", # Check Google Drive configuration
"description": f"Folder ID: {getattr(settings, 'google_drive_folder_id', 'Not configured')}" providers["Google Drive"] = {
}, "name": "Google Drive",
"OneDrive": { "configured": bool(getattr(settings, 'google_drive_credentials_json', None) and
"configured": bool(getattr(settings, 'onedrive_client_id', None) and getattr(settings, 'google_drive_folder_id', None)),
getattr(settings, 'onedrive_refresh_token', None)), "enabled": True,
"icon": "microsoft", "details": {
"url": "https://onedrive.live.com", "folder_id": getattr(settings, 'google_drive_folder_id', 'Not set'),
"description": f"Upload to folder: {getattr(settings, 'onedrive_folder_path', 'Not configured')}" "delegate": getattr(settings, 'google_drive_delegate_to', 'Not set')
}
}
# Check OneDrive configuration
providers["OneDrive"] = {
"name": "OneDrive",
"configured": bool(getattr(settings, 'onedrive_client_id', None) and
getattr(settings, 'onedrive_client_secret', None) and
getattr(settings, 'onedrive_refresh_token', None)),
"enabled": True,
"details": {
"folder": getattr(settings, 'onedrive_folder_path', 'Not set')
}
}
# Check WebDAV configuration
providers["WebDAV"] = {
"name": "WebDAV",
"configured": bool(getattr(settings, 'webdav_url', None) and
getattr(settings, 'webdav_username', None) and
getattr(settings, 'webdav_password', None)),
"enabled": True,
"details": {
"url": getattr(settings, 'webdav_url', 'Not set'),
"folder": getattr(settings, 'webdav_folder', 'Not set')
} }
} }
return providers return providers
def dump_all_settings(): def dump_all_settings():
"""Dump all settings to the log for debugging""" """Log all settings values for diagnostic purposes"""
logger.info("================ SETTINGS DUMP ================") logger.info("--- DUMPING ALL SETTINGS FOR DIAGNOSTIC PURPOSES ---")
for key in dir(settings):
# Get all attributes from settings object if not key.startswith('_') and not callable(getattr(settings, key)):
attributes = inspect.getmembers(settings, lambda a: not inspect.isroutine(a)) value = getattr(settings, key)
settings_dict = {a[0]: a[1] for a in attributes # Mask sensitive values in logs
if not a[0].startswith('_') and not callable(a[1])} if key.lower().find('password') >= 0 or key.lower().find('secret') >= 0 or key.lower().find('token') >= 0 or key.lower().find('key') >= 0:
if value:
# Sort keys for better readability value = "********"
for key in sorted(settings_dict.keys()): logger.info(f"{key}: {value}")
value = settings_dict[key] logger.info("--- END OF SETTINGS DUMP ---")
# Hide sensitive values
if any(sensitive in key.lower() for sensitive in ['password', 'secret', 'token', 'key']):
if value:
value = "******** [HIDDEN FOR SECURITY]"
logger.info(f" {key} = {value}")
# Also log all environment variables
logger.info("----------- ENVIRONMENT VARIABLES -----------")
env_vars_to_log = {}
for key in sorted(os.environ.keys()):
value = os.environ[key]
# Hide sensitive values
if any(sensitive in key.lower() for sensitive in ['password', 'secret', 'token', 'key']):
if value:
value = "******** [HIDDEN FOR SECURITY]"
env_vars_to_log[key] = value
for key in sorted(env_vars_to_log.keys()):
logger.info(f" {key} = {env_vars_to_log[key]}")
logger.info("=============================================")
def get_settings_for_display(show_values=False): def get_settings_for_display(show_values=False):
"""Get all settings organized by category for display in UI""" """
# Get all attributes from settings object Group settings into logical categories and check if they are configured.
attributes = inspect.getmembers(settings, lambda a: not inspect.isroutine(a)) Returns a dictionary with categories as keys and lists of setting items as values.
settings_dict = {a[0]: a[1] for a in attributes Each setting item is a dict with name, value, and is_configured.
if not a[0].startswith('_') and not callable(a[1])}
# Categorize settings If show_values is False, sensitive values are masked.
categories = { """
"Core": [], # First include system info with version in result
"Email": [], result = {
"IMAP": [], "System Info": [
"Storage": [], {
"Authentication": [], "name": "App Version",
"Integration": [], "value": settings.version,
"Other": [] "is_configured": True
}
]
} }
# Sort keys for better readability # Define categories and their settings
for key in sorted(settings_dict.keys()): categories = {
value = settings_dict[key] "Core": [
# Mask sensitive values if show_values is False "debug", # Explicitly include debug setting
display_value = value "external_hostname",
if not show_values or any(sensitive in key.lower() for sensitive in ['password', 'secret', 'token', 'key']): "workdir",
if value: "database_url",
display_value = "******** [HIDDEN]" "redis_url",
else: "gotenberg_url"
display_value = None ],
"Authentication": [
"auth_enabled",
"authentik_client_id",
"authentik_client_secret",
"authentik_config_url"
],
"Email": [
"email_host",
"email_port",
"email_username",
"email_password",
"email_use_tls",
"email_sender",
"email_default_recipient"
],
"IMAP": [
"imap1_host",
"imap1_port",
"imap1_username",
"imap1_password",
"imap1_ssl",
"imap1_poll_interval_minutes",
"imap1_delete_after_process",
"imap2_host",
"imap2_port",
"imap2_username",
"imap2_password",
"imap2_ssl",
"imap2_poll_interval_minutes",
"imap2_delete_after_process"
],
"Dropbox": [
"dropbox_app_key",
"dropbox_app_secret",
"dropbox_folder",
"dropbox_refresh_token"
],
"NextCloud": [
"nextcloud_upload_url",
"nextcloud_username",
"nextcloud_password",
"nextcloud_folder"
],
"Paperless": [
"paperless_host",
"paperless_ngx_api_token"
],
"Google Drive": [
"google_drive_credentials_json",
"google_drive_folder_id",
"google_drive_delegate_to"
],
"OneDrive": [
"onedrive_client_id",
"onedrive_client_secret",
"onedrive_tenant_id",
"onedrive_refresh_token",
"onedrive_folder_path"
],
"WebDAV": [
"webdav_url",
"webdav_username",
"webdav_password",
"webdav_folder",
"webdav_verify_ssl"
],
"SFTP": [
"sftp_host",
"sftp_port",
"sftp_username",
"sftp_password",
"sftp_folder",
"sftp_private_key",
"sftp_private_key_passphrase"
],
"FTP": [
"ftp_host",
"ftp_port",
"ftp_username",
"ftp_password",
"ftp_folder"
],
"S3/AWS": [
"aws_access_key_id",
"aws_secret_access_key",
"aws_region",
"s3_bucket_name",
"s3_folder_prefix",
"s3_storage_class",
"s3_acl"
],
"AI Services": [
"openai_api_key",
"openai_base_url",
"openai_model",
"azure_ai_key",
"azure_endpoint",
"azure_region"
],
"Monitoring": [
"uptime_kuma_url",
"uptime_kuma_ping_interval"
]
}
# Categorize by key prefix # Handle any settings that don't fit into the predefined categories
setting_item = {"name": key, "value": display_value, "is_configured": value is not None and value != ""} all_settings = set([key for key in dir(settings)
if not key.startswith('_') and
not callable(getattr(settings, key)) and
key not in ["model_computed_fields", "model_config",
"model_extra", "model_fields",
"model_fields_set"]])
if key.startswith(('email_', 'smtp_')): # Ensure 'version' is excluded since we display it separately
categories["Email"].append(setting_item) all_settings.discard("version")
elif key.startswith('imap'):
categories["IMAP"].append(setting_item)
elif key.startswith(('s3_', 'aws_', 'dropbox_', 'nextcloud_', 'sftp_', 'ftp_', 'google_drive_')):
categories["Storage"].append(setting_item)
elif key.startswith(('auth_', 'jwt_', 'oauth_')):
categories["Authentication"].append(setting_item)
elif key.startswith(('paperless_', 'tesseract_', 'azure_')):
categories["Integration"].append(setting_item)
elif key in ('workdir', 'external_hostname', 'debug', 'version', 'env', 'log_level'):
categories["Core"].append(setting_item)
else:
categories["Other"].append(setting_item)
# Remove empty categories categorized_settings = set()
return {k: v for k, v in categories.items() if v} for cat_settings in categories.values():
categorized_settings.update(cat_settings)
uncategorized = all_settings - categorized_settings
if uncategorized:
categories["Other"] = list(uncategorized)
# Build the result
for category, setting_keys in categories.items():
items = []
for key in setting_keys:
if hasattr(settings, key):
value = getattr(settings, key)
# List of patterns that indicate sensitive values
sensitive_patterns = [
'password', 'secret', 'token', 'api_key', 'private_key',
'credentials', 'access_key', 'auth'
]
# Check if this is a sensitive value that should be masked
is_sensitive = any(pattern in key.lower() for pattern in sensitive_patterns)
# Mask sensitive values regardless of debug mode
# Other values are only hidden if debug mode is off AND show_values is False
if (is_sensitive or not show_values) and value:
if is_sensitive:
value = "********"
# Check if the setting is configured (has a non-None value)
# For boolean settings, consider them configured even if False
is_configured = value is not None
if is_configured and isinstance(value, str):
is_configured = len(value) > 0
items.append({
"name": key,
"value": value,
"is_configured": is_configured
})
if items: # Only add categories that have items
result[category] = items
return result
def check_all_configs(): def check_all_configs():
"""Run all configuration validations and log results""" """Run all configuration validations and log results"""
app/utils/filename_utils.py
+31 -27
View File
@@ -94,38 +94,42 @@ def sanitize_filename(filename):
return sanitized return sanitized
def extract_remote_path(local_path, base_dir, remote_base=None): def extract_remote_path(file_path, base_dir, remote_base=""):
""" """
Extracts the appropriate remote path based on a local path structure. Extract a remote path for a file by preserving its directory structure
relative to the base directory, but with a new remote base path.
Args: Modified to skip 'processed' directory in the remote path.
local_path (str): The local file path
base_dir (str): The local base directory to remove from path
remote_base (str, optional): Remote base directory to prepend
Returns:
str: The calculated remote path
""" """
# Convert both paths to use forward slashes for consistency # Normalize paths for consistent handling across platforms
local_path = local_path.replace('\\', '/') file_path = os.path.normpath(file_path)
base_dir = base_dir.replace('\\', '/') base_dir = os.path.normpath(base_dir)
# Make sure base_dir ends with a slash # Get relative path from base directory
if not base_dir.endswith('/'): if file_path.startswith(base_dir):
base_dir += '/' rel_path = os.path.relpath(file_path, base_dir)
# Remove the base directory from the local path
if local_path.startswith(base_dir):
relative_path = local_path[len(base_dir):]
else: else:
# If local_path is not within base_dir, just use the filename # If not a subdirectory of base_dir, just use the filename
relative_path = os.path.basename(local_path) rel_path = os.path.basename(file_path)
# Prepend the remote base if provided # Skip 'processed' directory if it's in the path
path_parts = rel_path.split(os.sep)
if 'processed' in path_parts:
# Remove 'processed' from the path
path_parts.remove('processed')
rel_path = os.path.join(*path_parts)
# Combine with remote base path
if remote_base: if remote_base:
# Ensure remote_base ends with slash if remote_base.startswith('/'):
if not remote_base.endswith('/'): # Handle absolute path for services like Dropbox
remote_base += '/' remote_path = os.path.join(remote_base[1:], rel_path)
return remote_base + relative_path else:
remote_path = os.path.join(remote_base, rel_path)
else:
remote_path = rel_path
return relative_path # Convert to forward slashes for compatibility with most cloud services
remote_path = remote_path.replace(os.sep, '/')
return remote_path
docs/AmazonS3Setup.md
+100
View File
@@ -0,0 +1,100 @@
# Setting up Amazon S3 Integration
This guide explains how to set up the Amazon S3 integration for DocuNova.
## Required Configuration Parameters
| **Variable** | **Description** |
|---------------------------------|-------------------------------------------------------|
| `AWS_ACCESS_KEY_ID` | AWS IAM access key ID |
| `AWS_SECRET_ACCESS_KEY` | AWS IAM secret access key |
| `AWS_REGION` | AWS region where your S3 bucket is located (default: `us-east-1`) |
| `S3_BUCKET_NAME` | Name of your S3 bucket |
| `S3_FOLDER_PREFIX` | Optional prefix/folder path for uploaded files |
| `S3_STORAGE_CLASS` | Storage class for uploaded objects (default: `STANDARD`) |
| `S3_ACL` | Access control for uploaded files (default: `private`) |
For a complete list of configuration options, see the [Configuration Guide](ConfigurationGuide.md).
## Step-by-Step Setup Instructions
### 1. Create an S3 bucket
1. Go to the [Amazon S3 Console](https://s3.console.aws.amazon.com/)
2. Click "Create bucket"
3. Enter a globally unique name for your bucket
4. Select your preferred AWS region
5. Configure other settings as needed (block public access is recommended)
6. Click "Create bucket"
### 2. Create an IAM User with S3 Access
1. Go to the [AWS IAM Console](https://console.aws.amazon.com/iam/)
2. Navigate to "Users" and click "Add users"
3. Enter a name (e.g., "docunova-s3-access")
4. For access type, select "Programmatic access"
5. Click "Next: Permissions"
6. Choose "Attach existing policies directly" and search for "AmazonS3FullAccess"
7. For more security, you can create a custom policy limiting access to just your bucket
8. Click through to review and create the user
9. On the final page, you'll see the Access Key ID and Secret Access Key
10. Save these credentials securely as they won't be shown again
### 3. Configure DocuNova
1. Set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to the credentials from step 2
2. Set `AWS_REGION` to the region where your bucket was created (e.g., "us-east-1")
3. Set `S3_BUCKET_NAME` to your bucket name
4. Set `S3_FOLDER_PREFIX` to organize files in specific subfolder paths (e.g., "invoices/" or "documents/2023/")
5. Optionally customize `S3_STORAGE_CLASS` and `S3_ACL` for your storage needs
### 4. Optional: Create a Custom IAM Policy (for better security)
1. In IAM console, go to "Policies" and click "Create policy"
2. Use the JSON editor and paste a policy like this (replace `your-bucket-name`):
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::your-bucket-name",
"arn:aws:s3:::your-bucket-name/*"
]
}
]
}
```
3. After creating the policy, attach it to your user instead of the broader AmazonS3FullAccess
## Storage Class Options
Amazon S3 offers several storage classes to optimize costs:
| **Storage Class** | **Use Case** | **Retrieval Time** |
|------------------|--------------|-------------------|
| `STANDARD` | Default, frequently accessed data | Immediate |
| `INTELLIGENT_TIERING` | Data with changing or unknown access patterns | Immediate |
| `STANDARD_IA` | Long-lived, infrequently accessed data | Immediate |
| `ONEZONE_IA` | Long-lived, infrequently accessed, non-critical data | Immediate |
| `GLACIER_IR` | Archive data that needs immediate access | Immediate |
| `GLACIER` | Archive data that rarely needs to be accessed | Minutes to hours |
| `DEEP_ARCHIVE` | Long-term archive and digital preservation | Hours |
Set your preferred storage class using the `S3_STORAGE_CLASS` parameter.
## Access Control List (ACL) Options
Common ACL values include:
- `private` (default) - Only the bucket owner has access
- `public-read` - Anyone can read the file (use cautiously)
- `bucket-owner-full-control` - Useful for cross-account uploads
For most document storage scenarios, `private` is recommended for security.
docs/ConfigurationGuide.md
+45 -331
View File
@@ -55,12 +55,14 @@ DocuNova can monitor multiple IMAP mailboxes for document attachments. Each mail
### Dropbox ### Dropbox
| **Variable** | **Description** | **How to Obtain** | | **Variable** | **Description** |
|-------------------------|--------------------------------------------------|---------------------------------------------------------| |-------------------------|--------------------------------------------------|
| `DROPBOX_APP_KEY` | Dropbox API app key. | [Dropbox Developer Console](#setting-up-dropbox-integration) | | `DROPBOX_APP_KEY` | Dropbox API app key. |
| `DROPBOX_APP_SECRET` | Dropbox API app secret. | [Dropbox Developer Console](#setting-up-dropbox-integration) | | `DROPBOX_APP_SECRET` | Dropbox API app secret. |
| `DROPBOX_REFRESH_TOKEN` | OAuth2 refresh token for Dropbox. | Follow steps in [Dropbox Setup](#setting-up-dropbox-integration) | | `DROPBOX_REFRESH_TOKEN` | OAuth2 refresh token for Dropbox. |
| `DROPBOX_FOLDER` | Default folder path for Dropbox uploads. | e.g. `"/Documents/Uploads"` (leading slash optional) | | `DROPBOX_FOLDER` | Default folder path for Dropbox uploads. |
For detailed setup instructions, see the [Dropbox Setup Guide](DropboxSetup.md).
### Nextcloud ### Nextcloud
@@ -73,11 +75,13 @@ DocuNova can monitor multiple IMAP mailboxes for document attachments. Each mail
### Google Drive ### Google Drive
| **Variable** | **Description** | **How to Obtain** | | **Variable** | **Description** |
|---------------------------------|-------------------------------------------------------|------------------------------------------------------| |---------------------------------|-------------------------------------------------------|
| `GOOGLE_DRIVE_CREDENTIALS_JSON` | JSON string containing service account credentials | [Google Cloud Console](#setting-up-google-drive-api) | | `GOOGLE_DRIVE_CREDENTIALS_JSON` | JSON string containing service account credentials |
| `GOOGLE_DRIVE_FOLDER_ID` | Google Drive folder ID for file uploads | See [folder ID instructions](#get-google-drive-folder-id) | | `GOOGLE_DRIVE_FOLDER_ID` | Google Drive folder ID for file uploads |
| `GOOGLE_DRIVE_DELEGATE_TO` | Email address to delegate permissions (optional) | User email in your Google Workspace | | `GOOGLE_DRIVE_DELEGATE_TO` | Email address to delegate permissions (optional) |
For detailed setup instructions, see the [Google Drive Setup Guide](GoogleDriveSetup.md).
### WebDAV ### WebDAV
@@ -125,166 +129,36 @@ DocuNova can monitor multiple IMAP mailboxes for document attachments. Each mail
### OneDrive / Microsoft Graph ### OneDrive / Microsoft Graph
| **Variable** | **Description** | **How to Obtain** | | **Variable** | **Description** |
|---------------------------------|-------------------------------------------------------|------------------------------------------------------| |---------------------------------|-------------------------------------------------------|
| `ONEDRIVE_CLIENT_ID` | Azure AD application client ID | [Microsoft Azure Portal](#setting-up-onedrive-integration) | | `ONEDRIVE_CLIENT_ID` | Azure AD application client ID |
| `ONEDRIVE_CLIENT_SECRET` | Azure AD application client secret | [Microsoft Azure Portal](#setting-up-onedrive-integration) | | `ONEDRIVE_CLIENT_SECRET` | Azure AD application client secret |
| `ONEDRIVE_TENANT_ID` | Azure AD tenant ID: use "common" for personal accounts or your tenant ID for corporate accounts | [Microsoft Azure Portal](#setting-up-onedrive-integration) | | `ONEDRIVE_TENANT_ID` | Azure AD tenant ID: use "common" for personal accounts or your tenant ID for corporate accounts |
| `ONEDRIVE_REFRESH_TOKEN` | OAuth 2.0 refresh token (required for personal accounts) | Follow steps in [Personal OneDrive Setup](#personal-onedrive-setup) | | `ONEDRIVE_REFRESH_TOKEN` | OAuth 2.0 refresh token (required for personal accounts) |
| `ONEDRIVE_FOLDER_PATH` | Folder path in OneDrive for storing documents | e.g. `/Documents/Uploads` or `Documents/Uploads` | | `ONEDRIVE_FOLDER_PATH` | Folder path in OneDrive for storing documents |
## Setting up OneDrive Integration For detailed setup instructions, see the [OneDrive Setup Guide](OneDriveSetup.md).
There are two main types of Microsoft accounts that can be used with OneDrive integration:
1. **Personal Microsoft Accounts** - These include accounts with @outlook.com, @hotmail.com, @live.com domains, or personal Microsoft accounts linked to other email addresses (like Gmail)
2. **Work/School Microsoft Accounts** - These are accounts managed by an organization through Microsoft 365 or Azure Active Directory
The setup process differs slightly based on which account type you're using.
### Common Setup Steps (All Account Types)
1. **Register an application in Azure Active Directory**:
- Go to the [Azure Portal](https://portal.azure.com/)
- Navigate to "Azure Active Directory" > "App registrations"
- Click "New registration"
- Enter a name for your application (e.g., "DocuNova")
- For "Supported account types", select the appropriate option:
- For personal accounts: "Accounts in any organizational directory and personal Microsoft accounts"
- For corporate accounts only: "Accounts in this organizational directory only"
- For Redirect URI, select "Web" and enter a URL you can access (e.g., `http://localhost:8000/auth/callback`)
- Click "Register"
2. **Get Application (client) ID**:
- After registration, note the "Application (client) ID" from the overview page
- Set this value as `ONEDRIVE_CLIENT_ID`
3. **Create a client secret**:
- In your application page, go to "Certificates & secrets"
- Under "Client secrets," click "New client secret"
- Add a description and select an expiration period
- Click "Add" and immediately copy the secret value (it will only be shown once)
- Set this value as `ONEDRIVE_CLIENT_SECRET`
### For Personal Microsoft Accounts
If you're using a personal Microsoft account (@outlook.com, @hotmail.com, or personal accounts linked to other emails):
1. **Set Tenant ID to "common"**:
- Set `ONEDRIVE_TENANT_ID=common` in your configuration
2. **Configure API permissions**:
- In your application page, go to "API permissions"
- Click "Add a permission"
- Select "Microsoft Graph" > "Delegated permissions"
- Search for and add the following permissions:
- `Files.ReadWrite` (Allows the app to read and write files that the user has access to)
- `offline_access` (Needed for refresh tokens)
- Click "Add permissions"
3. **Generate a Refresh Token**:
- Use the following URL (replace CLIENT_ID and REDIRECT_URI with your values):
```
https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=YOUR_CLIENT_ID&response_type=code&redirect_uri=YOUR_REDIRECT_URI&response_mode=query&scope=offline_access%20Files.ReadWrite
```
- Open this URL in your browser
- Sign in with your personal Microsoft account
- After authentication, you'll be redirected to your redirect URI with a code parameter in the URL
- Copy the code value from the URL (everything after "code=")
4. **Exchange Code for Refresh Token**:
- Use the following command to exchange the code for tokens:
```bash
curl -X POST https://login.microsoftonline.com/common/oauth2/v2.0/token \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "client_id=YOUR_CLIENT_ID&scope=offline_access Files.ReadWrite&code=YOUR_AUTH_CODE&redirect_uri=YOUR_REDIRECT_URI&grant_type=authorization_code&client_secret=YOUR_CLIENT_SECRET"
```
- From the response JSON, copy the `refresh_token` value
- Set this as `ONEDRIVE_REFRESH_TOKEN` in your configuration
### For Corporate Microsoft Accounts
If you're using a work/school account provided by your organization:
1. **Get your Tenant ID**:
- In the Azure Portal, find your "Tenant ID" (also called "Directory ID")
- It will be in the Azure Active Directory overview or properties section
- Set this value as `ONEDRIVE_TENANT_ID` in your configuration
2. **Configuration based on use case**:
**Option A: Access your own OneDrive (Interactive Login)**
This option requires a refresh token just like personal accounts:
- Follow the same steps as for personal accounts, but use your work email to sign in
- Make sure to set `ONEDRIVE_TENANT_ID` to your organization's tenant ID instead of "common"
- Set the refresh token you receive as `ONEDRIVE_REFRESH_TOKEN`
**Option B: Access OneDrive as a system service (App-only access)**
This option is for service accounts or automated systems with no user interaction:
- In API permissions, add "Application permissions" instead of "Delegated permissions"
- Add `Files.ReadWrite.All` permission under "Application permissions"
- Click "Grant admin consent" (requires admin privileges)
- In this case, `ONEDRIVE_REFRESH_TOKEN` is not needed as the app will use client credentials flow
- Note: This approach can only access specific shared folders or sites, not personal OneDrives
### Troubleshooting OAuth Login Issues
If you encounter errors during authentication:
1. **Check account permissions**:
- Ensure your Microsoft account has the necessary permissions to grant access
- For corporate accounts, check if your admin has restricted third-party app access
2. **Permission errors**:
- Verify the app registration has the correct API permissions
- For corporate accounts, ensure an admin has consented to the permissions
3. **Refresh token expired**:
- If uploads stop working, you may need to generate a new refresh token
- Repeat the process to get a new authorization code and refresh token
### Configuration Examples
**Personal Microsoft Account:**
```dotenv
ONEDRIVE_CLIENT_ID=12345678-1234-1234-1234-123456789012
ONEDRIVE_CLIENT_SECRET=your_client_secret
ONEDRIVE_TENANT_ID=common
ONEDRIVE_REFRESH_TOKEN=your_refresh_token
ONEDRIVE_FOLDER_PATH=Documents/Uploads
```
**Corporate Account with Interactive Login:**
```dotenv
ONEDRIVE_CLIENT_ID=12345678-1234-1234-1234-123456789012
ONEDRIVE_CLIENT_SECRET=your_client_secret
ONEDRIVE_TENANT_ID=87654321-4321-4321-4321-210987654321
ONEDRIVE_REFRESH_TOKEN=your_refresh_token
ONEDRIVE_FOLDER_PATH=Documents/Uploads
```
**Corporate Account with App-Only Access:**
```dotenv
ONEDRIVE_CLIENT_ID=12345678-1234-1234-1234-123456789012
ONEDRIVE_CLIENT_SECRET=your_client_secret
ONEDRIVE_TENANT_ID=87654321-4321-4321-4321-210987654321
# No refresh token needed for app-only access
ONEDRIVE_FOLDER_PATH=Documents/Uploads
```
### Amazon S3 ### Amazon S3
| **Variable** | **Description** | **How to Obtain** | | **Variable** | **Description** |
|---------------------------------|-------------------------------------------------------|------------------------------------------------------| |---------------------------------|-------------------------------------------------------|
| `AWS_ACCESS_KEY_ID` | AWS IAM access key ID | [AWS IAM Console](#setting-up-amazon-s3-integration) | | `AWS_ACCESS_KEY_ID` | AWS IAM access key ID |
| `AWS_SECRET_ACCESS_KEY` | AWS IAM secret access key | [AWS IAM Console](#setting-up-amazon-s3-integration) | | `AWS_SECRET_ACCESS_KEY` | AWS IAM secret access key |
| `AWS_REGION` | AWS region where your S3 bucket is located (default: `us-east-1`) | [AWS S3 Console](https://s3.console.aws.amazon.com/) | | `AWS_REGION` | AWS region where your S3 bucket is located (default: `us-east-1`) |
| `S3_BUCKET_NAME` | Name of your S3 bucket | [AWS S3 Console](https://s3.console.aws.amazon.com/) | | `S3_BUCKET_NAME` | Name of your S3 bucket |
| `S3_FOLDER_PREFIX` | Optional prefix/folder path for uploaded files | e.g. `documents/` or `uploads/2023/` (include trailing slash) | | `S3_FOLDER_PREFIX` | Optional prefix/folder path for uploaded files |
| `S3_STORAGE_CLASS` | Storage class for uploaded objects (default: `STANDARD`) | [S3 Storage Classes](https://aws.amazon.com/s3/storage-classes/) | | `S3_STORAGE_CLASS` | Storage class for uploaded objects (default: `STANDARD`) |
| `S3_ACL` | Access control for uploaded files (default: `private`) | `private`, `public-read`, etc. | | `S3_ACL` | Access control for uploaded files (default: `private`) |
For detailed setup instructions, see the [Amazon S3 Setup Guide](AmazonS3Setup.md).
### Uptime Kuma
| **Variable** | **Description** |
|-----------------------------|----------------------------------------------------------------|
| `UPTIME_KUMA_URL` | Uptime Kuma push URL for monitoring the application's health. |
| `UPTIME_KUMA_PING_INTERVAL` | How often to ping Uptime Kuma in minutes (default: `5`). |
## Configuration Examples ## Configuration Examples
@@ -395,172 +269,12 @@ S3_BUCKET_NAME=my-document-bucket
S3_FOLDER_PREFIX=documents/uploads/2023/ # Will place files in this subfolder S3_FOLDER_PREFIX=documents/uploads/2023/ # Will place files in this subfolder
S3_STORAGE_CLASS=STANDARD S3_STORAGE_CLASS=STANDARD
S3_ACL=private S3_ACL=private
# Uptime Kuma
UPTIME_KUMA_URL=https://kuma.example.com/api/push/abcde12345?status=up
UPTIME_KUMA_PING_INTERVAL=5
``` ```
## Setting up Google Drive API
To use the Google Drive integration, follow these steps:
1. **Create a Google Cloud Project**:
- Go to the [Google Cloud Console](https://console.cloud.google.com/)
- Create a new project or select an existing one
- Navigate to "APIs & Services" > "Library"
- Search for and enable the "Google Drive API"
2. **Create Service Account**:
- Go to "APIs & Services" > "Credentials"
- Click "Create Credentials" > "Service Account"
- Fill in the service account details and click "Create"
- Add appropriate roles (e.g., "Editor" for full access)
- Click "Continue" and then "Done"
3. **Generate Service Account Key**:
- Find your service account in the list and click on it
- Go to the "Keys" tab
- Click "Add Key" > "Create New Key"
- Choose JSON format and click "Create"
- The key file will be downloaded automatically
4. **Configure DocuNova**:
- Open the downloaded JSON key file
- Set the entire JSON content as the `GOOGLE_DRIVE_CREDENTIALS_JSON` environment variable
- For security, ensure the JSON is properly escaped if your deployment method requires it
### Get Google Drive Folder ID
To find your Google Drive folder ID:
1. Navigate to the desired folder in Google Drive web interface
2. The URL will look like: `https://drive.google.com/drive/folders/1a2b3c4d5e6f7g8h9i0j`
3. The string after "folders/" is your folder ID (in this example: `1a2b3c4d5e6f7g8h9i0j`)
4. Set this value as `GOOGLE_DRIVE_FOLDER_ID` in your configuration
### Domain-Wide Delegation (Optional)
If you need the service account to access files on behalf of users in your Google Workspace:
1. In your [Google Workspace Admin Console](https://admin.google.com/), go to:
- Security > API Controls > Domain-wide Delegation
2. Click "Add new" and provide:
- Client ID: your service account's client ID (found in the JSON credentials file)
- OAuth Scopes: `https://www.googleapis.com/auth/drive`
3. Set `GOOGLE_DRIVE_DELEGATE_TO` to the email address of the user to impersonate
This setup is only relevant for Google Workspace environments where you need the service account to access user-specific files.
## Setting up Dropbox Integration
To use the Dropbox integration, you'll need to create a Dropbox app and generate OAuth2 credentials:
1. **Create a Dropbox App**:
- Go to the [Dropbox Developer Apps Console](https://www.dropbox.com/developers/apps)
- Click "Create app"
- Select "Scoped access" for API
- Choose "Full Dropbox" access (or "App folder" for more restricted access)
- Give your app a name (e.g., "DocuNova")
- Click "Create app"
2. **Configure App Permissions**:
- In your app's settings page, go to the "Permissions" tab
- Enable the following permissions:
- `files.content.write` (to upload files)
- `files.content.read` (if you need to read file content)
- Click "Submit" to save changes
3. **Get App Key and Secret**:
- On your app's settings page, find the "App key" and "App secret"
- Set these as `DROPBOX_APP_KEY` and `DROPBOX_APP_SECRET` in your configuration
4. **Generate a Refresh Token**:
- Go to the "OAuth 2" tab in your app settings
- Add a redirect URI: `http://localhost` (this is for the authorization flow)
- Generate an authorization URL with these instructions:
```
https://www.dropbox.com/oauth2/authorize?client_id=YOUR_APP_KEY&response_type=code&token_access_type=offline
```
- Replace `YOUR_APP_KEY` with your app key
- Open this URL in your browser
- Authorize the app when prompted
- You'll be redirected to `localhost` with a code parameter in the URL
- Copy this code parameter
5. **Exchange the Code for a Refresh Token**:
- Use this curl command to exchange the code for tokens:
```bash
curl -X POST https://api.dropboxapi.com/oauth2/token \
-d code=YOUR_AUTH_CODE \
-d grant_type=authorization_code \
-d client_id=YOUR_APP_KEY \
-d client_secret=YOUR_APP_SECRET \
-d redirect_uri=http://localhost
```
- From the response, copy the `refresh_token` value
6. **Configure DocuNova**:
- Set `DROPBOX_APP_KEY`, `DROPBOX_APP_SECRET`, and `DROPBOX_REFRESH_TOKEN` with your values
- Set `DROPBOX_FOLDER` to the path where files should be uploaded
The system will use the refresh token to automatically generate short-lived access tokens when needed, so you shouldn't need to worry about token expiration.
## Setting up Amazon S3 Integration
To use the Amazon S3 integration, you'll need an AWS account and an S3 bucket:
1. **Create an S3 bucket**:
- Go to the [Amazon S3 Console](https://s3.console.aws.amazon.com/)
- Click "Create bucket"
- Enter a globally unique name for your bucket
- Select your preferred AWS region
- Configure other settings as needed (block public access is recommended)
- Click "Create bucket"
2. **Create an IAM User with S3 Access**:
- Go to the [AWS IAM Console](https://console.aws.amazon.com/iam/)
- Navigate to "Users" and click "Add users"
- Enter a name (e.g., "docunova-s3-access")
- For access type, select "Programmatic access"
- Click "Next: Permissions"
- Choose "Attach existing policies directly" and search for "AmazonS3FullAccess"
- For more security, you can create a custom policy limiting access to just your bucket
- Click through to review and create the user
- On the final page, you'll see the Access Key ID and Secret Access Key
- Save these credentials securely as they won't be shown again
3. **Configure DocuNova**:
- Set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to the credentials from step 2
- Set `AWS_REGION` to the region where your bucket was created (e.g., "us-east-1")
- Set `S3_BUCKET_NAME` to your bucket name
- Set `S3_FOLDER_PREFIX` to organize files in specific subfolder paths (e.g., "invoices/" or "documents/2023/")
- Optionally customize `S3_STORAGE_CLASS` and `S3_ACL` for your storage needs
4. **Optional: Create a Custom IAM Policy** (for better security):
- In IAM console, go to "Policies" and click "Create policy"
- Use the JSON editor and paste a policy like this (replace `your-bucket-name`):
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::your-bucket-name",
"arn:aws:s3:::your-bucket-name/*"
]
}
]
}
```
- After creating the policy, attach it to your user instead of the broader AmazonS3FullAccess
## Selective Service Configuration ## Selective Service Configuration
You can choose which document storage services to use by only including the relevant environment variables. For example, if you only want to use Dropbox, include only the Dropbox variables and omit the Paperless NGX and Nextcloud variables. You can choose which document storage services to use by only including the relevant environment variables. For example, if you only want to use Dropbox, include only the Dropbox variables and omit the Paperless NGX and Nextcloud variables.
docs/ConfigurationMaster.md
+37
View File
@@ -0,0 +1,37 @@
# DocuNova Configuration
This section contains detailed documentation about configuring DocuNova for your environment.
## Configuration Overview
DocuNova is designed to be highly configurable through environment variables, typically set in a `.env` file. This allows you to enable only the services and integrations you need for your specific use case.
## Documentation Sections
- [Configuration Guide](ConfigurationGuide.md) - Complete list of all available configuration parameters
- [Google Drive Setup](GoogleDriveSetup.md) - How to set up Google Drive integration
- [Dropbox Setup](DropboxSetup.md) - How to set up Dropbox integration
- [OneDrive Setup](OneDriveSetup.md) - How to set up Microsoft OneDrive/Graph integration
- [Amazon S3 Setup](AmazonS3Setup.md) - How to set up Amazon S3 integration
## Configuration File Location
The `.env` file should be placed at the root of the project directory. When using Docker Compose, you can reference it with the `env_file` directive in your `docker-compose.yml`.
## Example Configuration
Below is a minimal example configuration to get started:
```dotenv
# Core settings
DATABASE_URL=sqlite:///./app/database.db
REDIS_URL=redis://redis:6379/0
WORKDIR=/workdir
GOTENBERG_URL=http://gotenberg:3000
EXTERNAL_HOSTNAME=docunova.example.com
# Enable only the services you need
# For detailed parameters, see the Configuration Guide
```
For a complete example with all possible parameters, see the [Configuration Guide](ConfigurationGuide.md).
docs/DropboxSetup.md
+76
View File
@@ -0,0 +1,76 @@
# Setting up Dropbox Integration
This guide explains how to set up the Dropbox integration for DocuNova.
## Required Configuration Parameters
| **Variable** | **Description** |
|-------------------------|--------------------------------------------------|
| `DROPBOX_APP_KEY` | Dropbox API app key |
| `DROPBOX_APP_SECRET` | Dropbox API app secret |
| `DROPBOX_REFRESH_TOKEN` | OAuth2 refresh token for Dropbox |
| `DROPBOX_FOLDER` | Default folder path for Dropbox uploads |
For a complete list of configuration options, see the [Configuration Guide](ConfigurationGuide.md).
## Step-by-Step Setup Instructions
### 1. Create a Dropbox App
1. Go to the [Dropbox Developer Apps Console](https://www.dropbox.com/developers/apps)
2. Click "Create app"
3. Select "Scoped access" for API
4. Choose "Full Dropbox" access (or "App folder" for more restricted access)
5. Give your app a name (e.g., "DocuNova")
6. Click "Create app"
### 2. Configure App Permissions
1. In your app's settings page, go to the "Permissions" tab
2. Enable the following permissions:
- `files.content.write` (to upload files)
- `files.content.read` (if you need to read file content)
3. Click "Submit" to save changes
### 3. Get App Key and Secret
1. On your app's settings page, find the "App key" and "App secret"
2. Set these as `DROPBOX_APP_KEY` and `DROPBOX_APP_SECRET` in your configuration
### 4. Generate a Refresh Token
1. Go to the "OAuth 2" tab in your app settings
2. Add a redirect URI: `http://localhost` (this is for the authorization flow)
3. Generate an authorization URL with these instructions:
```
https://www.dropbox.com/oauth2/authorize?client_id=YOUR_APP_KEY&response_type=code&token_access_type=offline
```
4. Replace `YOUR_APP_KEY` with your app key
5. Open this URL in your browser
6. Authorize the app when prompted
7. You'll be redirected to `localhost` with a code parameter in the URL
8. Copy this code parameter
### 5. Exchange the Code for a Refresh Token
1. Use this curl command to exchange the code for tokens:
```bash
curl -X POST https://api.dropboxapi.com/oauth2/token \
-d code=YOUR_AUTH_CODE \
-d grant_type=authorization_code \
-d client_id=YOUR_APP_KEY \
-d client_secret=YOUR_APP_SECRET \
-d redirect_uri=http://localhost
```
2. From the response, copy the `refresh_token` value
### 6. Configure DocuNova
1. Set `DROPBOX_APP_KEY`, `DROPBOX_APP_SECRET`, and `DROPBOX_REFRESH_TOKEN` with your values
2. Set `DROPBOX_FOLDER` to the path where files should be uploaded (e.g., `/Documents/Uploads`)
The system will use the refresh token to automatically generate short-lived access tokens when needed, so you shouldn't need to worry about token expiration.
docs/GoogleDriveSetup.md
+66
View File
@@ -0,0 +1,66 @@
# Setting up Google Drive Integration
This guide explains how to set up the Google Drive integration for DocuNova.
## Required Configuration Parameters
| **Variable** | **Description** |
|---------------------------------|-------------------------------------------------------|
| `GOOGLE_DRIVE_CREDENTIALS_JSON` | JSON string containing service account credentials |
| `GOOGLE_DRIVE_FOLDER_ID` | Google Drive folder ID for file uploads |
| `GOOGLE_DRIVE_DELEGATE_TO` | Email address to delegate permissions (optional) |
For a complete list of configuration options, see the [Configuration Guide](ConfigurationGuide.md).
## Step-by-Step Setup Instructions
### 1. Create a Google Cloud Project
1. Go to the [Google Cloud Console](https://console.cloud.google.com/)
2. Create a new project or select an existing one
3. Navigate to "APIs & Services" > "Library"
4. Search for and enable the "Google Drive API"
### 2. Create Service Account
1. Go to "APIs & Services" > "Credentials"
2. Click "Create Credentials" > "Service Account"
3. Fill in the service account details and click "Create"
4. Add appropriate roles (e.g., "Editor" for full access)
5. Click "Continue" and then "Done"
### 3. Generate Service Account Key
1. Find your service account in the list and click on it
2. Go to the "Keys" tab
3. Click "Add Key" > "Create New Key"
4. Choose JSON format and click "Create"
5. The key file will be downloaded automatically
### 4. Configure DocuNova
1. Open the downloaded JSON key file
2. Set the entire JSON content as the `GOOGLE_DRIVE_CREDENTIALS_JSON` environment variable
3. For security, ensure the JSON is properly escaped if your deployment method requires it
### Get Google Drive Folder ID
To find your Google Drive folder ID:
1. Navigate to the desired folder in Google Drive web interface
2. The URL will look like: `https://drive.google.com/drive/folders/1a2b3c4d5e6f7g8h9i0j`
3. The string after "folders/" is your folder ID (in this example: `1a2b3c4d5e6f7g8h9i0j`)
4. Set this value as `GOOGLE_DRIVE_FOLDER_ID` in your configuration
### Domain-Wide Delegation (Optional)
If you need the service account to access files on behalf of users in your Google Workspace:
1. In your [Google Workspace Admin Console](https://admin.google.com/), go to:
- Security > API Controls > Domain-wide Delegation
2. Click "Add new" and provide:
- Client ID: your service account's client ID (found in the JSON credentials file)
- OAuth Scopes: `https://www.googleapis.com/auth/drive`
3. Set `GOOGLE_DRIVE_DELEGATE_TO` to the email address of the user to impersonate
This setup is only relevant for Google Workspace environments where you need the service account to access user-specific files.
docs/OneDriveSetup.md
+164
View File
@@ -0,0 +1,164 @@
# Setting up OneDrive Integration
This guide explains how to set up the Microsoft OneDrive integration for DocuNova.
## Required Configuration Parameters
| **Variable** | **Description** |
|---------------------------------|-------------------------------------------------------|
| `ONEDRIVE_CLIENT_ID` | Azure AD application client ID |
| `ONEDRIVE_CLIENT_SECRET` | Azure AD application client secret |
| `ONEDRIVE_TENANT_ID` | Azure AD tenant ID: use "common" for personal accounts or your tenant ID for corporate accounts |
| `ONEDRIVE_REFRESH_TOKEN` | OAuth 2.0 refresh token (required for personal accounts) |
| `ONEDRIVE_FOLDER_PATH` | Folder path in OneDrive for storing documents |
For a complete list of configuration options, see the [Configuration Guide](ConfigurationGuide.md).
## Types of Microsoft Accounts
There are two main types of Microsoft accounts that can be used with OneDrive integration:
1. **Personal Microsoft Accounts** - These include accounts with @outlook.com, @hotmail.com, @live.com domains, or personal Microsoft accounts linked to other email addresses (like Gmail)
2. **Work/School Microsoft Accounts** - These are accounts managed by an organization through Microsoft 365 or Azure Active Directory
The setup process differs slightly based on which account type you're using.
## Common Setup Steps (All Account Types)
### 1. Register an application in Azure Active Directory
1. Go to the [Azure Portal](https://portal.azure.com/)
2. Navigate to "Azure Active Directory" > "App registrations"
3. Click "New registration"
4. Enter a name for your application (e.g., "DocuNova")
5. For "Supported account types", select the appropriate option:
- For personal accounts: "Accounts in any organizational directory and personal Microsoft accounts"
- For corporate accounts only: "Accounts in this organizational directory only"
6. For Redirect URI, select "Web" and enter a URL you can access (e.g., `http://localhost:8000/auth/callback`)
7. Click "Register"
### 2. Get Application (client) ID
1. After registration, note the "Application (client) ID" from the overview page
2. Set this value as `ONEDRIVE_CLIENT_ID`
### 3. Create a client secret
1. In your application page, go to "Certificates & secrets"
2. Under "Client secrets," click "New client secret"
3. Add a description and select an expiration period
4. Click "Add" and immediately copy the secret value (it will only be shown once)
5. Set this value as `ONEDRIVE_CLIENT_SECRET`
## For Personal Microsoft Accounts
If you're using a personal Microsoft account (@outlook.com, @hotmail.com, or personal accounts linked to other emails):
### 1. Set Tenant ID to "common"
- Set `ONEDRIVE_TENANT_ID=common` in your configuration
### 2. Configure API permissions
1. In your application page, go to "API permissions"
2. Click "Add a permission"
3. Select "Microsoft Graph" > "Delegated permissions"
4. Search for and add the following permissions:
- `Files.ReadWrite` (Allows the app to read and write files that the user has access to)
- `offline_access` (Needed for refresh tokens)
5. Click "Add permissions"
### 3. Generate a Refresh Token
1. Use the following URL (replace CLIENT_ID and REDIRECT_URI with your values):
```
https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=YOUR_CLIENT_ID&response_type=code&redirect_uri=YOUR_REDIRECT_URI&response_mode=query&scope=offline_access%20Files.ReadWrite
```
2. Open this URL in your browser
3. Sign in with your personal Microsoft account
4. After authentication, you'll be redirected to your redirect URI with a code parameter in the URL
5. Copy the code value from the URL (everything after "code=")
### 4. Exchange Code for Refresh Token
1. Use the following command to exchange the code for tokens:
```bash
curl -X POST https://login.microsoftonline.com/common/oauth2/v2.0/token \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "client_id=YOUR_CLIENT_ID&scope=offline_access Files.ReadWrite&code=YOUR_AUTH_CODE&redirect_uri=YOUR_REDIRECT_URI&grant_type=authorization_code&client_secret=YOUR_CLIENT_SECRET"
```
2. From the response JSON, copy the `refresh_token` value
3. Set this as `ONEDRIVE_REFRESH_TOKEN` in your configuration
## For Corporate Microsoft Accounts
If you're using a work/school account provided by your organization:
### 1. Get your Tenant ID
1. In the Azure Portal, find your "Tenant ID" (also called "Directory ID")
2. It will be in the Azure Active Directory overview or properties section
3. Set this value as `ONEDRIVE_TENANT_ID` in your configuration
### 2. Configuration based on use case
**Option A: Access your own OneDrive (Interactive Login)**
This option requires a refresh token just like personal accounts:
1. Follow the same steps as for personal accounts, but use your work email to sign in
2. Make sure to set `ONEDRIVE_TENANT_ID` to your organization's tenant ID instead of "common"
3. Set the refresh token you receive as `ONEDRIVE_REFRESH_TOKEN`
**Option B: Access OneDrive as a system service (App-only access)**
This option is for service accounts or automated systems with no user interaction:
1. In API permissions, add "Application permissions" instead of "Delegated permissions"
2. Add `Files.ReadWrite.All` permission under "Application permissions"
3. Click "Grant admin consent" (requires admin privileges)
4. In this case, `ONEDRIVE_REFRESH_TOKEN` is not needed as the app will use client credentials flow
5. Note: This approach can only access specific shared folders or sites, not personal OneDrives
## Troubleshooting OAuth Login Issues
If you encounter errors during authentication:
1. **Check account permissions**:
- Ensure your Microsoft account has the necessary permissions to grant access
- For corporate accounts, check if your admin has restricted third-party app access
2. **Permission errors**:
- Verify the app registration has the correct API permissions
- For corporate accounts, ensure an admin has consented to the permissions
3. **Refresh token expired**:
- If uploads stop working, you may need to generate a new refresh token
- Repeat the process to get a new authorization code and refresh token
## Configuration Examples
**Personal Microsoft Account:**
```dotenv
ONEDRIVE_CLIENT_ID=12345678-1234-1234-1234-123456789012
ONEDRIVE_CLIENT_SECRET=your_client_secret
ONEDRIVE_TENANT_ID=common
ONEDRIVE_REFRESH_TOKEN=your_refresh_token
ONEDRIVE_FOLDER_PATH=Documents/Uploads
```
**Corporate Account with Interactive Login:**
```dotenv
ONEDRIVE_CLIENT_ID=12345678-1234-1234-1234-123456789012
ONEDRIVE_CLIENT_SECRET=your_client_secret
ONEDRIVE_TENANT_ID=87654321-4321-4321-4321-210987654321
ONEDRIVE_REFRESH_TOKEN=your_refresh_token
ONEDRIVE_FOLDER_PATH=Documents/Uploads
```
**Corporate Account with App-Only Access:**
```dotenv
ONEDRIVE_CLIENT_ID=12345678-1234-1234-1234-123456789012
ONEDRIVE_CLIENT_SECRET=your_client_secret
ONEDRIVE_TENANT_ID=87654321-4321-4321-4321-210987654321
# No refresh token needed for app-only access
ONEDRIVE_FOLDER_PATH=Documents/Uploads
```
docs/setup-guides/dropbox-setup.md
View File
frontend/templates/env_debug.html
+7 -9
View File
@@ -34,16 +34,14 @@
{{ item.name }} {{ item.name }}
</td> </td>
<td class="px-6 py-4 whitespace-nowrap text-sm text-gray-500"> <td class="px-6 py-4 whitespace-nowrap text-sm text-gray-500">
{% if debug_enabled %} {% if item.value is none %}
{% if item.value is none %} <span class="text-gray-400">NULL</span>
<span class="text-gray-400">NULL</span> {% elif item.value == "" %}
{% elif item.value == "" %} <span class="text-gray-400">(empty string)</span>
<span class="text-gray-400">(empty string)</span> {% elif item.value == "********" %}
{% else %} <span class="text-gray-400">********</span>
{{ item.value }}
{% endif %}
{% else %} {% else %}
<span class="text-gray-400">*** hidden in non-debug mode ***</span> {{ item.value }}
{% endif %} {% endif %}
</td> </td>
<td class="px-6 py-4 whitespace-nowrap"> <td class="px-6 py-4 whitespace-nowrap">
frontend/templates/status_dashboard.html
+2
View File
@@ -44,6 +44,8 @@
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14" /> <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14" />
{% elif provider.icon == "microsoft" %} {% elif provider.icon == "microsoft" %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z" /> <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z" />
{% elif provider.icon == "activity" %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 10V3L4 14h7v7l9-11h-7z" />
{% else %} {% else %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" /> <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
{% endif %} {% endif %}