gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #148 from christianlouis/copilot/fix-flake8-linter-errors

Browse Source 
fix: resolve all 1080 Flake8 linter errors across app/ directory
This commit is contained in:
Christian Krakau-Louis
2026-02-08 19:14:01 +01:00
committed by GitHub
parent 75b8dddc7b d08040ac4a
commit 152f54dade
73 changed files with 2200 additions and 2185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/__init__.py
+14 -11
View File
@@ -1,21 +1,24 @@
"""
API Router module that combines all API endpoints
"""
from fastapi import APIRouter
import logging
from fastapi import APIRouter
from app.api.azure import router as azure_router
from app.api.diagnostic import router as diagnostic_router
from app.api.dropbox import router as dropbox_router
from app.api.files import router as files_router
from app.api.google_drive import router as google_drive_router
from app.api.logs import router as logs_router
from app.api.onedrive import router as onedrive_router
from app.api.openai import router as openai_router
from app.api.process import router as process_router
from app.api.settings import router as settings_router
# Import all the individual routers
from app.api.user import router as user_router
from app.api.files import router as files_router
from app.api.process import router as process_router
from app.api.diagnostic import router as diagnostic_router
from app.api.onedrive import router as onedrive_router
from app.api.dropbox import router as dropbox_router
from app.api.openai import router as openai_router
from app.api.azure import router as azure_router
from app.api.google_drive import router as google_drive_router
from app.api.logs import router as logs_router
from app.api.settings import router as settings_router
# Set up logging
logger = logging.getLogger(__name__)
app/api/azure.py
+26 -36
View File
@@ -1,24 +1,25 @@
"""
Azure AI API endpoints
"""
from fastapi import APIRouter, Request, HTTPException, status
import logging
import os
from app.auth import require_login
from app.config import settings
import logging
import azure.core.exceptions
from azure.ai.documentintelligence import DocumentIntelligenceAdministrationClient
# Import the Azure modules including the administration client
from azure.core.credentials import AzureKeyCredential
from azure.ai.documentintelligence import DocumentIntelligenceClient
from azure.ai.documentintelligence import DocumentIntelligenceAdministrationClient
import azure.core.exceptions
from fastapi import APIRouter, Request
from app.auth import require_login
from app.config import settings
# Set up logging
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/azure/test")
@require_login
async def test_azure_connection(request: Request):
@@ -40,15 +41,14 @@ async def test_azure_connection(request: Request):
return {
"status": "error",
"message": f"Azure Document Intelligence configuration is incomplete. Missing: {', '.join(missing)}"
"message": f"Azure Document Intelligence configuration is incomplete. Missing: {', '.join(missing)}",
}
# Try to initialize the admin client and make a request to list operations
try:
# Initialize the admin client with credentials
admin_client = DocumentIntelligenceAdministrationClient(
endpoint=settings.azure_endpoint,
credential=AzureKeyCredential(settings.azure_ai_key)
endpoint=settings.azure_endpoint, credential=AzureKeyCredential(settings.azure_ai_key)
)
# Test the connection by listing operations - this is a documented method in the admin client
@@ -61,12 +61,12 @@ async def test_azure_connection(request: Request):
operations_info = []
try:
for op in operations:
if hasattr(op, 'operation_id') and op.operation_id:
if hasattr(op, "operation_id") and op.operation_id:
op_info = {
"id": op.operation_id,
"status": op.status if hasattr(op, 'status') else "Unknown",
"created": str(op.created_on) if hasattr(op, 'created_on') else "Unknown",
"kind": op.kind if hasattr(op, 'kind') else "Unknown"
"status": op.status if hasattr(op, "status") else "Unknown",
"created": str(op.created_on) if hasattr(op, "created_on") else "Unknown",
"kind": op.kind if hasattr(op, "kind") else "Unknown",
}
operations_info.append(op_info)
@@ -76,49 +76,39 @@ async def test_azure_connection(request: Request):
"message": f"Azure Document Intelligence connection is valid. Found {operation_count} operations.",
"endpoint": settings.azure_endpoint,
"operations_count": operation_count,
"recent_operations": operations_info[:3] if operations_info else []
"recent_operations": operations_info[:3] if operations_info else [],
}
except Exception as e:
# If error occurs while processing operations info, still return success
logger.warning(f"Connected to Azure but couldn't parse operations: {e}")
return {
"status": "success",
"message": "Azure Document Intelligence connection is valid, but couldn't retrieve operations details.",
"endpoint": settings.azure_endpoint
"message": "Azure Document Intelligence connection is valid, "
"but couldn't retrieve operations details.",
"endpoint": settings.azure_endpoint,
}
except azure.core.exceptions.ClientAuthenticationError as e:
logger.error(f"Azure authentication error: {e}")
return {
"status": "error",
"message": f"Authentication error: Invalid API key or credentials",
"detail": str(e)
"message": "Authentication error: Invalid API key or credentials",
"detail": str(e),
}
except azure.core.exceptions.ServiceRequestError as e:
logger.error(f"Azure service request error: {e}")
return {
"status": "error",
"message": f"Service request error: Could not reach the Azure endpoint",
"detail": str(e)
"message": "Service request error: Could not reach the Azure endpoint",
"detail": str(e),
}
except ValueError as e:
logger.error(f"Azure configuration value error: {e}")
return {
"status": "error",
"message": f"Configuration error: {str(e)}",
"detail": str(e)
}
return {"status": "error", "message": f"Configuration error: {str(e)}", "detail": str(e)}
except Exception as e:
logger.error(f"Azure connection test failed with unexpected error: {e}")
return {
"status": "error",
"message": f"Connection test failed with unexpected error",
"detail": str(e)
}
return {"status": "error", "message": "Connection test failed with unexpected error", "detail": str(e)}
except Exception as e:
logger.exception("Unexpected error testing Azure Document Intelligence connection")
return {
"status": "error",
"message": f"Unexpected error: {str(e)}"
}
return {"status": "error", "message": f"Unexpected error: {str(e)}"}
app/api/common.py
+2 -1
View File
@@ -5,10 +5,11 @@ Common utilities for API routes
import logging
import os
from pathlib import Path
from fastapi import HTTPException, status
from app.database import SessionLocal
from app.config import settings
from app.database import SessionLocal
# Set up logging
logger = logging.getLogger(__name__)
app/api/diagnostic.py
+33 -27
View File
@@ -1,10 +1,12 @@
"""
Diagnostic API endpoints
"""
from fastapi import APIRouter, Request, Depends
import logging
from app.auth import require_login, get_current_user
from fastapi import APIRouter, Depends, Request
from app.auth import get_current_user, require_login
from app.config import settings
# Set up logging
@@ -12,6 +14,7 @@ logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/diagnostic/settings")
@require_login
async def diagnostic_settings(request: Request, current_user: dict = Depends(get_current_user)):
@@ -19,7 +22,8 @@ async def diagnostic_settings(request: Request, current_user: dict = Depends(get
API endpoint to dump settings to the log and view basic config information
This endpoint doesn't expose sensitive information like passwords or tokens
"""
from app.utils.config_validator import dump_all_settings, get_settings_for_display
from app.utils.config_validator import dump_all_settings
# Dump full settings to log for admin to see
dump_all_settings()
@@ -28,33 +32,35 @@ async def diagnostic_settings(request: Request, current_user: dict = Depends(get
"workdir": settings.workdir,
"external_hostname": settings.external_hostname,
"configured_services": {
"email": bool(getattr(settings, 'email_host', None)),
"s3": bool(getattr(settings, 's3_bucket_name', None)),
"dropbox": bool(getattr(settings, 'dropbox_refresh_token', None)),
"onedrive": bool(getattr(settings, 'onedrive_refresh_token', None)),
"nextcloud": bool(getattr(settings, 'nextcloud_upload_url', None)),
"sftp": bool(getattr(settings, 'sftp_host', None)),
"paperless": bool(getattr(settings, 'paperless_host', None)),
"google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)),
"uptime_kuma": bool(getattr(settings, 'uptime_kuma_url', None)),
"auth": bool(getattr(settings, 'authentik_config_url', None)),
"openai": bool(getattr(settings, 'openai_api_key', None)),
"azure": bool(getattr(settings, 'azure_api_key', None) and getattr(settings, 'azure_endpoint', None)),
"email": bool(getattr(settings, "email_host", None)),
"s3": bool(getattr(settings, "s3_bucket_name", None)),
"dropbox": bool(getattr(settings, "dropbox_refresh_token", None)),
"onedrive": bool(getattr(settings, "onedrive_refresh_token", None)),
"nextcloud": bool(getattr(settings, "nextcloud_upload_url", None)),
"sftp": bool(getattr(settings, "sftp_host", None)),
"paperless": bool(getattr(settings, "paperless_host", None)),
"google_drive": bool(getattr(settings, "google_drive_credentials_json", None)),
"uptime_kuma": bool(getattr(settings, "uptime_kuma_url", None)),
"auth": bool(getattr(settings, "authentik_config_url", None)),
"openai": bool(getattr(settings, "openai_api_key", None)),
"azure": bool(getattr(settings, "azure_api_key", None) and getattr(settings, "azure_endpoint", None)),
},
"imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)),
"imap_enabled": bool(getattr(settings, "imap1_host", None) or getattr(settings, "imap2_host", None)),
}
return {
"status": "success",
"settings": safe_settings,
"message": "Full settings have been dumped to application logs"
"message": "Full settings have been dumped to application logs",
}
@router.post("/diagnostic/test-notification")
@require_login
async def test_notification(request: Request):
# Add request_time to request.state
import datetime
request.state.request_time = datetime.datetime.utcnow().isoformat()
"""
Send a test notification through all configured notification channels
@@ -62,20 +68,23 @@ async def test_notification(request: Request):
from app.utils.notification import send_notification
try:
notification_urls = getattr(settings, 'notification_urls', [])
notification_urls = getattr(settings, "notification_urls", [])
if not notification_urls:
return {
"status": "warning",
"message": "No notification services configured. Add notification URLs to your configuration."
"message": "No notification services configured. Add notification URLs to your configuration.",
}
# Send a test notification
hostname = settings.external_hostname or "Document Processor"
result = send_notification(
title=f"Test Notification from {hostname}",
message=f"This is a test notification sent at {request.state.request_time}. If you're receiving this, notifications are working!",
message=(
f"This is a test notification sent at {request.state.request_time}. "
"If you're receiving this, notifications are working!"
),
notification_type="success",
tags=["test", "notification", "diagnostic"]
tags=["test", "notification", "diagnostic"],
)
if result:
@@ -83,18 +92,15 @@ async def test_notification(request: Request):
return {
"status": "success",
"message": f"Test notification sent successfully to {len(notification_urls)} service(s)",
"services_count": len(notification_urls)
"services_count": len(notification_urls),
}
else:
logger.warning("Test notification send attempt returned False")
return {
"status": "error",
"message": "Failed to send test notification. Check application logs for details."
"message": "Failed to send test notification. Check application logs for details.",
}
except Exception as e:
logger.exception(f"Error sending test notification: {e}")
return {
"status": "error",
"message": f"Error sending notification: {str(e)}"
}
return {"status": "error", "message": f"Error sending notification: {str(e)}"}
app/api/dropbox.py
+2 -1
View File
@@ -2,10 +2,11 @@
Dropbox API endpoints
"""
from fastapi import APIRouter, Request, HTTPException, status, Form
import logging
import os
import requests
from fastapi import APIRouter, Form, HTTPException, Request, status
from app.auth import require_login
from app.config import settings
app/api/files.py
+8 -8
View File
@@ -435,15 +435,15 @@ def retry_subtask(
# Map subtask names to their corresponding Celery tasks
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_s3 import upload_to_s3
from app.tasks.upload_to_webdav import upload_to_webdav
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_email import upload_to_email
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_s3 import upload_to_s3
from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_webdav import upload_to_webdav
task_map = {
"upload_to_dropbox": upload_to_dropbox,
app/api/google_drive.py
+3 -2
View File
@@ -2,11 +2,12 @@
Google Drive API endpoints
"""
from fastapi import APIRouter, Request, HTTPException, status, Form
import logging
import os
from typing import Optional
from datetime import datetime
from typing import Optional
from fastapi import APIRouter, Form, HTTPException, Request, status
from app.auth import require_login
from app.config import settings
app/api/logs.py
+53 -64
View File
@@ -1,21 +1,24 @@
"""
Processing logs API endpoints
"""
from fastapi import APIRouter, Request, HTTPException, Depends, Query
from sqlalchemy.orm import Session
from sqlalchemy import desc
from typing import Optional
import logging
from app.auth import require_login
from app.models import ProcessingLog, FileRecord
import logging
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from sqlalchemy import desc
from sqlalchemy.orm import Session
from app.api.common import get_db
from app.auth import require_login
from app.models import FileRecord, ProcessingLog
# Set up logging
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/logs")
@require_login
def list_processing_logs(
@@ -23,7 +26,7 @@ def list_processing_logs(
db: Session = Depends(get_db),
file_id: Optional[int] = Query(None, description="Filter by file ID"),
task_id: Optional[str] = Query(None, description="Filter by task ID"),
limit: int = Query(100, ge=1, le=1000, description="Number of logs to return")
limit: int = Query(100, ge=1, le=1000, description="Number of logs to return"),
):
"""
Returns a JSON list of ProcessingLog entries.
@@ -62,24 +65,23 @@ def list_processing_logs(
# Return a simple list of dicts
result = []
for log in logs:
result.append({
"id": log.id,
"file_id": log.file_id,
"task_id": log.task_id,
"step_name": log.step_name,
"status": log.status,
"message": log.message,
"timestamp": log.timestamp.isoformat() if log.timestamp else None
})
result.append(
{
"id": log.id,
"file_id": log.file_id,
"task_id": log.task_id,
"step_name": log.step_name,
"status": log.status,
"message": log.message,
"timestamp": log.timestamp.isoformat() if log.timestamp else None,
}
)
return result
@router.get("/logs/file/{file_id}")
@require_login
def get_file_processing_logs(
request: Request,
file_id: int,
db: Session = Depends(get_db)
):
def get_file_processing_logs(request: Request, file_id: int, db: Session = Depends(get_db)):
"""
Get all processing logs for a specific file.
Returns logs ordered by timestamp (oldest first to show processing flow).
@@ -89,27 +91,24 @@ def get_file_processing_logs(
# Check if file exists
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record:
raise HTTPException(
status_code=404,
detail=f"File with ID {file_id} not found"
)
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
# Get all logs for this file
logs = db.query(ProcessingLog).filter(
ProcessingLog.file_id == file_id
).order_by(ProcessingLog.timestamp).all()
logs = db.query(ProcessingLog).filter(ProcessingLog.file_id == file_id).order_by(ProcessingLog.timestamp).all()
# Build response
log_list = []
for log in logs:
log_list.append({
"id": log.id,
"task_id": log.task_id,
"step_name": log.step_name,
"status": log.status,
"message": log.message,
"timestamp": log.timestamp.isoformat() if log.timestamp else None
})
log_list.append(
{
"id": log.id,
"task_id": log.task_id,
"step_name": log.step_name,
"status": log.status,
"message": log.message,
"timestamp": log.timestamp.isoformat() if log.timestamp else None,
}
)
return {
"file": {
@@ -117,48 +116,38 @@ def get_file_processing_logs(
"original_filename": file_record.original_filename,
"file_size": file_record.file_size,
"mime_type": file_record.mime_type,
"created_at": file_record.created_at.isoformat() if file_record.created_at else None
"created_at": file_record.created_at.isoformat() if file_record.created_at else None,
},
"logs": log_list,
"total_logs": len(log_list)
"total_logs": len(log_list),
}
@router.get("/logs/task/{task_id}")
@require_login
def get_task_processing_logs(
request: Request,
task_id: str,
db: Session = Depends(get_db)
):
def get_task_processing_logs(request: Request, task_id: str, db: Session = Depends(get_db)):
"""
Get all processing logs for a specific task.
Returns logs ordered by timestamp (oldest first to show processing flow).
"""
# Get all logs for this task
logs = db.query(ProcessingLog).filter(
ProcessingLog.task_id == task_id
).order_by(ProcessingLog.timestamp).all()
logs = db.query(ProcessingLog).filter(ProcessingLog.task_id == task_id).order_by(ProcessingLog.timestamp).all()
if not logs:
raise HTTPException(
status_code=404,
detail=f"No logs found for task {task_id}"
)
raise HTTPException(status_code=404, detail=f"No logs found for task {task_id}")
# Build response
log_list = []
for log in logs:
log_list.append({
"id": log.id,
"file_id": log.file_id,
"step_name": log.step_name,
"status": log.status,
"message": log.message,
"timestamp": log.timestamp.isoformat() if log.timestamp else None
})
log_list.append(
{
"id": log.id,
"file_id": log.file_id,
"step_name": log.step_name,
"status": log.status,
"message": log.message,
"timestamp": log.timestamp.isoformat() if log.timestamp else None,
}
)
return {
"task_id": task_id,
"logs": log_list,
"total_logs": len(log_list)
}
return {"task_id": task_id, "logs": log_list, "total_logs": len(log_list)}
app/api/onedrive.py
+3 -2
View File
@@ -2,12 +2,13 @@
OneDrive API endpoints
"""
from fastapi import APIRouter, Request, HTTPException, status, Form
import logging
import os
import requests
from datetime import datetime, timedelta
import requests
from fastapi import APIRouter, Form, HTTPException, Request, status
from app.auth import require_login
from app.config import settings
from app.utils.oauth_helper import exchange_oauth_token
app/api/openai.py
+9 -17
View File
@@ -1,10 +1,10 @@
"""
OpenAI API endpoints
"""
from fastapi import APIRouter, Request, HTTPException, status
import logging
import os
import requests
from fastapi import APIRouter, Request
from app.auth import require_login
from app.config import settings
@@ -14,6 +14,7 @@ logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/openai/test")
@require_login
async def test_openai_connection(request: Request):
@@ -28,10 +29,7 @@ async def test_openai_connection(request: Request):
# Check if API key is configured
if not settings.openai_api_key:
logger.warning("No OpenAI API key configured")
return {
"status": "error",
"message": "No OpenAI API key is configured"
}
return {"status": "error", "message": "No OpenAI API key is configured"}
# Configure the client
client = openai.OpenAI(api_key=settings.openai_api_key)
@@ -46,7 +44,7 @@ async def test_openai_connection(request: Request):
return {
"status": "success",
"message": "OpenAI API key is valid",
"models_available": len(models.data) if hasattr(models, "data") else "Unknown"
"models_available": len(models.data) if hasattr(models, "data") else "Unknown",
}
except Exception as e:
error_msg = str(e)
@@ -58,18 +56,12 @@ async def test_openai_connection(request: Request):
return {
"status": "error",
"message": f"API key validation failed: {error_msg}",
"is_auth_error": is_auth_error
"is_auth_error": is_auth_error,
}
except ImportError:
logger.exception("OpenAI package not installed")
return {
"status": "error",
"message": "OpenAI package not installed"
}
return {"status": "error", "message": "OpenAI package not installed"}
except Exception as e:
logger.exception("Unexpected error testing OpenAI connection")
return {
"status": "error",
"message": f"Unexpected error: {str(e)}"
}
return {"status": "error", "message": f"Unexpected error: {str(e)}"}
app/api/settings.py
+36 -95
View File
@@ -3,21 +3,22 @@ API endpoints for managing application settings.
"""
import logging
from typing import Dict, Any, Optional
from fastapi import APIRouter, Depends, HTTPException, Request, status
from sqlalchemy.orm import Session
from pydantic import BaseModel, Field
from typing import Any, Dict, Optional
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from app.database import get_db
from app.config import settings
from app.database import get_db
from app.utils.settings_service import (
get_all_settings_from_db,
save_setting_to_db,
SETTING_METADATA,
delete_setting_from_db,
get_all_settings_from_db,
get_setting_metadata,
get_settings_by_category,
save_setting_to_db,
validate_setting_value,
SETTING_METADATA,
)
logger = logging.getLogger(__name__)
@@ -34,21 +35,20 @@ def require_admin(request: Request) -> dict:
"""
user = request.session.get("user")
if not user or not user.get("is_admin"):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Admin access required"
)
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
return user
class SettingUpdate(BaseModel):
"""Model for updating a setting"""
key: str = Field(..., description="Setting key")
value: Optional[str] = Field(None, description="Setting value (None to delete)")
class SettingResponse(BaseModel):
"""Model for setting response"""
key: str
value: Optional[str]
metadata: Dict[str, Any]
@@ -56,17 +56,14 @@ class SettingResponse(BaseModel):
class SettingsListResponse(BaseModel):
"""Model for list of settings"""
settings: Dict[str, Any]
categories: Dict[str, list]
db_settings: Dict[str, str]
@router.get("/", response_model=SettingsListResponse)
async def get_settings(
request: Request,
db: Session = Depends(get_db),
admin: dict = Depends(require_admin)
):
async def get_settings(request: Request, db: Session = Depends(get_db), admin: dict = Depends(require_admin)):
"""
Get all application settings with metadata.
Admin only.
@@ -77,10 +74,7 @@ async def get_settings(
for key in SETTING_METADATA.keys():
if hasattr(settings, key):
value = getattr(settings, key)
current_settings[key] = {
"value": value,
"metadata": get_setting_metadata(key)
}
current_settings[key] = {"value": value, "metadata": get_setting_metadata(key)}
# Get settings stored in database
db_settings = get_all_settings_from_db(db)
@@ -88,26 +82,14 @@ async def get_settings(
# Get settings organized by category
categories = get_settings_by_category()
return SettingsListResponse(
settings=current_settings,
categories=categories,
db_settings=db_settings
)
return SettingsListResponse(settings=current_settings, categories=categories, db_settings=db_settings)
except Exception as e:
logger.error(f"Error retrieving settings: {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to retrieve settings"
)
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to retrieve settings")
@router.get("/{key}", response_model=SettingResponse)
async def get_setting(
key: str,
request: Request,
db: Session = Depends(get_db),
admin: dict = Depends(require_admin)
):
async def get_setting(key: str, request: Request, db: Session = Depends(get_db), admin: dict = Depends(require_admin)):
"""
Get a specific setting by key.
Admin only.
@@ -119,16 +101,11 @@ async def get_setting(
# Get metadata
metadata = get_setting_metadata(key)
return SettingResponse(
key=key,
value=str(value) if value is not None else None,
metadata=metadata
)
return SettingResponse(key=key, value=str(value) if value is not None else None, metadata=metadata)
except Exception as e:
logger.error(f"Error retrieving setting {key}: {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to retrieve setting: {key}"
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to retrieve setting: {key}"
)
@@ -138,7 +115,7 @@ async def update_setting(
setting: SettingUpdate,
request: Request,
db: Session = Depends(get_db),
admin: dict = Depends(require_admin)
admin: dict = Depends(require_admin),
):
"""
Update a specific setting.
@@ -149,17 +126,13 @@ async def update_setting(
if setting.value is not None:
is_valid, error_message = validate_setting_value(key, setting.value)
if not is_valid:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=error_message
)
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=error_message)
# Save to database
success = save_setting_to_db(db, key, setting.value)
if not success:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to save setting to database"
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to save setting to database"
)
# Get metadata
@@ -171,24 +144,20 @@ async def update_setting(
"message": f"Setting '{key}' updated successfully",
"restart_required": restart_required,
"key": key,
"value": setting.value
"value": setting.value,
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error updating setting {key}: {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to update setting: {key}"
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to update setting: {key}"
)
@router.delete("/{key}")
async def delete_setting(
key: str,
request: Request,
db: Session = Depends(get_db),
admin: dict = Depends(require_admin)
key: str, request: Request, db: Session = Depends(get_db), admin: dict = Depends(require_admin)
):
"""
Delete a setting from the database (reverts to environment variable or default).
@@ -197,31 +166,24 @@ async def delete_setting(
try:
success = delete_setting_from_db(db, key)
if not success:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Setting '{key}' not found in database"
)
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Setting '{key}' not found in database")
return {
"success": True,
"message": f"Setting '{key}' deleted from database (will use environment variable or default)"
"message": f"Setting '{key}' deleted from database (will use environment variable or default)",
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error deleting setting {key}: {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to delete setting: {key}"
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to delete setting: {key}"
)
@router.post("/bulk-update")
async def bulk_update_settings(
updates: list[SettingUpdate],
request: Request,
db: Session = Depends(get_db),
admin: dict = Depends(require_admin)
updates: list[SettingUpdate], request: Request, db: Session = Depends(get_db), admin: dict = Depends(require_admin)
):
"""
Update multiple settings at once.
@@ -236,40 +198,19 @@ async def bulk_update_settings(
if update.value is not None:
is_valid, error_message = validate_setting_value(update.key, update.value)
if not is_valid:
errors.append({
"key": update.key,
"error": error_message
})
errors.append({"key": update.key, "error": error_message})
continue
# Save to database
success = save_setting_to_db(db, update.key, update.value)
if success:
results.append({
"key": update.key,
"value": update.value,
"status": "success"
})
results.append({"key": update.key, "value": update.value, "status": "success"})
else:
errors.append({
"key": update.key,
"error": "Failed to save to database"
})
errors.append({"key": update.key, "error": "Failed to save to database"})
except Exception as e:
logger.error(f"Error updating setting {update.key}: {e}")
errors.append({
"key": update.key,
"error": str(e)
})
errors.append({"key": update.key, "error": str(e)})
restart_required = any(
get_setting_metadata(result["key"]).get("restart_required", False)
for result in results
)
restart_required = any(get_setting_metadata(result["key"]).get("restart_required", False) for result in results)
return {
"success": len(errors) == 0,
"updated": results,
"errors": errors,
"restart_required": restart_required
}
return {"success": len(errors) == 0, "updated": results, "errors": errors, "restart_required": restart_required}
app/api/user.py
+7 -2
View File
@@ -1,15 +1,18 @@
"""
User-related API endpoints
"""
from fastapi import APIRouter, Request, HTTPException
from hashlib import md5
import logging
from hashlib import md5
from fastapi import APIRouter, HTTPException, Request
# Set up logging
logger = logging.getLogger(__name__)
router = APIRouter()
async def whoami_handler(request: Request):
"""
Returns user info if logged in, else 401.
@@ -33,11 +36,13 @@ async def whoami_handler(request: Request):
return user_response
# Register the same handler under two different paths
@router.get("/whoami")
async def whoami(request: Request):
return await whoami_handler(request)
@router.get("/auth/whoami")
async def auth_whoami(request: Request):
return await whoami_handler(request)
app/auth.py
+14 -26
View File
@@ -1,13 +1,12 @@
import os
import inspect
import hashlib
import inspect
import pathlib
from functools import wraps
from authlib.integrations.starlette_client import OAuth
from fastapi import APIRouter, Request, status
from starlette.responses import RedirectResponse
from fastapi.templating import Jinja2Templates
import pathlib
from starlette.responses import RedirectResponse
from app.config import settings
@@ -63,11 +62,12 @@ def get_gravatar_url(email):
"""Generate a Gravatar URL for the given email"""
email = email.lower().strip()
# MD5 is used here for Gravatar's URL generation (not for security), so usedforsecurity=False
email_hash = hashlib.md5(email.encode('utf-8'), usedforsecurity=False).hexdigest()
email_hash = hashlib.md5(email.encode("utf-8"), usedforsecurity=False).hexdigest()
return f"https://www.gravatar.com/avatar/{email_hash}?d=identicon"
if AUTH_ENABLED:
@router.get("/login")
async def login(request: Request):
"""Show login page with appropriate authentication options"""
@@ -79,18 +79,15 @@ if AUTH_ENABLED:
"message": request.query_params.get("message"),
"show_oauth": OAUTH_CONFIGURED,
"oauth_provider_name": OAUTH_PROVIDER_NAME,
"app_version": settings.version # Changed from app_version to version
}
"app_version": settings.version, # Changed from app_version to version
},
)
@router.get("/oauth-login")
async def oauth_login(request: Request):
"""Handle OAuth login flow"""
if not OAUTH_CONFIGURED:
return RedirectResponse(
url="/login?error=OAuth+not+configured",
status_code=status.HTTP_302_FOUND
)
return RedirectResponse(url="/login?error=OAuth+not+configured", status_code=status.HTTP_302_FOUND)
redirect_uri = request.url_for("oauth_callback")
return await oauth.authentik.authorize_redirect(request, redirect_uri)
@@ -103,8 +100,7 @@ if AUTH_ENABLED:
userinfo = token.get("userinfo")
if not userinfo:
return RedirectResponse(
url="/login?error=Failed+to+retrieve+user+information",
status_code=status.HTTP_302_FOUND
url="/login?error=Failed+to+retrieve+user+information", status_code=status.HTTP_302_FOUND
)
# Store user info in session
@@ -137,8 +133,7 @@ if AUTH_ENABLED:
except Exception as e:
print(f"OAuth authentication error: {str(e)}")
return RedirectResponse(
url=f"/login?error=Authentication+failed:+{str(e)}",
status_code=status.HTTP_302_FOUND
url=f"/login?error=Authentication+failed:+{str(e)}", status_code=status.HTTP_302_FOUND
)
@router.post("/auth")
@@ -148,8 +143,7 @@ if AUTH_ENABLED:
username = form_data.get("username")
password = form_data.get("password")
if (username == settings.admin_username and
password == settings.admin_password):
if username == settings.admin_username and password == settings.admin_password:
# Create user session
request.session["user"] = {
"id": "admin",
@@ -157,25 +151,19 @@ if AUTH_ENABLED:
"email": f"{username}@local.docuelevate",
"preferred_username": username,
"picture": "/static/images/default-avatar.svg",
"is_admin": True
"is_admin": True,
}
# Redirect to original destination or default
redirect_url = request.session.pop("redirect_after_login", "/upload")
return RedirectResponse(url=redirect_url, status_code=302)
else:
return RedirectResponse(
url="/login?error=Invalid+username+or+password",
status_code=302
)
return RedirectResponse(url="/login?error=Invalid+username+or+password", status_code=302)
@router.get("/logout")
async def logout(request: Request):
"""Handle user logout"""
request.session.pop("user", None)
return RedirectResponse(
url="/login?message=You+have+been+logged+out+successfully",
status_code=302
)
return RedirectResponse(url="/login?message=You+have+been+logged+out+successfully", status_code=302)
@router.get("/api/auth/whoami")
app/celery_app.py
+10 -7
View File
@@ -1,6 +1,8 @@
# app/celery_app.py
from celery import Celery
from celery.signals import task_failure
from app.config import settings
celery = Celery(
@@ -14,29 +16,30 @@ celery = Celery(
celery.conf.broker_connection_retry_on_startup = True
# Set the default queue and routing so that tasks are enqueued on "document_processor"
celery.conf.task_default_queue = 'document_processor'
celery.conf.task_default_queue = "document_processor"
celery.conf.task_routes = {
"app.tasks.*": {"queue": "document_processor"},
}
# Task failure notification handler
from celery.signals import task_failure
@task_failure.connect
def task_failure_handler(sender=None, task_id=None, exception=None, args=None,
kwargs=None, traceback=None, einfo=None, **kw):
def task_failure_handler(
sender=None, task_id=None, exception=None, args=None, kwargs=None, traceback=None, einfo=None, **kw
):
"""Handler for Celery task failures to send notifications"""
if getattr(settings, 'notify_on_task_failure', True):
if getattr(settings, "notify_on_task_failure", True):
try:
# Import here to avoid circular imports
from app.utils.notification import notify_celery_failure
notify_celery_failure(
task_name=sender.name if sender else "Unknown",
task_id=task_id or "N/A",
exc=exception,
args=args or [],
kwargs=kwargs or {}
kwargs=kwargs or {},
)
except Exception as e:
import logging
logging.exception(f"Failed to send task failure notification: {e}")
app/celery_worker.py
+46 -39
View File
@@ -1,65 +1,72 @@
#!/usr/bin/env python3
from app.config import settings
from celery.schedules import crontab
# Ensure tasks are loaded
from app import tasks # noqa: F401 - Imports app/tasks.py so Celery can register tasks
# Import the shared Celery instance
from app.celery_app import celery
# Ensure tasks are loaded
from app import tasks # <— This imports app/tasks.py so Celery can register tasks
from app.config import settings
from app.tasks.check_credentials import check_credentials
from app.tasks.convert_to_pdf import convert_to_pdf # noqa: F401
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf # noqa: F401
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt # noqa: F401
from app.tasks.imap_tasks import pull_all_inboxes # noqa: F401
# **Ensure all tasks are imported before Celery starts**
from app.tasks.process_document import process_document
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence
from app.tasks.rotate_pdf_pages import rotate_pdf_pages
from app.tasks.refine_text_with_gpt import refine_text_with_gpt
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
from app.tasks.convert_to_pdf import convert_to_pdf
from app.tasks.process_document import process_document # noqa: F401
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence # noqa: F401
from app.tasks.refine_text_with_gpt import refine_text_with_gpt # noqa: F401
from app.tasks.rotate_pdf_pages import rotate_pdf_pages # noqa: F401
from app.tasks.send_to_all import send_to_all_destinations # noqa: F401
# Import new send tasks
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_webdav import upload_to_webdav
from app.tasks.upload_to_s3 import upload_to_s3
from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_email import upload_to_email
from app.tasks.imap_tasks import pull_all_inboxes
from app.tasks.send_to_all import send_to_all_destinations
from app.tasks.uptime_kuma_tasks import ping_uptime_kuma
from app.tasks.check_credentials import check_credentials
from app.tasks.upload_to_dropbox import upload_to_dropbox # noqa: F401
from app.tasks.upload_to_email import upload_to_email # noqa: F401
from app.tasks.upload_to_ftp import upload_to_ftp # noqa: F401
from app.tasks.upload_to_google_drive import upload_to_google_drive # noqa: F401
from app.tasks.upload_to_nextcloud import upload_to_nextcloud # noqa: F401
from app.tasks.upload_to_onedrive import upload_to_onedrive # noqa: F401
from app.tasks.upload_to_paperless import upload_to_paperless # noqa: F401
from app.tasks.upload_to_s3 import upload_to_s3 # noqa: F401
from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401
from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401
from app.tasks.uptime_kuma_tasks import ping_uptime_kuma # noqa: F401
celery.conf.task_routes = {
"app.tasks.*": {"queue": "default"},
}
@celery.task
def test_task():
return "Celery is working!"
# If you want Celery Beat to run the poll task every minute, add:
from celery.schedules import crontab
# Run the check_credentials task at startup
check_credentials.apply_async(countdown=10) # Run 10 seconds after worker starts
celery.conf.beat_schedule = {
"poll-inboxes-every-minute": {
"task": "app.tasks.imap_tasks.pull_all_inboxes",
"schedule": crontab(minute="*/1"), # every 1 minute
"options": {"expires": 55}, # Ensure tasks don't pile up
} if (settings.imap1_host or settings.imap2_host) else None,
"poll-inboxes-every-minute": (
{
"task": "app.tasks.imap_tasks.pull_all_inboxes",
"schedule": crontab(minute="*/1"), # every 1 minute
"options": {"expires": 55}, # Ensure tasks don't pile up
}
if (settings.imap1_host or settings.imap2_host)
else None
),
# Add Uptime Kuma ping task if configured
"ping-uptime-kuma": {
"task": "app.tasks.uptime_kuma_tasks.ping_uptime_kuma",
"schedule": crontab(minute=f"*/{settings.uptime_kuma_ping_interval}"),
"options": {"expires": 55}, # Ensure tasks don't pile up
} if settings.uptime_kuma_url else None,
"ping-uptime-kuma": (
{
"task": "app.tasks.uptime_kuma_tasks.ping_uptime_kuma",
"schedule": crontab(minute=f"*/{settings.uptime_kuma_ping_interval}"),
"options": {"expires": 55}, # Ensure tasks don't pile up
}
if settings.uptime_kuma_url
else None
),
# Check credentials every 5 minutes
"check-credentials-regularly": {
"task": "app.tasks.check_credentials.check_credentials",
@@ -71,7 +78,7 @@ celery.conf.beat_schedule = {
"task": "app.tasks.check_credentials.check_credentials",
"schedule": crontab(hour="0", minute="0"), # Midnight
"options": {"expires": 3600}, # 1 hour expiry
}
},
}
# Remove None entries from beat_schedule
app/config.py
+1 -2
View File
@@ -1,8 +1,7 @@
#!/usr/bin/env python3
import os
from datetime import datetime
from typing import Any, Dict, List, Optional, Union
from typing import Any, List, Optional, Union
from pydantic import Field, validator
from pydantic_settings import BaseSettings
app/database.py
+2 -2
View File
@@ -1,12 +1,12 @@
# app/database.py
import os
import logging
import os
from sqlalchemy import create_engine, exc
from sqlalchemy.engine.url import make_url
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.engine.url import make_url
from app.config import settings
app/frontend.py
+2 -2
View File
@@ -2,9 +2,9 @@
Frontend routes for the application.
This module is now a re-export of the modularized view routers.
"""
# Import and re-export the router from the views package
from app.views import router
from app.views import router # noqa: F401
# Keep the original router name for compatibility
# This allows existing imports in main.py to continue working
app/main.py
+26 -34
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
import os
import logging
import os
import pathlib
from contextlib import asynccontextmanager
@@ -8,21 +8,20 @@ from fastapi import FastAPI, HTTPException, Request, status
from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from starlette.middleware.sessions import SessionMiddleware
from starlette.config import Config
from starlette.middleware.sessions import SessionMiddleware
from starlette.middleware.trustedhost import TrustedHostMiddleware
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
from pathlib import Path
from app.database import init_db
from app.api import router as api_router
from app.auth import router as auth_router
from app.config import settings
from app.database import init_db
from app.utils.config_validator import check_all_configs
from app.utils.notification import init_apprise, send_notification, notify_startup, notify_shutdown
from app.utils.notification import init_apprise, notify_shutdown, notify_startup
# Import the routers - now using views directly instead of frontend
from app.views import router as frontend_router
from app.api import router as api_router
from app.auth import router as auth_router
# Explicitly include the files router
from app.views.files import router as files_router
@@ -32,8 +31,13 @@ config = Config(".env")
# Use settings.session_secret which has proper validation
# Fallback to raising an error if not set when auth is enabled
if settings.auth_enabled and not settings.session_secret:
raise ValueError("SESSION_SECRET must be set when AUTH_ENABLED=True. Generate one with: python -c 'import secrets; print(secrets.token_hex(32))'")
SESSION_SECRET = settings.session_secret or "INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS"
raise ValueError(
"SESSION_SECRET must be set when AUTH_ENABLED=True. "
"Generate one with: python -c 'import secrets; print(secrets.token_hex(32))'"
)
SESSION_SECRET = (
settings.session_secret or "INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS"
)
@asynccontextmanager
@@ -60,14 +64,15 @@ async def lifespan(app: FastAPI):
# Force settings dump to log for troubleshooting
from app.utils.config_validator import dump_all_settings
dump_all_settings()
# Validate configuration
config_issues = check_all_configs()
# Log overall status
has_issues = any(config_issues['email']) or any(
len(issues) > 0 for provider, issues in config_issues['storage'].items()
has_issues = any(config_issues["email"]) or any(
len(issues) > 0 for provider, issues in config_issues["storage"].items()
)
if has_issues:
logging.warning("Application started with configuration issues - some features may be unavailable")
@@ -101,11 +106,7 @@ app.add_middleware(SessionMiddleware, secret_key=SESSION_SECRET)
app.add_middleware(ProxyHeadersMiddleware, trusted_hosts="*")
# 3) (Optional but recommended) Restrict valid hosts:
app.add_middleware(TrustedHostMiddleware, allowed_hosts=[
settings.external_hostname,
"localhost",
"127.0.0.1"
])
app.add_middleware(TrustedHostMiddleware, allowed_hosts=[settings.external_hostname, "localhost", "127.0.0.1"])
# Mount the static files directory
static_dir = pathlib.Path(__file__).parents[1] / "frontend" / "static"
@@ -114,6 +115,7 @@ if os.path.exists(static_dir):
else:
print(f"WARNING: Static directory not found at {static_dir}. Static files will not be served.")
# Custom exception handlers that return JSON for API routes and HTML for frontend routes
@app.exception_handler(HTTPException)
async def http_exception_handler(request: Request, exc: HTTPException):
@@ -123,30 +125,24 @@ async def http_exception_handler(request: Request, exc: HTTPException):
"""
# For API routes, always return JSON
if request.url.path.startswith("/api/"):
return JSONResponse(
status_code=exc.status_code,
content={"detail": exc.detail}
)
return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})
# For frontend routes, return appropriate HTML templates
templates = Jinja2Templates(directory=str(static_dir.parent / "templates"))
# Handle 404 errors with a custom template
if exc.status_code == 404:
return templates.TemplateResponse(
"404.html",
{"request": request},
status_code=status.HTTP_404_NOT_FOUND
)
return templates.TemplateResponse("404.html", {"request": request}, status_code=status.HTTP_404_NOT_FOUND)
# For other HTTP errors, we could create specific templates or use a generic one
# For now, return a simple error page
return templates.TemplateResponse(
"404.html", # Reuse 404 template for other errors, or create a generic error template
{"request": request},
status_code=exc.status_code
status_code=exc.status_code,
)
@app.exception_handler(500)
async def custom_500_handler(request: Request, exc: Exception):
"""
@@ -156,27 +152,23 @@ async def custom_500_handler(request: Request, exc: Exception):
# For API routes, return JSON instead of HTML
if request.url.path.startswith("/api/"):
return JSONResponse(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
content={"detail": "Internal server error"}
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={"detail": "Internal server error"}
)
# Serve the 500 template for non-API routes
templates = Jinja2Templates(directory=str(static_dir.parent / "templates"))
return templates.TemplateResponse(
"500.html",
{"request": request, "exc": exc},
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
"500.html", {"request": request, "exc": exc}, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
)
@app.get("/test-500")
def test_500():
raise RuntimeError("Testing forced 500 error!")
# Include the routers
app.include_router(frontend_router)
app.include_router(files_router) # Explicitly include the files router
app.include_router(auth_router)
app.include_router(api_router, prefix="/api")
app/models.py
+9 -5
View File
@@ -1,10 +1,10 @@
# app/models.py
#!/usr/bin/env python3
from sqlalchemy import Column, String, Integer, DateTime, func, ForeignKey
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, func
from app.database import Base
class DocumentMetadata(Base):
__tablename__ = "documents"
@@ -15,6 +15,7 @@ class DocumentMetadata(Base):
tags = Column(String)
summary = Column(String)
class FileRecord(Base):
__tablename__ = "files"
@@ -38,18 +39,21 @@ class FileRecord(Base):
# Timestamp when we inserted this record
created_at = Column(DateTime(timezone=True), server_default=func.now())
class ProcessingLog(Base):
__tablename__ = "processing_logs"
id = Column(Integer, primary_key=True, index=True)
file_id = Column(Integer, ForeignKey("files.id"), nullable=True) # Optional file association
task_id = Column(String, index=True) # Celery task ID
step_name = Column(String) # e.g., "OCR", "convert_to_pdf", "upload_s3"
status = Column(String) # "pending", "in_progress", "success", "failure"
step_name = Column(String) # e.g., "OCR", "convert_to_pdf", "upload_s3"
status = Column(String) # "pending", "in_progress", "success", "failure"
message = Column(String, nullable=True) # Error text or success note
timestamp = Column(DateTime(timezone=True), server_default=func.now())
class ApplicationSettings(Base):
"""Store application settings in database with precedence over environment variables"""
__tablename__ = "application_settings"
id = Column(Integer, primary_key=True, index=True)
app/routes/license_routes.py
+4 -3
View File
@@ -1,10 +1,11 @@
from fastapi import APIRouter, HTTPException
from fastapi.responses import PlainTextResponse, HTMLResponse
from pathlib import Path
import os
from fastapi import APIRouter, HTTPException
from fastapi.responses import PlainTextResponse
router = APIRouter()
@router.get("/licenses/lgpl.txt", response_class=PlainTextResponse)
async def get_lgpl_license():
"""
app/tasks/__init__.py
+2 -2
View File
@@ -1,3 +1,3 @@
# Import tasks so they can be discovered by Celery
from app.tasks.process_document import process_document
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence
from app.tasks.process_document import process_document # noqa: F401
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence # noqa: F401
app/tasks/check_credentials.py
+48 -36
View File
@@ -1,26 +1,29 @@
import logging
from app.celery_app import celery
from app.config import settings
from app.utils.notification import notify_credential_failure
import time
import os
import json
import asyncio
import inspect
import json
import logging
import os
import time
# Import the test functions from API routes
from app.api.openai import test_openai_connection
from app.api.azure import test_azure_connection
from app.api.dropbox import test_dropbox_token
from app.api.google_drive import test_google_drive_token
from app.api.onedrive import test_onedrive_token
# Import the test functions from API routes
from app.api.openai import test_openai_connection
from app.celery_app import celery
from app.config import settings
# Import config validation utilities
from app.utils.config_validator import validate_storage_configs, get_provider_status
from app.utils.config_validator import get_provider_status, validate_storage_configs
from app.utils.notification import notify_credential_failure
# Create an enhanced mock Request object for API functions that expect it
class MockRequest:
"""Mock request object with session and other attributes needed for API functions"""
def __init__(self):
self.session = {"user": {"id": "credential_checker", "name": "System Credential Checker"}}
self.app = None
@@ -34,16 +37,18 @@ class MockRequest:
async def form(self):
return {}
logger = logging.getLogger(__name__)
# Path to store failure counts
FAILURE_STATE_FILE = os.path.join(settings.workdir, 'credential_failures.json')
FAILURE_STATE_FILE = os.path.join(settings.workdir, "credential_failures.json")
def get_failure_state():
"""Read the failure state from file"""
try:
if os.path.exists(FAILURE_STATE_FILE):
with open(FAILURE_STATE_FILE, 'r') as f:
with open(FAILURE_STATE_FILE, "r") as f:
return json.load(f)
except Exception as e:
logger.error(f"Error reading failure state file: {e}")
@@ -51,14 +56,16 @@ def get_failure_state():
# Default empty state
return {}
def save_failure_state(state):
"""Save failure state to file"""
try:
with open(FAILURE_STATE_FILE, 'w') as f:
with open(FAILURE_STATE_FILE, "w") as f:
json.dump(state, f)
except Exception as e:
logger.error(f"Error saving failure state file: {e}")
# Helper function to get the inner function without the decorator
def unwrap_decorated_function(func):
"""Get the original function from a decorated function"""
@@ -66,6 +73,7 @@ def unwrap_decorated_function(func):
return unwrap_decorated_function(func.__wrapped__)
return func
# Create synchronous versions of the test functions that bypass authentication
def sync_test_openai_connection():
"""Synchronous wrapper for the OpenAI test function that bypasses auth"""
@@ -76,6 +84,7 @@ def sync_test_openai_connection():
return asyncio.run(inner_func(request))
return inner_func(request)
def sync_test_azure_connection():
"""Synchronous wrapper for the Azure test function that bypasses auth"""
inner_func = unwrap_decorated_function(test_azure_connection)
@@ -84,6 +93,7 @@ def sync_test_azure_connection():
return asyncio.run(inner_func(request))
return inner_func(request)
def sync_test_dropbox_token():
"""Synchronous wrapper for the Dropbox test function that bypasses auth"""
inner_func = unwrap_decorated_function(test_dropbox_token)
@@ -92,6 +102,7 @@ def sync_test_dropbox_token():
return asyncio.run(inner_func(request))
return inner_func(request)
def sync_test_google_drive_token():
"""Synchronous wrapper for the Google Drive test function that bypasses auth"""
inner_func = unwrap_decorated_function(test_google_drive_token)
@@ -100,6 +111,7 @@ def sync_test_google_drive_token():
return asyncio.run(inner_func(request))
return inner_func(request)
def sync_test_onedrive_token():
"""Synchronous wrapper for the OneDrive test function that bypasses auth"""
inner_func = unwrap_decorated_function(test_onedrive_token)
@@ -108,6 +120,7 @@ def sync_test_onedrive_token():
return asyncio.run(inner_func(request))
return inner_func(request)
@celery.task
def check_credentials():
"""Check all configured credentials and notify if any are invalid"""
@@ -129,32 +142,32 @@ def check_credentials():
"name": "OpenAI",
"check_func": sync_test_openai_connection,
"configured": provider_status.get("OpenAI", {}).get("configured", False),
"config_issues": [] # OpenAI isn't in storage_configs
"config_issues": [], # OpenAI isn't in storage_configs
},
{
"name": "Azure Document Intelligence",
"check_func": sync_test_azure_connection,
"configured": provider_status.get("Azure AI", {}).get("configured", False),
"config_issues": [] # Azure isn't in storage_configs
"config_issues": [], # Azure isn't in storage_configs
},
{
"name": "Dropbox",
"check_func": sync_test_dropbox_token,
"configured": provider_status.get("Dropbox", {}).get("configured", False),
"config_issues": storage_configs.get("dropbox", [])
"config_issues": storage_configs.get("dropbox", []),
},
{
"name": "Google Drive",
"check_func": sync_test_google_drive_token,
"configured": provider_status.get("Google Drive", {}).get("configured", False),
"config_issues": storage_configs.get("google_drive", [])
"config_issues": storage_configs.get("google_drive", []),
},
{
"name": "OneDrive",
"check_func": sync_test_onedrive_token,
"configured": provider_status.get("OneDrive", {}).get("configured", False),
"config_issues": storage_configs.get("onedrive", [])
}
"config_issues": storage_configs.get("onedrive", []),
},
]
# Check each service
@@ -168,13 +181,10 @@ def check_credentials():
# Skip services that aren't configured
if not service["configured"]:
config_issues = service["config_issues"]
issue_msg = f"Not properly configured" + (f": {', '.join(config_issues)}" if config_issues else "")
issue_msg = "Not properly configured" + (f": {', '.join(config_issues)}" if config_issues else "")
logger.info(f"Skipping {service_name}: {issue_msg}")
results[service_name] = {
"status": "unconfigured",
"message": issue_msg
}
results[service_name] = {"status": "unconfigured", "message": issue_msg}
continue
try:
@@ -186,10 +196,7 @@ def check_credentials():
error_message = result.get("message", "Unknown error")
# Store the result
results[service_name] = {
"status": "valid" if is_valid else "invalid",
"message": error_message
}
results[service_name] = {"status": "valid" if is_valid else "invalid", "message": error_message}
if not is_valid:
failures.append(service_name)
@@ -204,10 +211,15 @@ def check_credentials():
notify_credential_failure(service_name, error_message)
service_state["last_notified"] = current_time
service_state["recovered"] = False
logger.warning(f"{service_name} credentials check failed ({service_state['count']} times): {error_message}")
logger.warning(
f"{service_name} credentials check failed ({service_state['count']} times): {error_message}"
)
else:
# We're in cooldown mode
logger.warning(f"{service_name} credentials check failed ({service_state['count']} times): {error_message} - notification suppressed")
logger.warning(
f"{service_name} credentials check failed ({service_state['count']} times): "
f"{error_message} - notification suppressed"
)
# Update failure state
failure_state[service_name] = service_state
@@ -243,10 +255,7 @@ def check_credentials():
failure_state[service_name] = service_state
# Store the error result
results[service_name] = {
"status": "error",
"message": error_message
}
results[service_name] = {"status": "error", "message": error_message}
# Save updated failure state
save_failure_state(failure_state)
@@ -256,12 +265,15 @@ def check_credentials():
num_configured = len(configured_services)
# Summarize results
logger.info(f"Credential check completed. Configured services: {num_configured}, Valid: {num_configured - len(failures)}, Invalid: {len(failures)}")
logger.info(
f"Credential check completed. Configured services: {num_configured}, "
f"Valid: {num_configured - len(failures)}, Invalid: {len(failures)}"
)
return {
"checked": num_configured,
"unconfigured": len(services) - num_configured,
"failures": len(failures),
"results": results,
"failure_state": failure_state
"failure_state": failure_state,
}
app/tasks/embed_metadata_into_pdf.py
+44 -20
View File
@@ -1,20 +1,21 @@
#!/usr/bin/env python3
import json
import logging
import os
import shutil
import tempfile
import logging
import PyPDF2 # Replace fitz with PyPDF2
import json
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.finalize_document_storage import finalize_document_storage
# Import the shared Celery instance
from app.celery_app import celery
from app.utils import log_task_progress
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord
from app.tasks.finalize_document_storage import finalize_document_storage
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
@@ -26,6 +27,7 @@ logger = logging.getLogger(__name__)
TMP_SUBDIR = "tmp"
PROCESSED_SUBDIR = "processed"
def unique_filepath(directory, base_filename, extension=".pdf"):
"""
Returns a unique filepath in the specified directory.
@@ -41,6 +43,7 @@ def unique_filepath(directory, base_filename, extension=".pdf"):
return candidate
counter += 1
def persist_metadata(metadata, final_pdf_path):
"""
Saves the metadata dictionary to a JSON file with the same base name as the final PDF.
@@ -53,6 +56,7 @@ def persist_metadata(metadata, final_pdf_path):
json.dump(metadata, f, ensure_ascii=False, indent=2)
return json_path
@celery.task(base=BaseTaskWithRetry, bind=True)
def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, metadata: dict, file_id: int = None):
"""
@@ -70,7 +74,13 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting metadata embedding for: {local_file_path}")
log_task_progress(task_id, "embed_metadata_into_pdf", "in_progress", f"Embedding metadata into {os.path.basename(local_file_path)}", file_id=file_id)
log_task_progress(
task_id,
"embed_metadata_into_pdf",
"in_progress",
f"Embedding metadata into {os.path.basename(local_file_path)}",
file_id=file_id,
)
# Get file_id from database if not provided (fallback only, prefer passing file_id explicitly)
if file_id is None:
@@ -93,7 +103,7 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
original_file = local_file_path
# Create a temporary file with the same extension as the original
_, ext = os.path.splitext(local_file_path)
tmp_file = tempfile.NamedTemporaryFile(mode='wb', suffix=ext, prefix='processed_', delete=False)
tmp_file = tempfile.NamedTemporaryFile(mode="wb", suffix=ext, prefix="processed_", delete=False)
processed_file = tmp_file.name
tmp_file.close()
@@ -105,7 +115,7 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
log_task_progress(task_id, "modify_pdf", "in_progress", "Modifying PDF metadata", file_id=file_id)
# Open the PDF and modify metadata
with open(processed_file, 'rb') as file:
with open(processed_file, "rb") as file:
pdf_reader = PyPDF2.PdfReader(file)
pdf_writer = PyPDF2.PdfWriter()
@@ -114,15 +124,17 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
pdf_writer.add_page(page)
# Set PDF metadata
pdf_writer.add_metadata({
"/Title": metadata.get("filename", "Unknown Document"),
"/Author": metadata.get("absender", "Unknown"),
"/Subject": metadata.get("document_type", "Unknown"),
"/Keywords": ", ".join(metadata.get("tags", []))
})
pdf_writer.add_metadata(
{
"/Title": metadata.get("filename", "Unknown Document"),
"/Author": metadata.get("absender", "Unknown"),
"/Subject": metadata.get("document_type", "Unknown"),
"/Keywords": ", ".join(metadata.get("tags", [])),
}
)
# Write the modified PDF
with open(processed_file, 'wb') as output_file:
with open(processed_file, "wb") as output_file:
pdf_writer.write(output_file)
logger.info(f"[{task_id}] Metadata embedded successfully in {processed_file}")
@@ -139,24 +151,36 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
final_file_path = unique_filepath(final_dir, suggested_filename, extension=".pdf")
logger.info(f"[{task_id}] Moving file to: {final_file_path}")
log_task_progress(task_id, "move_to_processed", "in_progress", f"Moving to processed: {suggested_filename}.pdf", file_id=file_id)
log_task_progress(
task_id,
"move_to_processed",
"in_progress",
f"Moving to processed: {suggested_filename}.pdf",
file_id=file_id,
)
# Move the processed file using shutil.move to handle cross-device moves.
shutil.move(processed_file, final_file_path)
# Ensure the temporary file is deleted if it still exists.
if os.path.exists(processed_file):
os.remove(processed_file)
log_task_progress(task_id, "move_to_processed", "success", f"Moved to: {os.path.basename(final_file_path)}", file_id=file_id)
log_task_progress(
task_id, "move_to_processed", "success", f"Moved to: {os.path.basename(final_file_path)}", file_id=file_id
)
# Persist the metadata into a JSON file with the same base name.
logger.info(f"[{task_id}] Persisting metadata to JSON")
log_task_progress(task_id, "save_metadata_json", "in_progress", "Saving metadata JSON", file_id=file_id)
json_path = persist_metadata(metadata, final_file_path)
logger.info(f"[{task_id}] Metadata persisted to {json_path}")
log_task_progress(task_id, "save_metadata_json", "success", f"Saved: {os.path.basename(json_path)}", file_id=file_id)
log_task_progress(
task_id, "save_metadata_json", "success", f"Saved: {os.path.basename(json_path)}", file_id=file_id
)
# Trigger the next step: final storage.
logger.info(f"[{task_id}] Queueing final storage task")
log_task_progress(task_id, "embed_metadata_into_pdf", "success", "Metadata embedded, queuing finalization", file_id=file_id)
log_task_progress(
task_id, "embed_metadata_into_pdf", "success", "Metadata embedded, queuing finalization", file_id=file_id
)
finalize_document_storage.delay(original_file, final_file_path, metadata, file_id=file_id)
# After triggering final storage, delete the original file if it is in workdir/tmp.
app/tasks/extract_metadata_with_gpt.py
+58 -49
View File
@@ -1,33 +1,32 @@
#!/usr/bin/env python3
import json
import re
import logging
import os
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
import re
import openai
# Import the shared Celery instance
from app.celery_app import celery
import openai
import logging
from app.utils import log_task_progress
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
# Initialize OpenAI client dynamically with better error handling
try:
client = openai.OpenAI(
api_key=settings.openai_api_key,
base_url=settings.openai_base_url
)
client = openai.OpenAI(api_key=settings.openai_api_key, base_url=settings.openai_base_url)
logger.info("OpenAI client initialized successfully")
except Exception as e:
logger.error(f"Failed to initialize OpenAI client: {e}")
client = None
def extract_json_from_text(text):
"""
Try to extract a JSON object from the text.
@@ -42,15 +41,18 @@ def extract_json_from_text(text):
start = text.find("{")
end = text.rfind("}")
if start != -1 and end != -1 and end > start:
return text[start:end+1]
return text[start : end + 1]
return None
@celery.task(base=BaseTaskWithRetry, bind=True)
def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: int = None):
"""Uses OpenAI to classify document metadata."""
task_id = self.request.id
logger.info(f"[{task_id}] Starting metadata extraction for: {filename}")
log_task_progress(task_id, "extract_metadata_with_gpt", "in_progress", f"Extracting metadata for {filename}", file_id=file_id)
log_task_progress(
task_id, "extract_metadata_with_gpt", "in_progress", f"Extracting metadata for {filename}", file_id=file_id
)
# Get file_id from database if not provided
if file_id is None:
@@ -62,37 +64,38 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
if file_record:
file_id = file_record.id
prompt = f"""
You are a specialized document analyzer trained to extract structured metadata from documents.
Your task is to analyze the given text and return a well-structured JSON object.
Extract and return the following fields:
1. **filename**: Machine-readable filename (YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, periods, and underscores).
2. **empfaenger**: The recipient, or "Unknown" if not found.
3. **absender**: The sender, or "Unknown" if not found.
4. **correspondent**: The entity or company that issued the document (shortest possible name, e.g., "Amazon" instead of "Amazon EU SARL, German branch").
5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].
6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, Private_Korrespondenz, Sonstige_Informationen].
7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).
8. **tags**: A list of up to 4 relevant thematic keywords.
9. **language**: Detected document language (ISO 639-1 code, e.g., "de" or "en").
10. **title**: A human-readable title summarizing the document content.
11. **confidence_score**: A numeric value (0-100) indicating the confidence level of the extracted metadata.
12. **reference_number**: Extracted invoice/order/reference number if available.
13. **monetary_amounts**: A list of key monetary values detected in the document.
### Important Rules:
- **OCR Correction**: Assume the text has been corrected for OCR errors.
- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.
- **Title**: Concise, no addresses, and contains key identifying features.
- **Date Selection**: Use the most relevant date if multiple are found.
- **Output Language**: Maintain the document's original language.
Extracted text:
{cleaned_text}
Return only valid JSON with no additional commentary.
"""
prompt = (
"You are a specialized document analyzer trained to extract structured metadata from documents.\n"
"Your task is to analyze the given text and return a well-structured JSON object.\n\n"
"Extract and return the following fields:\n"
"1. **filename**: Machine-readable filename "
"(YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, periods, and underscores).\n"
"2. **empfaenger**: The recipient, or \"Unknown\" if not found.\n"
"3. **absender**: The sender, or \"Unknown\" if not found.\n"
"4. **correspondent**: The entity or company that issued the document "
"(shortest possible name, e.g., \"Amazon\" instead of \"Amazon EU SARL, German branch\").\n"
"5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, "
"Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].\n"
"6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, "
"Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, "
"Private_Korrespondenz, Sonstige_Informationen].\n"
"7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).\n"
"8. **tags**: A list of up to 4 relevant thematic keywords.\n"
"9. **language**: Detected document language (ISO 639-1 code, e.g., \"de\" or \"en\").\n"
"10. **title**: A human-readable title summarizing the document content.\n"
"11. **confidence_score**: A numeric value (0-100) indicating the confidence level "
"of the extracted metadata.\n"
"12. **reference_number**: Extracted invoice/order/reference number if available.\n"
"13. **monetary_amounts**: A list of key monetary values detected in the document.\n\n"
"### Important Rules:\n"
"- **OCR Correction**: Assume the text has been corrected for OCR errors.\n"
"- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.\n"
"- **Title**: Concise, no addresses, and contains key identifying features.\n"
"- **Date Selection**: Use the most relevant date if multiple are found.\n"
"- **Output Language**: Maintain the document's original language.\n\n"
f"Extracted text:\n{cleaned_text}\n\n"
"Return only valid JSON with no additional commentary.\n"
)
try:
logger.info(f"[{task_id}] Sending classification request for {filename}...")
@@ -101,9 +104,9 @@ Return only valid JSON with no additional commentary.
model=settings.openai_model,
messages=[
{"role": "system", "content": "You are an intelligent document classifier."},
{"role": "user", "content": prompt}
{"role": "user", "content": prompt},
],
temperature=0
temperature=0,
)
content = completion.choices[0].message.content
@@ -113,16 +116,22 @@ Return only valid JSON with no additional commentary.
json_text = extract_json_from_text(content)
if not json_text:
logger.error(f"[{task_id}] Could not find valid JSON in GPT response for {filename}.")
log_task_progress(task_id, "extract_metadata_with_gpt", "failure", "Invalid JSON in response", file_id=file_id)
log_task_progress(
task_id, "extract_metadata_with_gpt", "failure", "Invalid JSON in response", file_id=file_id
)
return {}
metadata = json.loads(json_text)
logger.info(f"[{task_id}] Extracted metadata: {metadata}")
log_task_progress(task_id, "parse_metadata", "success", f"Parsed metadata: {list(metadata.keys())}", file_id=file_id)
log_task_progress(
task_id, "parse_metadata", "success", f"Parsed metadata: {list(metadata.keys())}", file_id=file_id
)
# Trigger the next step: embedding metadata into the PDF
logger.info(f"[{task_id}] Queueing metadata embedding task")
log_task_progress(task_id, "extract_metadata_with_gpt", "success", "Metadata extracted, queuing embed task", file_id=file_id)
log_task_progress(
task_id, "extract_metadata_with_gpt", "success", "Metadata extracted, queuing embed task", file_id=file_id
)
embed_metadata_into_pdf.delay(filename, cleaned_text, metadata, file_id)
return {"s3_file": filename, "metadata": metadata}
app/tasks/finalize_document_storage.py
+23 -22
View File
@@ -2,21 +2,22 @@
import logging
import os
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
# Import the shared Celery instance
from app.celery_app import celery
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord
from app.tasks.retry_config import BaseTaskWithRetry
# Import the aggregator task and validator
from app.tasks.send_to_all import send_to_all_destinations, get_configured_services_from_validator
# Import notification utility
from app.utils.notification import notify_file_processed
from app.tasks.send_to_all import get_configured_services_from_validator, send_to_all_destinations
# Import database and logging utils from main
from app.utils import log_task_progress
from app.database import SessionLocal
from app.models import FileRecord
# Import notification utility
from app.utils.notification import notify_file_processed
logger = logging.getLogger(__name__)
@@ -32,16 +33,20 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
logger.info(f"[{task_id}] Finalizing document storage for {processed_file}")
# 1. Update Database Status (From Main)
log_task_progress(task_id, "finalize_document_storage", "in_progress", f"Finalizing: {os.path.basename(processed_file)}", file_id=file_id)
log_task_progress(
task_id,
"finalize_document_storage",
"in_progress",
f"Finalizing: {os.path.basename(processed_file)}",
file_id=file_id,
)
# Get file_id from database if not provided (fallback logic from Main)
if file_id is None:
with SessionLocal() as db:
# Only as a last resort, try to find by exact match on local_filename
tmp_path = os.path.join(settings.workdir, "tmp", os.path.basename(original_file))
file_record = db.query(FileRecord).filter(
FileRecord.local_filename == tmp_path
).first()
file_record = db.query(FileRecord).filter(FileRecord.local_filename == tmp_path).first()
if file_record:
file_id = file_record.id
@@ -54,7 +59,7 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
for service_name, is_configured in configured_services.items():
if is_configured:
# Format service names for display
display_name = service_name.replace('_', ' ').title()
display_name = service_name.replace("_", " ").title()
configured_destinations.append(display_name)
except Exception as e:
logger.warning(f"[WARNING] Could not determine configured destinations: {e}")
@@ -63,7 +68,9 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
# 3. Queue Uploads (Merged)
# Uses Main branch signature to ensure file_id is passed, but keeps logic structure
logger.info(f"[{task_id}] Queueing uploads to all destinations")
log_task_progress(task_id, "finalize_document_storage", "success", "Queuing uploads to destinations", file_id=file_id)
log_task_progress(
task_id, "finalize_document_storage", "success", "Queuing uploads to destinations", file_id=file_id
)
# Note: send_to_all_destinations is asynchronous and queues upload tasks
# We pass 'True' (delete_after) and 'file_id' as per Main branch requirements
@@ -78,15 +85,9 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
filename = os.path.basename(processed_file)
notify_file_processed(
filename=filename,
file_size=file_size,
metadata=metadata,
destinations=configured_destinations
filename=filename, file_size=file_size, metadata=metadata, destinations=configured_destinations
)
except Exception as e:
logger.warning(f"[WARNING] Failed to send file processed notification: {e}")
return {
"status": "Completed",
"file": processed_file
}
return {"status": "Completed", "file": processed_file}
app/tasks/imap_tasks.py
+17 -22
View File
@@ -1,16 +1,18 @@
#!/usr/bin/env python3
import os
import json
import email
import imaplib
import json
import logging
import redis
import os
import re
from datetime import datetime, timedelta, timezone
import redis
from celery import shared_task
from app.config import settings
from app.tasks.process_document import process_document # Updated import
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
from app.tasks.process_document import process_document # Updated import
logger = logging.getLogger(__name__)
@@ -18,7 +20,7 @@ logger = logging.getLogger(__name__)
redis_client = redis.StrictRedis.from_url(settings.redis_url, decode_responses=True)
LOCK_KEY = "imap_lock" # Unique key for locking
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
# Local cache file for tracking processed emails
CACHE_FILE = os.path.join(settings.workdir, "processed_mails.json")
@@ -141,8 +143,7 @@ def check_and_pull_mailbox(
)
def pull_inbox(mailbox_key, host, port, username, password, use_ssl,
delete_after_process):
def pull_inbox(mailbox_key, host, port, username, password, use_ssl, delete_after_process):
"""
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
and processes attachments while preserving the original unread status.
@@ -153,8 +154,7 @@ def pull_inbox(mailbox_key, host, port, username, password, use_ssl,
For non-Gmail mailboxes, it falls back to selecting the INBOX with a SINCE/UNSEEN filter.
"""
logger.info("Connecting to %s at %s:%s (SSL=%s)",
mailbox_key, host, port, use_ssl)
logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
processed_emails = load_processed_emails()
try:
@@ -177,13 +177,11 @@ def pull_inbox(mailbox_key, host, port, username, password, use_ssl,
else:
# For non-Gmail, select INBOX and use SINCE/UNSEEN query.
mail.select("INBOX")
since_date = (datetime.now(timezone.utc) - timedelta(days=3)
).strftime("%d-%b-%Y")
status, search_data = mail.search(None, f'(SINCE {since_date} UNSEEN)')
since_date = (datetime.now(timezone.utc) - timedelta(days=3)).strftime("%d-%b-%Y")
status, search_data = mail.search(None, f"(SINCE {since_date} UNSEEN)")
if status != "OK":
logger.warning("Search failed on mailbox %s. Status=%s",
mailbox_key, status)
logger.warning("Search failed on mailbox %s. Status=%s", mailbox_key, status)
mail.close()
mail.logout()
return
@@ -194,8 +192,7 @@ def pull_inbox(mailbox_key, host, port, username, password, use_ssl,
for num in msg_numbers:
status, msg_data = mail.fetch(num, "(RFC822)")
if status != "OK":
logger.warning("Failed to fetch message %s in %s. Status=%s",
num, mailbox_key, status)
logger.warning("Failed to fetch message %s in %s. Status=%s", num, mailbox_key, status)
continue
raw_email = msg_data[0][1]
@@ -213,8 +210,7 @@ def pull_inbox(mailbox_key, host, port, username, password, use_ssl,
# For Gmail, check if the email already has the "Ingested" label.
if is_gmail_host:
if email_already_has_label(mail, num, "Ingested"):
logger.info("Skipping email %s in %s, already labeled 'Ingested'.",
msg_id, mailbox_key)
logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
continue
# Process attachments (and convert non-PDF files).
@@ -296,13 +292,12 @@ def fetch_attachments_and_enqueue(email_message):
continue
# Check if it's a PDF file by extension, regardless of MIME type
is_pdf_by_extension = filename.lower().endswith('.pdf')
is_pdf_by_extension = filename.lower().endswith(".pdf")
mime_type = part.get_content_type()
# Accept file if it has an allowed MIME type OR it's a PDF by extension
if mime_type not in ALLOWED_MIME_TYPES and not is_pdf_by_extension:
logger.info("Skipping attachment %s with MIME type %s",
filename, mime_type)
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
continue
file_path = os.path.join(settings.workdir, filename)
@@ -401,7 +396,7 @@ def find_all_mail_xlist(mail):
Returns the folder name if found, otherwise None.
"""
tag = mail._new_tag().decode("ascii")
command_str = f"{tag} XLIST \"\" \"*\""
command_str = f'{tag} XLIST "" "*"'
mail.send((command_str + "\r\n").encode("utf-8"))
all_mail_folder = None
app/tasks/process_with_azure_document_intelligence.py
+17 -14
View File
@@ -1,23 +1,23 @@
import os
import logging
import os
import azure.core.exceptions
import PyPDF2
from azure.core.credentials import AzureKeyCredential
from azure.ai.documentintelligence import DocumentIntelligenceClient
from azure.ai.documentintelligence.models import AnalyzeOutputOption, AnalyzeResult
import azure.core.exceptions
from azure.core.credentials import AzureKeyCredential
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.rotate_pdf_pages import rotate_pdf_pages
from app.celery_app import celery
logger = logging.getLogger(__name__)
# Initialize Azure Document Intelligence client with error handling
try:
document_intelligence_client = DocumentIntelligenceClient(
endpoint=settings.azure_endpoint,
credential=AzureKeyCredential(settings.azure_ai_key)
endpoint=settings.azure_endpoint, credential=AzureKeyCredential(settings.azure_ai_key)
)
logger.info("Azure Document Intelligence client initialized successfully")
except (ValueError, azure.core.exceptions.ClientAuthenticationError) as e:
@@ -33,16 +33,18 @@ AZURE_DOC_INTELLIGENCE_LIMITS = {
"max_pages": 2000,
}
def get_pdf_page_count(file_path):
"""Get the number of pages in a PDF file."""
try:
with open(file_path, 'rb') as file:
with open(file_path, "rb") as file:
pdf_reader = PyPDF2.PdfReader(file)
return len(pdf_reader.pages)
except Exception as e:
logger.error(f"Error getting PDF page count: {e}")
return None
def check_page_rotation(result, filename):
"""
Checks if pages in the document are rotated and logs the rotation information.
@@ -57,12 +59,12 @@ def check_page_rotation(result, filename):
logger.error(f"Checking rotation for document: {filename}")
rotation_data = {}
if not hasattr(result, 'pages') or not result.pages:
if not hasattr(result, "pages") or not result.pages:
logger.error(f"No page information available for rotation check: {filename}")
return rotation_data
for i, page in enumerate(result.pages):
if hasattr(page, 'angle'):
if hasattr(page, "angle"):
rotation_angle = page.angle
if rotation_angle != 0:
logger.error(f"Page {i+1} is rotated by {rotation_angle} degrees")
@@ -75,6 +77,7 @@ def check_page_rotation(result, filename):
return rotation_data
@celery.task(base=BaseTaskWithRetry)
def process_with_azure_document_intelligence(filename: str, file_id: int = None):
"""
@@ -101,13 +104,15 @@ def process_with_azure_document_intelligence(filename: str, file_id: int = None)
# Check file size against service limits
file_size = os.path.getsize(tmp_file_path)
if file_size > AZURE_DOC_INTELLIGENCE_LIMITS["max_file_size_bytes"]:
error_msg = f"File size ({file_size / (1024 * 1024):.2f} MB) exceeds Azure Document Intelligence limit of 500 MB"
error_msg = (
f"File size ({file_size / (1024 * 1024):.2f} MB) exceeds Azure Document Intelligence limit of 500 MB"
)
logger.error(error_msg)
return {"error": error_msg, "file": filename, "status": "Failed - Size limit exceeded"}
# For PDF files, check page count against service limits
# "Fail open" approach: only reject if we're sure it exceeds the limit
if filename.lower().endswith('.pdf'):
if filename.lower().endswith(".pdf"):
page_count = get_pdf_page_count(tmp_file_path)
if page_count is not None and page_count > AZURE_DOC_INTELLIGENCE_LIMITS["max_pages"]:
error_msg = f"PDF page count ({page_count}) exceeds Azure Document Intelligence limit of 2000 pages"
@@ -130,9 +135,7 @@ def process_with_azure_document_intelligence(filename: str, file_id: int = None)
rotation_data = check_page_rotation(result, filename)
# Retrieve the processed searchable PDF
response = document_intelligence_client.get_analyze_result_pdf(
model_id=result.model_id, result_id=operation_id
)
response = document_intelligence_client.get_analyze_result_pdf(model_id=result.model_id, result_id=operation_id)
searchable_pdf_path = tmp_file_path # Overwrite the original PDF location
with open(searchable_pdf_path, "wb") as writer:
writer.writelines(response)
app/tasks/refine_text_with_gpt.py
+14 -10
View File
@@ -1,17 +1,15 @@
#!/usr/bin/env python3
from app.config import settings
import openai
from app.tasks.retry_config import BaseTaskWithRetry
# Import the shared Celery instance
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
# Initialize OpenAI client dynamically
client = openai.OpenAI(
api_key=settings.openai_api_key,
base_url=settings.openai_base_url
)
client = openai.OpenAI(api_key=settings.openai_api_key, base_url=settings.openai_base_url)
@celery.task(base=BaseTaskWithRetry)
def refine_text_with_gpt(filename: str, raw_text: str):
@@ -19,16 +17,22 @@ def refine_text_with_gpt(filename: str, raw_text: str):
response = client.chat.completions.create(
model=settings.openai_model,
messages=[
{"role": "system", "content": "Clean and format the following text. The idea is that the text you see comes from an OCR system and your task is to eliminate OCR errors. Keep the original language when doing so."},
{"role": "user", "content": raw_text}
]
{
"role": "system",
"content": (
"Clean and format the following text. The idea is that the text you see comes from an OCR "
"system and your task is to eliminate OCR errors. Keep the original language when doing so."
),
},
{"role": "user", "content": raw_text},
],
)
cleaned_text = response.choices[0].message.content
# Trigger next task (import locally if needed to avoid circular imports)
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
extract_metadata_with_gpt.delay(filename, cleaned_text)
return {"filename": filename, "cleaned_text": cleaned_text}
app/tasks/retry_config.py
+1 -1
View File
@@ -2,8 +2,8 @@
from celery import Task
class BaseTaskWithRetry(Task):
autoretry_for = (Exception,)
retry_kwargs = {"max_retries": 3, "countdown": 10} # 3 retries, 10s delay
retry_backoff = True # Exponential backoff
app/tasks/rotate_pdf_pages.py
+24 -13
View File
@@ -1,16 +1,17 @@
import os
import logging
import PyPDF2
import math
import json
import logging
import os
import PyPDF2
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
from app.celery_app import celery
from app.config import settings
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
from app.tasks.retry_config import BaseTaskWithRetry
logger = logging.getLogger(__name__)
def determine_rotation_angle(detected_angle):
"""
Determine the optimal rotation angle based on detected angle.
@@ -46,6 +47,7 @@ def determine_rotation_angle(detected_angle):
logger.info(f"Detected angle {detected_angle}° rounded to {closest_90_multiple}°, will rotate by {rotation_value}°")
return rotation_value
@celery.task(base=BaseTaskWithRetry)
def rotate_pdf_pages(filename: str, extracted_text: str, rotation_data=None, file_id: int = None):
"""
@@ -85,7 +87,7 @@ def rotate_pdf_pages(filename: str, extracted_text: str, rotation_data=None, fil
applied_rotations = {}
# Load the PDF
with open(pdf_path, 'rb') as file:
with open(pdf_path, "rb") as file:
pdf_reader = PyPDF2.PdfReader(file)
pdf_writer = PyPDF2.PdfWriter()
@@ -101,21 +103,30 @@ def rotate_pdf_pages(filename: str, extracted_text: str, rotation_data=None, fil
if rotation_angle > 0:
# PyPDF2 uses clockwise rotation in 90-degree increments
page.rotate(rotation_angle)
logger.info(f"Page {page_idx+1} rotated by {rotation_angle}° (from detected {detected_angle}°)")
logger.info(
f"Page {page_idx+1} rotated by {rotation_angle}° "
f"(from detected {detected_angle}°)"
)
applied_rotations[str(page_idx)] = rotation_angle
else:
logger.info(f"Page {page_idx+1} had detected angle {detected_angle}° but determined it doesn't need rotation")
logger.info(
f"Page {page_idx+1} had detected angle {detected_angle}° "
"but determined it doesn't need rotation"
)
pdf_writer.add_page(page)
# Save the rotated PDF
with open(pdf_path, 'wb') as output_file:
with open(pdf_path, "wb") as output_file:
pdf_writer.write(output_file)
if applied_rotations:
logger.info(f"Successfully rotated PDF: {filename} with rotations: {json.dumps(applied_rotations)}")
else:
logger.info(f"Detected rotations in {filename} but no rotations were actually applied (angles too small or not multiples of 90°)")
logger.info(
f"Detected rotations in {filename} but no rotations were actually applied "
"(angles too small or not multiples of 90°)"
)
# Continue with metadata extraction
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
@@ -124,7 +135,7 @@ def rotate_pdf_pages(filename: str, extracted_text: str, rotation_data=None, fil
"file": filename,
"status": "rotated" if applied_rotations else "no_rotation_needed",
"detected_rotations": rotation_data,
"applied_rotations": applied_rotations
"applied_rotations": applied_rotations,
}
except Exception as e:
app/tasks/send_to_all.py
+68 -62
View File
@@ -1,83 +1,80 @@
#!/usr/bin/env python3
import os
import logging
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_webdav import upload_to_webdav
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_email import upload_to_email
from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_s3 import upload_to_s3
from app.utils.config_validator import get_provider_status
import os
from app.celery_app import celery
from app.utils import log_task_progress
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_email import upload_to_email
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_s3 import upload_to_s3
from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_webdav import upload_to_webdav
from app.utils import log_task_progress
from app.utils.config_validator import get_provider_status
logger = logging.getLogger(__name__)
def _should_upload_to_dropbox():
return (settings.dropbox_app_key and
settings.dropbox_app_secret and
settings.dropbox_refresh_token)
return settings.dropbox_app_key and settings.dropbox_app_secret and settings.dropbox_refresh_token
def _should_upload_to_nextcloud():
return (settings.nextcloud_upload_url and
settings.nextcloud_username and
settings.nextcloud_password)
return settings.nextcloud_upload_url and settings.nextcloud_username and settings.nextcloud_password
def _should_upload_to_paperless():
return (settings.paperless_ngx_api_token and
settings.paperless_host)
return settings.paperless_ngx_api_token and settings.paperless_host
def _should_upload_to_google_drive():
# Check for OAuth configuration
if getattr(settings, 'google_drive_use_oauth', False):
return (settings.google_drive_client_id and
settings.google_drive_client_secret and
settings.google_drive_refresh_token and
settings.google_drive_folder_id)
if getattr(settings, "google_drive_use_oauth", False):
return (
settings.google_drive_client_id
and settings.google_drive_client_secret
and settings.google_drive_refresh_token
and settings.google_drive_folder_id
)
# Or check for service account configuration
else:
return (settings.google_drive_credentials_json and
settings.google_drive_folder_id)
return settings.google_drive_credentials_json and settings.google_drive_folder_id
def _should_upload_to_webdav():
return (settings.webdav_url and
settings.webdav_username and
settings.webdav_password)
return settings.webdav_url and settings.webdav_username and settings.webdav_password
def _should_upload_to_ftp():
return (settings.ftp_host and
settings.ftp_username and
settings.ftp_password)
return settings.ftp_host and settings.ftp_username and settings.ftp_password
def _should_upload_to_sftp():
return (settings.sftp_host and
settings.sftp_username and
(settings.sftp_password or settings.sftp_private_key))
return settings.sftp_host and settings.sftp_username and (settings.sftp_password or settings.sftp_private_key)
def _should_upload_to_email():
return (settings.email_host and
settings.email_username and
settings.email_password and
settings.email_default_recipient)
return (
settings.email_host and settings.email_username and settings.email_password and settings.email_default_recipient
)
def _should_upload_to_onedrive():
return (settings.onedrive_client_id and
settings.onedrive_client_secret and
settings.onedrive_refresh_token)
return settings.onedrive_client_id and settings.onedrive_client_secret and settings.onedrive_refresh_token
def _should_upload_to_s3():
return (settings.s3_bucket_name and
settings.aws_access_key_id and
settings.aws_secret_access_key)
return settings.s3_bucket_name and settings.aws_access_key_id and settings.aws_secret_access_key
def get_configured_services_from_validator():
"""
@@ -97,16 +94,17 @@ def get_configured_services_from_validator():
"SFTP Storage": "sftp",
"Email": "email",
"OneDrive": "onedrive",
"S3 Storage": "s3"
"S3 Storage": "s3",
}
result = {}
for provider_name, internal_name in service_map.items():
if provider_name in providers:
result[internal_name] = providers[provider_name].get('configured', False)
result[internal_name] = providers[provider_name].get("configured", False)
return result
@celery.task(base=BaseTaskWithRetry, bind=True)
def send_to_all_destinations(self, file_path: str, use_validator=True, file_id: int = None):
"""
@@ -126,16 +124,24 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
raise FileNotFoundError(f"File not found: {file_path}")
logger.info(f"[{task_id}] Sending {file_path} to all configured destinations")
log_task_progress(task_id, "send_to_all_destinations", "in_progress", f"Distributing: {os.path.basename(file_path)}", file_id=file_id)
log_task_progress(
task_id,
"send_to_all_destinations",
"in_progress",
f"Distributing: {os.path.basename(file_path)}",
file_id=file_id,
)
# Get file_id from database if not provided (fallback only, prefer passing file_id explicitly)
if file_id is None:
with SessionLocal() as db:
# Only as a last resort, try to find by basename match
# This should not be needed if file_id is passed correctly through the chain
file_record = db.query(FileRecord).filter(
FileRecord.local_filename == os.path.join(settings.workdir, "tmp", os.path.basename(file_path))
).first()
file_record = (
db.query(FileRecord)
.filter(FileRecord.local_filename == os.path.join(settings.workdir, "tmp", os.path.basename(file_path)))
.first()
)
if file_record:
file_id = file_record.id
@@ -226,12 +232,16 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
# Queue the upload task if service is configured
if is_configured:
logger.info(f"[{task_id}] Queueing {file_path} for {service_name} upload")
log_task_progress(task_id, f"queue_{service_name}", "in_progress", f"Queueing upload to {service_name}", file_id=file_id)
log_task_progress(
task_id, f"queue_{service_name}", "in_progress", f"Queueing upload to {service_name}", file_id=file_id
)
try:
task = service["upload_func"].delay(file_path, file_id=file_id)
results[f"{service_name}_task_id"] = task.id
queued_count += 1
log_task_progress(task_id, f"queue_{service_name}", "success", f"Queued for {service_name}", file_id=file_id)
log_task_progress(
task_id, f"queue_{service_name}", "success", f"Queued for {service_name}", file_id=file_id
)
except Exception as e:
logger.error(f"[{task_id}] Failed to queue {service_name} task: {str(e)}")
results[f"{service_name}_error"] = str(e)
@@ -240,8 +250,4 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
logger.info(f"[{task_id}] Queued {queued_count} upload tasks")
log_task_progress(task_id, "send_to_all_destinations", "success", f"Queued {queued_count} uploads", file_id=file_id)
return {
"status": "Queued",
"file_path": file_path,
"tasks": results
}
return {"status": "Queued", "file_path": file_path, "tasks": results}
app/tasks/upload_to_dropbox.py
+37 -32
View File
@@ -1,31 +1,32 @@
#!/usr/bin/env python3
import os
import logging
import requests
import os
import dropbox
import requests
from dropbox.exceptions import ApiError, AuthError
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
from app.utils import log_task_progress
from app.database import SessionLocal
from app.models import FileRecord
from app.utils.filename_utils import extract_remote_path, get_unique_filename
logger = logging.getLogger(__name__)
def _validate_dropbox_settings():
"""Validate that all required Dropbox settings are available."""
missing = []
if not hasattr(settings, 'dropbox_refresh_token') or not settings.dropbox_refresh_token:
if not hasattr(settings, "dropbox_refresh_token") or not settings.dropbox_refresh_token:
missing.append("refresh token")
if not hasattr(settings, 'dropbox_app_key') or not settings.dropbox_app_key:
if not hasattr(settings, "dropbox_app_key") or not settings.dropbox_app_key:
missing.append("app key")
if not hasattr(settings, 'dropbox_app_secret') or not settings.dropbox_app_secret:
if not hasattr(settings, "dropbox_app_secret") or not settings.dropbox_app_secret:
missing.append("app secret")
if missing:
@@ -34,6 +35,7 @@ def _validate_dropbox_settings():
return True
def get_dropbox_access_token():
"""Refresh the Dropbox access token using the stored refresh token from ENV."""
@@ -59,6 +61,7 @@ def get_dropbox_access_token():
logger.error(error_msg)
raise Exception(error_msg)
def get_dropbox_client():
"""
Create and return an authenticated Dropbox client using the configured refresh token.
@@ -83,11 +86,7 @@ def get_dropbox_client():
# Create a Dropbox client with refresh token
try:
dbx = dropbox.Dropbox(
app_key=app_key,
app_secret=app_secret,
oauth2_refresh_token=refresh_token
)
dbx = dropbox.Dropbox(app_key=app_key, app_secret=app_secret, oauth2_refresh_token=refresh_token)
# Test the connection
dbx.users_get_current_account()
@@ -102,6 +101,7 @@ def get_dropbox_client():
logger.error(f"Error creating Dropbox client: {str(e)}")
raise
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_dropbox(self, file_path: str, file_id: int = None):
"""
@@ -113,7 +113,13 @@ def upload_to_dropbox(self, file_path: str, file_id: int = None):
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting Dropbox upload: {file_path}")
log_task_progress(task_id, "upload_to_dropbox", "in_progress", f"Uploading to Dropbox: {os.path.basename(file_path)}", file_id=file_id)
log_task_progress(
task_id,
"upload_to_dropbox",
"in_progress",
f"Uploading to Dropbox: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
@@ -122,9 +128,14 @@ def upload_to_dropbox(self, file_path: str, file_id: int = None):
raise FileNotFoundError(error_msg)
# Check if Dropbox is properly configured
if not (hasattr(settings, 'dropbox_app_key') and settings.dropbox_app_key and
hasattr(settings, 'dropbox_app_secret') and settings.dropbox_app_secret and
hasattr(settings, 'dropbox_refresh_token') and settings.dropbox_refresh_token):
if not (
hasattr(settings, "dropbox_app_key")
and settings.dropbox_app_key
and hasattr(settings, "dropbox_app_secret")
and settings.dropbox_app_secret
and hasattr(settings, "dropbox_refresh_token")
and settings.dropbox_refresh_token
):
logger.info(f"[{task_id}] Dropbox upload skipped: Missing configuration")
log_task_progress(task_id, "upload_to_dropbox", "success", "Skipped: Not configured", file_id=file_id)
return {"status": "Skipped", "reason": "Dropbox settings not configured"}
@@ -151,7 +162,7 @@ def upload_to_dropbox(self, file_path: str, file_id: int = None):
# Get a unique path in case of collision
remote_full_path = f"/{remote_path}" # Dropbox paths should start with /
remote_full_path = remote_full_path.replace('//', '/') # Clean double slashes
remote_full_path = remote_full_path.replace("//", "/") # Clean double slashes
# Check for potential file collision and get a unique name if needed
dropbox_path = get_unique_filename(remote_full_path, check_exists_in_dropbox)
@@ -159,7 +170,7 @@ def upload_to_dropbox(self, file_path: str, file_id: int = None):
# Upload the file
logger.info(f"[{task_id}] Uploading {filename} to Dropbox at {dropbox_path}")
log_task_progress(task_id, "upload_file", "in_progress", f"Uploading to {dropbox_path}", file_id=file_id)
with open(file_path, 'rb') as file_data:
with open(file_path, "rb") as file_data:
# Use files_upload_session for large files to avoid timeouts
file_size = os.path.getsize(file_path)
if file_size > 10 * 1024 * 1024: # 10 MB threshold for chunked upload
@@ -178,7 +189,7 @@ def upload_to_dropbox(self, file_path: str, file_id: int = None):
dbx.files_upload_session_finish(
file_data.read(chunk_size),
cursor,
dropbox.files.CommitInfo(path=dropbox_path, mode=dropbox.files.WriteMode.overwrite)
dropbox.files.CommitInfo(path=dropbox_path, mode=dropbox.files.WriteMode.overwrite),
)
else:
# More chunks to upload
@@ -187,19 +198,13 @@ def upload_to_dropbox(self, file_path: str, file_id: int = None):
else:
# Small file, direct upload
file_data.seek(0)
dbx.files_upload(
file_data.read(),
dropbox_path,
mode=dropbox.files.WriteMode.overwrite
)
dbx.files_upload(file_data.read(), dropbox_path, mode=dropbox.files.WriteMode.overwrite)
logger.info(f"[{task_id}] Successfully uploaded {filename} to Dropbox at {dropbox_path}")
log_task_progress(task_id, "upload_to_dropbox", "success", f"Uploaded to Dropbox: {dropbox_path}", file_id=file_id)
return {
"status": "Completed",
"file_path": file_path,
"dropbox_path": dropbox_path
}
log_task_progress(
task_id, "upload_to_dropbox", "success", f"Uploaded to Dropbox: {dropbox_path}", file_id=file_id
)
return {"status": "Completed", "file_path": file_path, "dropbox_path": dropbox_path}
except AuthError:
error_msg = f"Authentication failed while uploading {filename} to Dropbox. Check token."
app/tasks/upload_to_email.py
+48 -34
View File
@@ -1,24 +1,26 @@
#!/usr/bin/env python3
import os
import json
import logging
import os
import smtplib
import socket
import logging
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from datetime import datetime
from email.mime.application import MIMEApplication
from email.mime.image import MIMEImage
from datetime import datetime
from pathlib import Path
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from jinja2 import Environment, FileSystemLoader, select_autoescape
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
def get_email_template(template_name="default.html"):
"""
Load email template from one of these locations in order of precedence:
@@ -30,10 +32,9 @@ def get_email_template(template_name="default.html"):
workdir_template_path = os.path.join(settings.workdir, "templates", "email")
if os.path.exists(workdir_template_path):
env = Environment(
loader=FileSystemLoader(workdir_template_path),
autoescape=select_autoescape(['html', 'xml'])
loader=FileSystemLoader(workdir_template_path), autoescape=select_autoescape(["html", "xml"])
)
env.globals['now'] = datetime.now # Add the now function to the Jinja environment
env.globals["now"] = datetime.now # Add the now function to the Jinja environment
template = env.get_template(template_name)
logger.info(f"Using custom email template from workdir: {template_name}")
return template
@@ -45,11 +46,8 @@ def get_email_template(template_name="default.html"):
# Get the app directory path (where this file is)
current_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
app_template_path = os.path.join(current_dir, "templates", "email")
env = Environment(
loader=FileSystemLoader(app_template_path),
autoescape=select_autoescape(['html', 'xml'])
)
env.globals['now'] = datetime.now # Add the now function to the Jinja environment
env = Environment(loader=FileSystemLoader(app_template_path), autoescape=select_autoescape(["html", "xml"]))
env.globals["now"] = datetime.now # Add the now function to the Jinja environment
template = env.get_template(template_name)
logger.info(f"Using built-in email template: {template_name}")
return template
@@ -57,6 +55,7 @@ def get_email_template(template_name="default.html"):
logger.error(f"Failed to load built-in email template: {str(e)}")
raise ValueError(f"Could not find any valid email template: {str(e)}")
def extract_metadata_from_file(file_path):
"""
Try to extract metadata from a file using several methods:
@@ -68,10 +67,10 @@ def extract_metadata_from_file(file_path):
metadata = {}
# Check for separate metadata JSON file
metadata_path = os.path.splitext(file_path)[0] + '.json'
metadata_path = os.path.splitext(file_path)[0] + ".json"
if os.path.exists(metadata_path):
try:
with open(metadata_path, 'r', encoding='utf-8') as f:
with open(metadata_path, "r", encoding="utf-8") as f:
metadata = json.load(f)
logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
return metadata
@@ -83,6 +82,7 @@ def extract_metadata_from_file(file_path):
return metadata
def attach_logo(msg):
"""Attach the DocuElevate logo to the email with proper Content-ID."""
try:
@@ -99,14 +99,14 @@ def attach_logo(msg):
logo_path = os.path.join(app_dir, "..", "frontend", "static", "logo.png")
if os.path.exists(logo_path):
with open(logo_path, 'rb') as img:
with open(logo_path, "rb") as img:
logo_data = img.read()
# Determine image MIME type based on extension
mimetype = 'image/svg+xml' if logo_path.endswith('.svg') else 'image/png'
mimetype = "image/svg+xml" if logo_path.endswith(".svg") else "image/png"
logo_attach = MIMEImage(logo_data, mimetype)
logo_attach.add_header('Content-ID', '<logo>')
logo_attach.add_header('Content-Disposition', 'inline', filename='logo.png')
logo_attach.add_header("Content-ID", "<logo>")
logo_attach.add_header("Content-Disposition", "inline", filename="logo.png")
msg.attach(logo_attach)
logger.info(f"Logo attached from {logo_path}")
return True
@@ -118,6 +118,7 @@ def attach_logo(msg):
logger.warning(f"Error attaching logo: {str(e)}")
return False
def _prepare_recipients(recipients):
"""Helper function to prepare email recipients list."""
if not recipients:
@@ -130,6 +131,7 @@ def _prepare_recipients(recipients):
return [recipients], None # Convert single email to list
return recipients, None
def _send_email_with_smtp(msg, filename, recipients):
"""Helper function to handle SMTP connection and sending."""
try:
@@ -160,8 +162,18 @@ def _send_email_with_smtp(msg, filename, recipients):
logger.error(error_msg)
return {"status": "Failed", "reason": error_msg, "error": str(e)}
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_email(self, file_path: str, recipients=None, subject=None, message=None, template_name="default.html", include_metadata=True, file_id: int = None):
def upload_to_email(
self,
file_path: str,
recipients=None,
subject=None,
message=None,
template_name="default.html",
include_metadata=True,
file_id: int = None,
):
"""
Sends a file via email to the specified recipients.
If recipients is None, uses the configured default email recipient.
@@ -198,8 +210,10 @@ def upload_to_email(self, file_path: str, recipients=None, subject=None, message
return {"status": "Skipped", "reason": error_msg}
# Log email configuration for debugging
logger.debug(f"[{task_id}] Email config - Host: {settings.email_host}, Port: {settings.email_port}, "
f"Username: {settings.email_username}, TLS: {settings.email_use_tls}")
logger.debug(
f"[{task_id}] Email config - Host: {settings.email_host}, Port: {settings.email_port}, "
f"Username: {settings.email_username}, TLS: {settings.email_use_tls}"
)
# Process recipients
recipients, error = _prepare_recipients(recipients)
@@ -218,13 +232,13 @@ def upload_to_email(self, file_path: str, recipients=None, subject=None, message
try:
# Create the email
msg = MIMEMultipart('related')
msg['From'] = settings.email_sender or settings.email_username
msg['To'] = ", ".join(recipients)
msg['Subject'] = subject
msg = MIMEMultipart("related")
msg["From"] = settings.email_sender or settings.email_username
msg["To"] = ", ".join(recipients)
msg["Subject"] = subject
# Create alternative part for HTML content
alt_part = MIMEMultipart('alternative')
alt_part = MIMEMultipart("alternative")
msg.attach(alt_part)
# Attach logo to the email
@@ -243,24 +257,24 @@ def upload_to_email(self, file_path: str, recipients=None, subject=None, message
"metadata": metadata,
"has_metadata": bool(metadata),
"has_logo": has_logo,
"current_year": datetime.now().year
"current_year": datetime.now().year,
}
# Render HTML body
html_content = template.render(**context)
alt_part.attach(MIMEText(html_content, 'html'))
alt_part.attach(MIMEText(html_content, "html"))
# Attach the file
with open(file_path, "rb") as file:
attachment = MIMEApplication(file.read(), _subtype="pdf")
attachment.add_header('Content-Disposition', f'attachment; filename="{filename}"')
attachment.add_header("Content-Disposition", f'attachment; filename="{filename}"')
msg.attach(attachment)
# Send the email through SMTP
error_result = _send_email_with_smtp(msg, filename, recipients)
if error_result:
logger.error(f"[{task_id}] Failed to send email: {error_result.get('reason')}")
log_task_progress(task_id, "upload_to_email", "failure", error_result.get('reason'), file_id=file_id)
log_task_progress(task_id, "upload_to_email", "failure", error_result.get("reason"), file_id=file_id)
return error_result
logger.info(f"[{task_id}] Successfully sent {filename} via email to {len(recipients)} recipients")
@@ -272,7 +286,7 @@ def upload_to_email(self, file_path: str, recipients=None, subject=None, message
"recipients": recipients,
"subject": subject,
"metadata_included": bool(metadata),
"logo_included": has_logo
"logo_included": has_logo,
}
except Exception as e:
app/tasks/upload_to_ftp.py
+19 -35
View File
@@ -1,17 +1,19 @@
#!/usr/bin/env python3
import os
# Security Warning: FTP is an insecure protocol. FTPS (FTP_TLS) is strongly recommended.
# This module attempts to use FTPS by default and falls back to plaintext FTP only if configured.
import ftplib # nosec B402 - FTP usage is intentional for legacy server support
import logging
import os
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
import logging
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_ftp(self, file_path: str, file_id: int = None):
"""
@@ -49,23 +51,17 @@ def upload_to_ftp(self, file_path: str, file_id: int = None):
try:
# First attempt FTPS (FTP with TLS)
use_tls = getattr(settings, 'ftp_use_tls', True) # Default to try TLS
allow_plaintext = getattr(settings, 'ftp_allow_plaintext', True) # Default to allow plaintext fallback
use_tls = getattr(settings, "ftp_use_tls", True) # Default to try TLS
allow_plaintext = getattr(settings, "ftp_allow_plaintext", True) # Default to allow plaintext fallback
if use_tls:
try:
logger.info(f"Attempting FTPS connection to {settings.ftp_host}")
ftp = ftplib.FTP_TLS()
ftp.connect(
host=settings.ftp_host,
port=settings.ftp_port or 21
)
ftp.connect(host=settings.ftp_host, port=settings.ftp_port or 21)
# Login with credentials
ftp.login(
user=settings.ftp_username,
passwd=settings.ftp_password
)
ftp.login(user=settings.ftp_username, passwd=settings.ftp_password)
# Enable data protection - encrypt the data channel
ftp.prot_p()
@@ -79,16 +75,10 @@ def upload_to_ftp(self, file_path: str, file_id: int = None):
logger.warning(f"FTPS connection failed, falling back to regular FTP: {str(e)}")
# Fall back to regular FTP - only if explicitly allowed by configuration
ftp = ftplib.FTP() # nosec B321 - Fallback to FTP intentional when configured
ftp.connect(
host=settings.ftp_host,
port=settings.ftp_port or 21
)
ftp.connect(host=settings.ftp_host, port=settings.ftp_port or 21)
# Login with credentials
ftp.login(
user=settings.ftp_username,
passwd=settings.ftp_password
)
ftp.login(user=settings.ftp_username, passwd=settings.ftp_password)
else:
# Check if plaintext is allowed when TLS is explicitly disabled
if not allow_plaintext:
@@ -99,16 +89,10 @@ def upload_to_ftp(self, file_path: str, file_id: int = None):
# Directly use regular FTP if TLS is explicitly disabled
logger.warning("Using plaintext FTP - connection is NOT encrypted!")
ftp = ftplib.FTP() # nosec B321 - Plaintext FTP intentional when explicitly configured
ftp.connect(
host=settings.ftp_host,
port=settings.ftp_port or 21
)
ftp.connect(host=settings.ftp_host, port=settings.ftp_port or 21)
# Login with credentials
ftp.login(
user=settings.ftp_username,
passwd=settings.ftp_password
)
ftp.login(user=settings.ftp_username, passwd=settings.ftp_password)
# Change to target directory if specified
if settings.ftp_folder:
@@ -116,7 +100,7 @@ def upload_to_ftp(self, file_path: str, file_id: int = None):
# Try to navigate to the directory, create if it doesn't exist
ftp_folder = settings.ftp_folder
# Remove leading slash if present
if ftp_folder.startswith('/'):
if ftp_folder.startswith("/"):
ftp_folder = ftp_folder[1:]
# Try to change to the directory
@@ -124,8 +108,8 @@ def upload_to_ftp(self, file_path: str, file_id: int = None):
ftp.cwd(ftp_folder)
except ftplib.error_perm:
# Create directory structure if it doesn't exist
folders = ftp_folder.split('/')
current_dir = ''
folders = ftp_folder.split("/")
current_dir = ""
for folder in folders:
if folder:
current_dir += f"/{folder}"
@@ -140,8 +124,8 @@ def upload_to_ftp(self, file_path: str, file_id: int = None):
raise Exception(error_msg)
# Upload the file
with open(file_path, 'rb') as file_data:
ftp.storbinary(f'STOR {filename}', file_data)
with open(file_path, "rb") as file_data:
ftp.storbinary(f"STOR {filename}", file_data)
# Close FTP connection
ftp.quit()
@@ -153,7 +137,7 @@ def upload_to_ftp(self, file_path: str, file_id: int = None):
"file": file_path,
"ftp_host": settings.ftp_host,
"ftp_path": f"{settings.ftp_folder}/{filename}" if settings.ftp_folder else filename,
"used_tls": isinstance(ftp, ftplib.FTP_TLS)
"used_tls": isinstance(ftp, ftplib.FTP_TLS),
}
except Exception as e:
app/tasks/upload_to_google_drive.py
+52 -44
View File
@@ -2,24 +2,25 @@
app/tasks/upload_to_google_drive.py
"""
import os
import json
import logging
import os
from google.auth.exceptions import RefreshError
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials as OAuthCredentials
from google.oauth2.service_account import Credentials
from googleapiclient.discovery import build
from googleapiclient.http import MediaFileUpload
from google.oauth2.service_account import Credentials
from google.oauth2.credentials import Credentials as OAuthCredentials
from google_auth_oauthlib.flow import Flow
from google.auth.transport.requests import Request
from google.auth.exceptions import RefreshError
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
def get_drive_service_oauth():
"""
Get Google Drive service using OAuth credentials.
@@ -27,9 +28,11 @@ def get_drive_service_oauth():
"""
try:
# Check for required OAuth settings
if not (settings.google_drive_client_id and
settings.google_drive_client_secret and
settings.google_drive_refresh_token):
if not (
settings.google_drive_client_id
and settings.google_drive_client_secret
and settings.google_drive_refresh_token
):
logger.error("Google Drive OAuth credentials not fully configured")
return None
@@ -41,14 +44,14 @@ def get_drive_service_oauth():
client_id=settings.google_drive_client_id,
client_secret=settings.google_drive_client_secret,
# Use only drive.file scope
scopes=['https://www.googleapis.com/auth/drive.file']
scopes=["https://www.googleapis.com/auth/drive.file"],
)
# Refresh the access token
credentials.refresh(Request())
# Build and return the service
service = build('drive', 'v3', credentials=credentials)
service = build("drive", "v3", credentials=credentials)
return service
except RefreshError as e:
@@ -58,6 +61,7 @@ def get_drive_service_oauth():
logger.error(f"Failed to authenticate with Google Drive OAuth: {str(e)}")
return None
def get_google_drive_service():
"""
Authenticate with Google Drive API using service account credentials
@@ -65,7 +69,7 @@ def get_google_drive_service():
"""
try:
# Check if we should use OAuth instead of service account
if getattr(settings, 'google_drive_use_oauth', False):
if getattr(settings, "google_drive_use_oauth", False):
return get_drive_service_oauth()
# Load service account credentials from settings
@@ -75,8 +79,7 @@ def get_google_drive_service():
credentials_dict = json.loads(settings.google_drive_credentials_json)
credentials = Credentials.from_service_account_info(
credentials_dict,
scopes=['https://www.googleapis.com/auth/drive']
credentials_dict, scopes=["https://www.googleapis.com/auth/drive"]
)
# Delegate to user if specified
@@ -84,13 +87,14 @@ def get_google_drive_service():
credentials = credentials.with_subject(settings.google_drive_delegate_to)
# Build and return the service
service = build('drive', 'v3', credentials=credentials)
service = build("drive", "v3", credentials=credentials)
return service
except Exception as e:
logger.error(f"Failed to authenticate with Google Drive: {str(e)}")
return None
def extract_metadata_from_file(file_path):
"""
Try to extract metadata from a file using several methods:
@@ -99,10 +103,10 @@ def extract_metadata_from_file(file_path):
Returns a dictionary of metadata or empty dict if not found
"""
# Check for separate metadata JSON file
metadata_path = os.path.splitext(file_path)[0] + '.json'
metadata_path = os.path.splitext(file_path)[0] + ".json"
if os.path.exists(metadata_path):
try:
with open(metadata_path, 'r', encoding='utf-8') as f:
with open(metadata_path, "r", encoding="utf-8") as f:
metadata = json.load(f)
logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
return metadata
@@ -111,6 +115,7 @@ def extract_metadata_from_file(file_path):
return {}
def truncate_property_value(key, value, max_bytes=100):
"""
Truncate a property value to ensure the key+value stays under the byte limit.
@@ -121,8 +126,8 @@ def truncate_property_value(key, value, max_bytes=100):
str_value = str(value)
# Calculate current size of key and value in bytes
key_bytes = len(key.encode('utf-8'))
value_bytes = len(str_value.encode('utf-8'))
key_bytes = len(key.encode("utf-8"))
value_bytes = len(str_value.encode("utf-8"))
total_bytes = key_bytes + value_bytes
# If under limit, return original value
@@ -134,7 +139,7 @@ def truncate_property_value(key, value, max_bytes=100):
bytes_to_trim = total_bytes - max_bytes + 4
# Iteratively truncate the string until it's under the byte limit
while len(str_value.encode('utf-8')) > value_bytes - bytes_to_trim:
while len(str_value.encode("utf-8")) > value_bytes - bytes_to_trim:
str_value = str_value[:-1]
# Add ellipsis to indicate truncation
@@ -143,6 +148,7 @@ def truncate_property_value(key, value, max_bytes=100):
return str_value
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id: int = None):
"""
@@ -156,7 +162,11 @@ def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id:
task_id = self.request.id
logger.info(f"[{task_id}] Starting Google Drive upload: {file_path}")
log_task_progress(
task_id, "upload_to_google_drive", "in_progress", f"Uploading to Google Drive: {os.path.basename(file_path)}", file_id=file_id
task_id,
"upload_to_google_drive",
"in_progress",
f"Uploading to Google Drive: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
@@ -184,17 +194,17 @@ def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id:
# Prepare the file metadata
file_metadata = {
'name': filename,
"name": filename,
}
# If folder ID is specified, set parent folder
if settings.google_drive_folder_id:
file_metadata['parents'] = [settings.google_drive_folder_id]
file_metadata["parents"] = [settings.google_drive_folder_id]
# Add custom properties if metadata exists
if metadata:
# Google Drive properties must be strings and can't be nested objects
file_metadata['properties'] = {}
file_metadata["properties"] = {}
# Only add a few important top-level metadata fields as properties
# Skip nested objects and long values to avoid the 124-byte limit
@@ -212,46 +222,44 @@ def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id:
logger.warning(f"Skipping metadata property {key}: {str(e)}")
# Only use the safe properties
file_metadata['properties'] = safe_properties
file_metadata["properties"] = safe_properties
# Add minimal appProperties
file_metadata['appProperties'] = {
'docuelevate': 'true'
}
file_metadata["appProperties"] = {"docuelevate": "true"}
# Add metadata to file description for better visibility in Google Drive UI
# Description has much higher size limits than properties
formatted_json = json.dumps(metadata, indent=2)
file_metadata['description'] = f"Document Metadata:\n\n```json\n{formatted_json}\n```"
file_metadata["description"] = f"Document Metadata:\n\n```json\n{formatted_json}\n```"
logger.debug(f"Adding metadata to Google Drive file: {json.dumps(file_metadata['properties'])}")
# Upload file with metadata
media = MediaFileUpload(
file_path,
mimetype='application/pdf',
resumable=True
media = MediaFileUpload(file_path, mimetype="application/pdf", resumable=True)
file = (
service.files()
.create(
body=file_metadata, media_body=media, fields="id,name,webViewLink,properties,appProperties,description"
)
.execute()
)
file = service.files().create(
body=file_metadata,
media_body=media,
fields='id,name,webViewLink,properties,appProperties,description'
).execute()
# Log success details
google_drive_file_id = file.get('id')
web_view_link = file.get('webViewLink')
google_drive_file_id = file.get("id")
web_view_link = file.get("webViewLink")
logger.info(f"[{task_id}] Successfully uploaded {filename} to Google Drive with ID: {google_drive_file_id}")
logger.info(f"[{task_id}] File accessible at: {web_view_link}")
log_task_progress(task_id, "upload_to_google_drive", "success", f"Uploaded to Google Drive: {filename}", file_id=file_id)
log_task_progress(
task_id, "upload_to_google_drive", "success", f"Uploaded to Google Drive: {filename}", file_id=file_id
)
result = {
"status": "Completed",
"file_path": file_path,
"google_drive_file_id": google_drive_file_id,
"google_drive_web_link": web_view_link
"google_drive_web_link": web_view_link,
}
# Add metadata info to result if included
app/tasks/upload_to_nextcloud.py
+46 -36
View File
@@ -1,19 +1,20 @@
#!/usr/bin/env python3
import os
import logging
import os
import requests
from requests.auth import HTTPBasicAuth
from app.config import settings
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
from app.utils import log_task_progress
from app.database import SessionLocal
from app.models import FileRecord
from app.utils.filename_utils import extract_remote_path, get_unique_filename
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_nextcloud(self, file_path: str, file_id: int = None):
"""
@@ -25,7 +26,13 @@ def upload_to_nextcloud(self, file_path: str, file_id: int = None):
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting Nextcloud upload: {file_path}")
log_task_progress(task_id, "upload_to_nextcloud", "in_progress", f"Uploading to Nextcloud: {os.path.basename(file_path)}", file_id=file_id)
log_task_progress(
task_id,
"upload_to_nextcloud",
"in_progress",
f"Uploading to Nextcloud: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
@@ -35,43 +42,44 @@ def upload_to_nextcloud(self, file_path: str, file_id: int = None):
# For Nextcloud, we need to check for 'nextcloud_upload_url' instead of 'nextcloud_url'
# This is what's shown in your env view
if not (getattr(settings, 'nextcloud_upload_url', None) and
getattr(settings, 'nextcloud_username', None) and
getattr(settings, 'nextcloud_password', None)):
if not (
getattr(settings, "nextcloud_upload_url", None)
and getattr(settings, "nextcloud_username", None)
and getattr(settings, "nextcloud_password", None)
):
logger.info(f"[{task_id}] Nextcloud upload skipped: Missing configuration")
log_task_progress(task_id, "upload_to_nextcloud", "success", "Skipped: Not configured", file_id=file_id)
return {"status": "Skipped", "reason": "Nextcloud settings not configured"}
filename = os.path.basename(file_path)
sanitized_filename = sanitize_filename(filename)
try:
# Prepare WebDAV URL - use nextcloud_upload_url instead of nextcloud_url
webdav_url = settings.nextcloud_upload_url
if not webdav_url.endswith('/'):
webdav_url += '/'
if not webdav_url.endswith("/"):
webdav_url += "/"
# Calculate remote path based on local file structure
remote_base = getattr(settings, 'nextcloud_folder', '') or ""
remote_base = getattr(settings, "nextcloud_folder", "") or ""
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
full_url = f"{webdav_url}/{remote_path}"
# Remove any double slashes (except in http://)
full_url = full_url.replace('://', '$PLACEHOLDER$')
while '//' in full_url:
full_url = full_url.replace('//', '/')
full_url = full_url.replace('$PLACEHOLDER$', '://')
full_url = full_url.replace("://", "$PLACEHOLDER$")
while "//" in full_url:
full_url = full_url.replace("//", "/")
full_url = full_url.replace("$PLACEHOLDER$", "://")
# Function to check if file exists in Nextcloud
def check_exists_in_nextcloud(path):
check_url = f"{webdav_url}{os.path.dirname(path)}"
try:
response = requests.request(
'PROPFIND',
"PROPFIND",
check_url,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
headers={'Depth': '1'},
timeout=10
headers={"Depth": "1"},
timeout=10,
)
return path in response.text
@@ -84,53 +92,55 @@ def upload_to_nextcloud(self, file_path: str, file_id: int = None):
full_url = f"{webdav_url}/{remote_path}"
# Fix double slashes again
full_url = full_url.replace('://', '$PLACEHOLDER$')
while '//' in full_url:
full_url = full_url.replace('//', '/')
full_url = full_url.replace('$PLACEHOLDER$', '://')
full_url = full_url.replace("://", "$PLACEHOLDER$")
while "//" in full_url:
full_url = full_url.replace("//", "/")
full_url = full_url.replace("$PLACEHOLDER$", "://")
# Create necessary parent folders
parent_dirs = os.path.dirname(remote_path)
if parent_dirs:
current_path = ""
for folder in parent_dirs.split('/'):
for folder in parent_dirs.split("/"):
if not folder:
continue
current_path += f"{folder}/"
mkdir_url = f"{webdav_url}/{current_path}"
# Fix double slashes
mkdir_url = mkdir_url.replace('://', '$PLACEHOLDER$')
while '//' in mkdir_url:
mkdir_url = mkdir_url.replace('//', '/')
mkdir_url = mkdir_url.replace('$PLACEHOLDER$', '://')
mkdir_url = mkdir_url.replace("://", "$PLACEHOLDER$")
while "//" in mkdir_url:
mkdir_url = mkdir_url.replace("//", "/")
mkdir_url = mkdir_url.replace("$PLACEHOLDER$", "://")
requests.request(
'MKCOL',
"MKCOL",
mkdir_url,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
timeout=10
timeout=10,
)
# Upload the file
logger.info(f"[{task_id}] Uploading {filename} to Nextcloud at {full_url}")
log_task_progress(task_id, "upload_file", "in_progress", f"Uploading to {remote_path}", file_id=file_id)
with open(file_path, 'rb') as file_data:
with open(file_path, "rb") as file_data:
response = requests.put(
full_url,
data=file_data,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
headers={'Content-Type': 'application/octet-stream'},
timeout=settings.http_request_timeout # Use configured timeout for large files
headers={"Content-Type": "application/octet-stream"},
timeout=settings.http_request_timeout, # Use configured timeout for large files
)
if response.status_code in (201, 204): # Created or No Content
logger.info(f"[{task_id}] Successfully uploaded {filename} to Nextcloud at {remote_path}")
log_task_progress(task_id, "upload_to_nextcloud", "success", f"Uploaded to Nextcloud: {remote_path}", file_id=file_id)
log_task_progress(
task_id, "upload_to_nextcloud", "success", f"Uploaded to Nextcloud: {remote_path}", file_id=file_id
)
return {
"status": "Completed",
"file_path": file_path,
"nextcloud_path": remote_path,
"response_code": response.status_code
"response_code": response.status_code,
}
else:
error_msg = f"Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
app/tasks/upload_to_onedrive.py
+27 -35
View File
@@ -1,18 +1,21 @@
#!/usr/bin/env python3
import logging
import os
import time
import logging
import requests
import msal
import urllib.parse
import msal
import requests
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
def get_onedrive_token():
"""
Get an access token for Microsoft Graph API using the appropriate flow.
@@ -36,14 +39,13 @@ def get_onedrive_token():
app = msal.ConfidentialClientApplication(
client_id=settings.onedrive_client_id,
client_credential=settings.onedrive_client_secret,
authority=f"https://login.microsoftonline.com/{tenant}"
authority=f"https://login.microsoftonline.com/{tenant}",
)
# Request new token using refresh token
logger.info("Attempting to acquire token using refresh token")
token_response = app.acquire_token_by_refresh_token(
refresh_token=settings.onedrive_refresh_token,
scopes=scopes
refresh_token=settings.onedrive_refresh_token, scopes=scopes
)
if "access_token" not in token_response:
@@ -51,7 +53,7 @@ def get_onedrive_token():
error_desc = token_response.get("error_description", "Unknown error")
# Log more details about the error
logger.error(f"Failed to get access token using refresh token")
logger.error("Failed to get access token using refresh token")
logger.error(f"Error code: {error}")
logger.error(f"Error description: {error_desc}")
@@ -78,13 +80,11 @@ def get_onedrive_token():
app = msal.ConfidentialClientApplication(
client_id=settings.onedrive_client_id,
client_credential=settings.onedrive_client_secret,
authority=authority
authority=authority,
)
# Acquire token for application
token_response = app.acquire_token_for_client(
scopes=scopes
)
token_response = app.acquire_token_for_client(scopes=scopes)
if "access_token" not in token_response:
error = token_response.get("error", "")
@@ -96,6 +96,7 @@ def get_onedrive_token():
else:
raise ValueError("For personal Microsoft accounts, ONEDRIVE_REFRESH_TOKEN must be configured")
def create_upload_session(filename, folder_path, access_token):
"""Creates an upload session for large files in Microsoft Graph API."""
# Construct the API endpoint
@@ -104,11 +105,11 @@ def create_upload_session(filename, folder_path, access_token):
# Format the folder path correctly and properly encode for URL
if folder_path:
# Remove leading/trailing slashes
folder_path = folder_path.strip('/')
folder_path = folder_path.strip("/")
# URL encode the path components separately
path_components = folder_path.split('/')
encoded_path = '/'.join(urllib.parse.quote(component) for component in path_components)
path_components = folder_path.split("/")
encoded_path = "/".join(urllib.parse.quote(component) for component in path_components)
# Also encode the filename
encoded_filename = urllib.parse.quote(filename)
@@ -121,16 +122,9 @@ def create_upload_session(filename, folder_path, access_token):
url = f"{base_url}{item_path}"
# Add required request body (can be empty JSON object)
request_body = {
"item": {
"@microsoft.graph.conflictBehavior": "replace"
}
}
request_body = {"item": {"@microsoft.graph.conflictBehavior": "replace"}}
headers = {
"Authorization": f"Bearer {access_token}",
"Content-Type": "application/json"
}
headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"}
logger.info(f"Creating upload session for {filename} at path {folder_path}")
@@ -148,6 +142,7 @@ def create_upload_session(filename, folder_path, access_token):
logger.error(f"Request body: {request_body}")
raise Exception(error_msg)
def upload_large_file(file_path, upload_url):
"""
Upload a large file to OneDrive using the upload session URL.
@@ -160,7 +155,7 @@ def upload_large_file(file_path, upload_url):
chunk_size = 10 * 1024 * 1024
# Open and read file in chunks
with open(file_path, 'rb') as f:
with open(file_path, "rb") as f:
# Process file in chunks
chunk_number = 0
while True:
@@ -176,10 +171,7 @@ def upload_large_file(file_path, upload_url):
content_range = f"bytes {chunk_start}-{chunk_end}/{file_size}"
# Upload chunk
headers = {
"Content-Length": str(len(chunk)),
"Content-Range": content_range
}
headers = {"Content-Length": str(len(chunk)), "Content-Range": content_range}
# Try to upload chunk with retries
max_retries = 3
@@ -188,10 +180,7 @@ def upload_large_file(file_path, upload_url):
for attempt in range(max_retries):
try:
response = requests.put(
upload_url,
headers=headers,
data=chunk,
timeout=settings.http_request_timeout
upload_url, headers=headers, data=chunk, timeout=settings.http_request_timeout
)
# Check if successful
@@ -208,7 +197,9 @@ def upload_large_file(file_path, upload_url):
time.sleep(retry_delay * (attempt + 1))
if response.status_code not in (201, 202):
raise Exception(f"Failed to upload chunk after {max_retries} attempts: {response.status_code} - {response.text}")
raise Exception(
f"Failed to upload chunk after {max_retries} attempts: {response.status_code} - {response.text}"
)
# Move to next chunk
chunk_number += 1
@@ -217,6 +208,7 @@ def upload_large_file(file_path, upload_url):
# The last response should contain the file metadata
return response.json()
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_onedrive(self, file_path: str, file_id: int = None):
"""
@@ -274,7 +266,7 @@ def upload_to_onedrive(self, file_path: str, file_id: int = None):
"status": "Completed",
"file_path": file_path,
"onedrive_path": f"{settings.onedrive_folder_path}/{filename}",
"web_url": web_url
"web_url": web_url,
}
except Exception as e:
app/tasks/upload_to_paperless.py
+33 -27
View File
@@ -1,29 +1,26 @@
#!/usr/bin/env python3
import os
import json
import time
import requests
import logging
from typing import Dict, Any
import os
import time
import requests
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
from app.database import SessionLocal
from app.models import FileRecord
logger = logging.getLogger(__name__)
POLL_MAX_ATTEMPTS = 10
POLL_INTERVAL_SEC = 3
def _get_headers():
"""Returns HTTP headers for Paperless-ngx API calls."""
return {
"Authorization": f"Token {settings.paperless_ngx_api_token}"
}
return {"Authorization": f"Token {settings.paperless_ngx_api_token}"}
def _paperless_api_url(path: str) -> str:
"""
@@ -35,6 +32,7 @@ def _paperless_api_url(path: str) -> str:
path = "/" + path
return f"{host}{path}"
def poll_task_for_document_id(task_id: str) -> int:
"""
Polls /api/tasks/?task_id=<uuid> until we get status=SUCCESS or FAILURE,
@@ -49,14 +47,13 @@ def poll_task_for_document_id(task_id: str) -> int:
while attempts < POLL_MAX_ATTEMPTS:
try:
resp = requests.get(url, headers=_get_headers(), params={"task_id": task_id}, timeout=settings.http_request_timeout)
resp = requests.get(
url, headers=_get_headers(), params={"task_id": task_id}, timeout=settings.http_request_timeout
)
resp.raise_for_status()
tasks_data = resp.json()
except requests.exceptions.RequestException as exc:
logger.warning(
"Failed to poll for task_id='%s'. Attempt=%d Error=%s",
task_id, attempts + 1, exc
)
logger.warning("Failed to poll for task_id='%s'. Attempt=%d Error=%s", task_id, attempts + 1, exc)
time.sleep(POLL_INTERVAL_SEC)
attempts += 1
continue
@@ -71,18 +68,15 @@ def poll_task_for_document_id(task_id: str) -> int:
doc_str = task_info.get("related_document")
if doc_str:
return int(doc_str)
raise RuntimeError(
f"Task {task_id} completed but no doc ID found. Task info: {task_info}"
)
raise RuntimeError(f"Task {task_id} completed but no doc ID found. Task info: {task_info}")
elif status == "FAILURE":
raise RuntimeError(f"Task {task_id} failed: {task_info.get('result')}")
attempts += 1
time.sleep(POLL_INTERVAL_SEC)
raise TimeoutError(
f"Task {task_id} didn't reach SUCCESS within {POLL_MAX_ATTEMPTS} attempts."
)
raise TimeoutError(f"Task {task_id} didn't reach SUCCESS within {POLL_MAX_ATTEMPTS} attempts.")
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_paperless(self, file_path: str, file_id: int = None):
@@ -95,7 +89,13 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting Paperless upload: {file_path}")
log_task_progress(task_id, "upload_to_paperless", "in_progress", f"Uploading to Paperless: {os.path.basename(file_path)}", file_id=file_id)
log_task_progress(
task_id,
"upload_to_paperless",
"in_progress",
f"Uploading to Paperless: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
@@ -125,13 +125,17 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
try:
logger.debug("Posting document to Paperless: file=%s", filename)
resp = requests.post(post_url, headers=_get_headers(), files=files, data=data, timeout=settings.http_request_timeout)
resp = requests.post(
post_url, headers=_get_headers(), files=files, data=data, timeout=settings.http_request_timeout
)
resp.raise_for_status()
except requests.exceptions.RequestException as exc:
error_msg = f"Failed to upload to Paperless: {exc}"
logger.error(
f"[{task_id}] Failed to upload document '%s' to Paperless. Error: %s. Response=%s",
file_path, exc, getattr(exc.response, "text", "<no response>")
file_path,
exc,
getattr(exc.response, "text", "<no response>"),
)
log_task_progress(task_id, "upload_to_paperless", "failure", error_msg, file_id=file_id)
raise
@@ -145,11 +149,13 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
log_task_progress(task_id, "poll_task", "in_progress", "Waiting for Paperless processing", file_id=file_id)
doc_id = poll_task_for_document_id(raw_task_id)
logger.info(f"[{task_id}] Document {file_path} successfully ingested => ID={doc_id}")
log_task_progress(task_id, "upload_to_paperless", "success", f"Uploaded to Paperless: Doc ID {doc_id}", file_id=file_id)
log_task_progress(
task_id, "upload_to_paperless", "success", f"Uploaded to Paperless: Doc ID {doc_id}", file_id=file_id
)
return {
"status": "Completed",
"paperless_task_id": raw_task_id,
"paperless_document_id": doc_id,
"file_path": file_path
"file_path": file_path,
}
app/tasks/upload_to_s3.py
+5 -8
View File
@@ -1,12 +1,14 @@
#!/usr/bin/env python3
import os
import logging
import os
import boto3
from botocore.exceptions import ClientError
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
@@ -76,12 +78,7 @@ def upload_to_s3(self, file_path: str, file_id: int = None):
extra_args["ACL"] = settings.s3_acl
# Upload file
s3_client.upload_file(
file_path,
settings.s3_bucket_name,
s3_key,
ExtraArgs=extra_args
)
s3_client.upload_file(file_path, settings.s3_bucket_name, s3_key, ExtraArgs=extra_args)
# Generate URL to the file (useful for public files)
# For private files, this is just a reference and won't be accessible directly
app/tasks/upload_to_sftp.py
+14 -16
View File
@@ -1,17 +1,19 @@
#!/usr/bin/env python3
import os
import logging
import os
import paramiko
from pathlib import Path
from app.config import settings
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
from app.utils import log_task_progress
from app.utils.filename_utils import extract_remote_path, get_unique_filename, sanitize_filename
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_sftp(self, file_path: str, file_id: int = None):
"""
@@ -48,7 +50,7 @@ def upload_to_sftp(self, file_path: str, file_id: int = None):
# Security: Host key verification
# WARNING: AutoAddPolicy automatically trusts unknown host keys (vulnerable to MITM attacks)
# For production, use RejectPolicy and configure known_hosts, or WarningPolicy at minimum
if getattr(settings, 'sftp_disable_host_key_verification', True):
if getattr(settings, "sftp_disable_host_key_verification", True):
logger.warning(
"SFTP host key verification is DISABLED - connections are vulnerable to MITM attacks. "
"For production, set SFTP_DISABLE_HOST_KEY_VERIFICATION=false and configure known_hosts."
@@ -68,8 +70,8 @@ def upload_to_sftp(self, file_path: str, file_id: int = None):
}
# Check for authentication methods - use key if available, otherwise try password
sftp_key_path = getattr(settings, 'sftp_private_key', None)
sftp_key_passphrase = getattr(settings, 'sftp_private_key_passphrase', None)
sftp_key_path = getattr(settings, "sftp_private_key", None)
sftp_key_passphrase = getattr(settings, "sftp_private_key_passphrase", None)
if sftp_key_path and os.path.exists(sftp_key_path):
logger.info(f"Using SSH key authentication with key: {sftp_key_path}")
@@ -96,8 +98,8 @@ def upload_to_sftp(self, file_path: str, file_id: int = None):
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
# Ensure the remote path starts with a slash if the base folder does
if remote_base.startswith('/') and not remote_path.startswith('/'):
remote_path = '/' + remote_path
if remote_base.startswith("/") and not remote_path.startswith("/"):
remote_path = "/" + remote_path
# Function to check if file exists in SFTP server
def check_exists_in_sftp(path):
@@ -138,19 +140,15 @@ def upload_to_sftp(self, file_path: str, file_id: int = None):
sftp.close()
ssh.close()
return {
"status": "Completed",
"file_path": file_path,
"sftp_path": remote_path
}
return {"status": "Completed", "file_path": file_path, "sftp_path": remote_path}
except Exception as e:
# Make sure connections are closed
try:
if 'sftp' in locals():
if "sftp" in locals():
sftp.close()
ssh.close()
except:
except Exception:
pass
error_msg = f"Failed to upload {filename} to SFTP server: {str(e)}"
app/tasks/upload_to_webdav.py
+15 -6
View File
@@ -1,16 +1,19 @@
#!/usr/bin/env python3
import logging
import os
import requests
from urllib.parse import urljoin
import requests
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
import logging
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_webdav(self, file_path: str, file_id: int = None):
"""
@@ -23,7 +26,11 @@ def upload_to_webdav(self, file_path: str, file_id: int = None):
task_id = self.request.id
logger.info(f"[{task_id}] Starting WebDAV upload: {file_path}")
log_task_progress(
task_id, "upload_to_webdav", "in_progress", f"Uploading to WebDAV: {os.path.basename(file_path)}", file_id=file_id
task_id,
"upload_to_webdav",
"in_progress",
f"Uploading to WebDAV: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
@@ -65,13 +72,15 @@ def upload_to_webdav(self, file_path: str, file_id: int = None):
auth=(settings.webdav_username, settings.webdav_password),
data=file_data,
verify=settings.webdav_verify_ssl if hasattr(settings, "webdav_verify_ssl") else True,
timeout=settings.http_request_timeout
timeout=settings.http_request_timeout,
)
# Check if upload was successful
if response.status_code in (200, 201, 204):
logger.info(f"[{task_id}] Successfully uploaded {filename} to WebDAV at {webdav_url}.")
log_task_progress(task_id, "upload_to_webdav", "success", f"Uploaded to WebDAV: {filename}", file_id=file_id)
log_task_progress(
task_id, "upload_to_webdav", "success", f"Uploaded to WebDAV: {filename}", file_id=file_id
)
return {"status": "Completed", "file": file_path, "url": webdav_url}
else:
error_msg = f"Failed to upload {filename} to WebDAV: {response.status_code} - {response.text}"
app/tasks/upload_with_rclone.py
+3 -2
View File
@@ -1,11 +1,12 @@
#!/usr/bin/env python3
import logging
import os
import subprocess
import logging
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
logger = logging.getLogger(__name__)
app/tasks/uptime_kuma_tasks.py
+2
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import logging
import requests
from celery import shared_task
@@ -8,6 +9,7 @@ from app.config import settings
logger = logging.getLogger(__name__)
@shared_task
def ping_uptime_kuma():
"""
app/utils.py
+2 -2
View File
@@ -1,7 +1,7 @@
# app/utils.py
# This file is deprecated. Functions have been moved to the utils package.
# To avoid breaking existing imports, we'll import and re-export the functions
from app.utils.file_operations import hash_file
from app.utils.logging import log_task_progress
from app.utils.file_operations import hash_file # noqa: F401
from app.utils.logging import log_task_progress # noqa: F401
# These functions are now available directly from the app.utils package
app/utils/__init__.py
+1 -1
View File
@@ -7,4 +7,4 @@ from app.utils.file_operations import hash_file
from app.utils.logging import log_task_progress
# Export all the functions that should be available when importing from app.utils
__all__ = ['hash_file', 'log_task_progress']
__all__ = ["hash_file", "log_task_progress"]
app/utils/config_loader.py
+6 -5
View File
@@ -9,6 +9,7 @@ This module provides functionality to:
import logging
from typing import Any, Optional, Union
from sqlalchemy.orm import Session
from app.models import ApplicationSettings
@@ -79,15 +80,15 @@ def convert_setting_value(value: Optional[str], field_type: Any) -> Any:
return None
# Handle Optional types
origin = getattr(field_type, '__origin__', None)
origin = getattr(field_type, "__origin__", None)
if origin is Union:
# Get the non-None type from Union (for Optional)
args = getattr(field_type, '__args__', ())
args = getattr(field_type, "__args__", ())
field_type = next((arg for arg in args if arg is not type(None)), str)
# Convert based on type
if field_type == bool:
return value.lower() in ('true', '1', 'yes', 'y', 't')
return value.lower() in ("true", "1", "yes", "y", "t")
elif field_type == int:
try:
return int(value)
@@ -100,10 +101,10 @@ def convert_setting_value(value: Optional[str], field_type: Any) -> Any:
except ValueError:
logger.warning(f"Failed to convert '{value}' to float, returning 0.0")
return 0.0
elif field_type == list or getattr(field_type, '__origin__', None) == list:
elif field_type == list or getattr(field_type, "__origin__", None) == list:
# Handle list types - assume comma-separated values
if isinstance(value, str):
return [item.strip() for item in value.split(',') if item.strip()]
return [item.strip() for item in value.split(",") if item.strip()]
return value
else:
# Default to string
app/utils/config_validator.py
+16 -20
View File
@@ -4,29 +4,25 @@ Configuration validation for the application.
This file serves as a backward-compatible interface to the config_validator package.
"""
# Import and re-export all functions from the new package
from app.utils.config_validator.validators import (
validate_email_config,
validate_storage_configs,
validate_notification_config,
check_all_configs
)
from app.utils.config_validator.masking import mask_sensitive_value
from app.utils.config_validator.providers import get_provider_status
from app.utils.config_validator.settings_display import (
get_settings_for_display,
dump_all_settings
from app.utils.config_validator.settings_display import dump_all_settings, get_settings_for_display
# Import and re-export all functions from the new package
from app.utils.config_validator.validators import (
check_all_configs,
validate_email_config,
validate_notification_config,
validate_storage_configs,
)
__all__ = [
'validate_email_config',
'validate_storage_configs',
'validate_notification_config',
'mask_sensitive_value',
'get_provider_status',
'get_settings_for_display',
'dump_all_settings',
'check_all_configs'
"validate_email_config",
"validate_storage_configs",
"validate_notification_config",
"mask_sensitive_value",
"get_provider_status",
"get_settings_for_display",
"dump_all_settings",
"check_all_configs",
]
app/utils/config_validator/__init__.py
+16 -19
View File
@@ -2,28 +2,25 @@
Configuration validation package for the application.
"""
from app.utils.config_validator.validators import (
validate_email_config,
validate_storage_configs,
validate_notification_config,
validate_auth_config,
check_all_configs
)
from app.utils.config_validator.masking import mask_sensitive_value
from app.utils.config_validator.providers import get_provider_status
from app.utils.config_validator.settings_display import (
get_settings_for_display,
dump_all_settings
from app.utils.config_validator.settings_display import dump_all_settings, get_settings_for_display
from app.utils.config_validator.validators import (
check_all_configs,
validate_auth_config,
validate_email_config,
validate_notification_config,
validate_storage_configs,
)
__all__ = [
'validate_email_config',
'validate_storage_configs',
'validate_notification_config',
'validate_auth_config',
'mask_sensitive_value',
'get_provider_status',
'get_settings_for_display',
'dump_all_settings',
'check_all_configs'
"validate_email_config",
"validate_storage_configs",
"validate_notification_config",
"validate_auth_config",
"mask_sensitive_value",
"get_provider_status",
"get_settings_for_display",
"dump_all_settings",
"check_all_configs",
]
app/utils/config_validator/masking.py
+1
View File
@@ -2,6 +2,7 @@
Module for masking sensitive information in configuration values
"""
def mask_sensitive_value(value):
"""
Masks sensitive values like API keys in logs and output
app/utils/config_validator/providers.py
+162 -133
View File
@@ -5,6 +5,7 @@ Module for handling provider status information
from app.config import settings
from app.utils.config_validator.masking import mask_sensitive_value
def get_provider_status():
"""
Returns status information for all configured providers
@@ -12,287 +13,315 @@ def get_provider_status():
providers = {}
# Add Authentication configuration
auth_enabled = getattr(settings, 'auth_enabled', False)
using_oidc = bool(getattr(settings, 'authentik_client_id', None) and
getattr(settings, 'authentik_client_secret', None) and
getattr(settings, 'authentik_config_url', None))
auth_enabled = getattr(settings, "auth_enabled", False)
using_oidc = bool(
getattr(settings, "authentik_client_id", None)
and getattr(settings, "authentik_client_secret", None)
and getattr(settings, "authentik_config_url", None)
)
auth_method = "OIDC" if using_oidc else "Basic Auth" if auth_enabled else "None"
providers["Authentication"] = {
"name": "Authentication",
"icon": "fa-solid fa-lock",
"configured": bool(auth_enabled and
(getattr(settings, 'admin_username', None) or
using_oidc)),
"configured": bool(auth_enabled and (getattr(settings, "admin_username", None) or using_oidc)),
"enabled": auth_enabled,
"description": "Access control and user authentication",
"details": {
"method": auth_method,
"provider_name": getattr(settings, 'oauth_provider_name', 'Not set') if using_oidc else "N/A",
"session_security": "Configured" if getattr(settings, 'session_secret', None) else "Not configured"
}
"provider_name": getattr(settings, "oauth_provider_name", "Not set") if using_oidc else "N/A",
"session_security": "Configured" if getattr(settings, "session_secret", None) else "Not configured",
},
}
# Add Notification configuration - Make sure this provider is near the top of the list
providers["Notifications"] = {
"name": "Notifications",
"icon": "fa-solid fa-bell",
"configured": bool(getattr(settings, 'notification_urls', None)),
"configured": bool(getattr(settings, "notification_urls", None)),
"enabled": True,
"description": "Send system notifications via various services",
"details": {
"services": str(len(getattr(settings, 'notification_urls', []))) + " service(s) configured" if getattr(settings, 'notification_urls', None) else "Not configured",
"task_failure": getattr(settings, 'notify_on_task_failure', True),
"credential_failure": getattr(settings, 'notify_on_credential_failure', True),
"startup": getattr(settings, 'notify_on_startup', True),
"shutdown": getattr(settings, 'notify_on_shutdown', False)
"services": (
str(len(getattr(settings, "notification_urls", []))) + " service(s) configured"
if getattr(settings, "notification_urls", None)
else "Not configured"
),
"task_failure": getattr(settings, "notify_on_task_failure", True),
"credential_failure": getattr(settings, "notify_on_credential_failure", True),
"startup": getattr(settings, "notify_on_startup", True),
"shutdown": getattr(settings, "notify_on_shutdown", False),
},
"testable": True,
"test_endpoint": "/api/diagnostic/test-notification"
"test_endpoint": "/api/diagnostic/test-notification",
}
# Add AI services first
providers["OpenAI"] = {
"name": "OpenAI",
"icon": "fa-brands fa-openai",
"configured": bool(getattr(settings, 'openai_api_key', None) and
str(getattr(settings, 'openai_api_key', '')).startswith('sk-')),
"configured": bool(
getattr(settings, "openai_api_key", None) and str(getattr(settings, "openai_api_key", "")).startswith("sk-")
),
"enabled": True,
"description": "AI-powered document analysis and metadata extraction",
"details": {
"api_key": mask_sensitive_value(getattr(settings, 'openai_api_key', None)),
"base_url": getattr(settings, 'openai_base_url', 'https://api.openai.com/v1'),
"model": getattr(settings, 'openai_model', 'gpt-4')
}
"api_key": mask_sensitive_value(getattr(settings, "openai_api_key", None)),
"base_url": getattr(settings, "openai_base_url", "https://api.openai.com/v1"),
"model": getattr(settings, "openai_model", "gpt-4"),
},
}
providers["Azure AI"] = {
"name": "Azure AI",
"icon": "fa-solid fa-robot",
"configured": bool(getattr(settings, 'azure_ai_key', None) and
getattr(settings, 'azure_endpoint', None)),
"configured": bool(getattr(settings, "azure_ai_key", None) and getattr(settings, "azure_endpoint", None)),
"enabled": True,
"description": "Microsoft Azure Document Intelligence",
"details": {
"api_key": mask_sensitive_value(getattr(settings, 'azure_ai_key', None)),
"endpoint": getattr(settings, 'azure_endpoint', 'Not set'),
"region": getattr(settings, 'azure_region', 'Not set')
}
"api_key": mask_sensitive_value(getattr(settings, "azure_ai_key", None)),
"endpoint": getattr(settings, "azure_endpoint", "Not set"),
"region": getattr(settings, "azure_region", "Not set"),
},
}
# Add Dropbox configuration - alphabetically ordered providers
providers["Dropbox"] = {
"name": "Dropbox",
"icon": "fa-brands fa-dropbox",
"configured": bool(getattr(settings, 'dropbox_app_key', None) and
getattr(settings, 'dropbox_app_secret', None) and
getattr(settings, 'dropbox_refresh_token', None)),
"configured": bool(
getattr(settings, "dropbox_app_key", None)
and getattr(settings, "dropbox_app_secret", None)
and getattr(settings, "dropbox_refresh_token", None)
),
"enabled": True,
"description": "Upload files to Dropbox cloud storage",
"details": {
"folder": getattr(settings, 'dropbox_folder', 'Not set'),
"app_key": getattr(settings, 'dropbox_app_key', 'Not set'),
"app_secret": mask_sensitive_value(getattr(settings, 'dropbox_app_secret', None)),
"refresh_token": mask_sensitive_value(getattr(settings, 'dropbox_refresh_token', None))
}
"folder": getattr(settings, "dropbox_folder", "Not set"),
"app_key": getattr(settings, "dropbox_app_key", "Not set"),
"app_secret": mask_sensitive_value(getattr(settings, "dropbox_app_secret", None)),
"refresh_token": mask_sensitive_value(getattr(settings, "dropbox_refresh_token", None)),
},
}
# Add Email configuration
providers["Email"] = {
"name": "Email",
"icon": "fa-solid fa-envelope",
"configured": bool(getattr(settings, 'email_host', None) and
getattr(settings, 'email_default_recipient', None)),
"configured": bool(
getattr(settings, "email_host", None) and getattr(settings, "email_default_recipient", None)
),
"enabled": True,
"description": "Send documents via email",
"details": {
"host": getattr(settings, 'email_host', 'Not set'),
"port": getattr(settings, 'email_port', 'Not set'),
"username": getattr(settings, 'email_username', 'Not set'),
"password": mask_sensitive_value(getattr(settings, 'email_password', None)),
"use_tls": getattr(settings, 'email_use_tls', 'Not set'),
"sender": getattr(settings, 'email_sender', 'Not set'),
"default_recipient": getattr(settings, 'email_default_recipient', 'Not set')
}
"host": getattr(settings, "email_host", "Not set"),
"port": getattr(settings, "email_port", "Not set"),
"username": getattr(settings, "email_username", "Not set"),
"password": mask_sensitive_value(getattr(settings, "email_password", None)),
"use_tls": getattr(settings, "email_use_tls", "Not set"),
"sender": getattr(settings, "email_sender", "Not set"),
"default_recipient": getattr(settings, "email_default_recipient", "Not set"),
},
}
# Add FTP configuration to providers
providers["FTP Storage"] = {
"name": "FTP Storage",
"icon": "fa-solid fa-server",
"configured": bool(getattr(settings, 'ftp_host', None) and
getattr(settings, 'ftp_username', None) and
getattr(settings, 'ftp_password', None)),
"configured": bool(
getattr(settings, "ftp_host", None)
and getattr(settings, "ftp_username", None)
and getattr(settings, "ftp_password", None)
),
"enabled": True,
"description": "Upload files to FTP server",
"details": {
"host": getattr(settings, 'ftp_host', 'Not set'),
"port": getattr(settings, 'ftp_port', 'Not set'),
"username": getattr(settings, 'ftp_username', 'Not set'),
"password": mask_sensitive_value(getattr(settings, 'ftp_password', None)),
"folder": getattr(settings, 'ftp_folder', 'Not set'),
"tls": getattr(settings, 'ftp_use_tls', True),
"allow_plaintext": getattr(settings, 'ftp_allow_plaintext', True)
}
"host": getattr(settings, "ftp_host", "Not set"),
"port": getattr(settings, "ftp_port", "Not set"),
"username": getattr(settings, "ftp_username", "Not set"),
"password": mask_sensitive_value(getattr(settings, "ftp_password", None)),
"folder": getattr(settings, "ftp_folder", "Not set"),
"tls": getattr(settings, "ftp_use_tls", True),
"allow_plaintext": getattr(settings, "ftp_allow_plaintext", True),
},
}
# Check Google Drive configuration
gdrive_oauth_configured = bool(getattr(settings, 'google_drive_client_id', None) and
getattr(settings, 'google_drive_client_secret', None) and
getattr(settings, 'google_drive_refresh_token', None))
gdrive_oauth_configured = bool(
getattr(settings, "google_drive_client_id", None)
and getattr(settings, "google_drive_client_secret", None)
and getattr(settings, "google_drive_refresh_token", None)
)
gdrive_sa_configured = bool(getattr(settings, 'google_drive_credentials_json', None))
gdrive_sa_configured = bool(getattr(settings, "google_drive_credentials_json", None))
# Determine if using OAuth or service account
use_oauth = getattr(settings, 'google_drive_use_oauth', False)
use_oauth = getattr(settings, "google_drive_use_oauth", False)
is_configured = (use_oauth and gdrive_oauth_configured) or (not use_oauth and gdrive_sa_configured)
providers["Google Drive"] = {
"name": "Google Drive",
"icon": "fa-brands fa-google-drive",
"configured": is_configured and bool(getattr(settings, 'google_drive_folder_id', None)),
"configured": is_configured and bool(getattr(settings, "google_drive_folder_id", None)),
"enabled": True,
"description": "Store documents in Google Drive",
"details": {
"auth_type": "OAuth" if use_oauth else "Service Account",
"client_id": getattr(settings, 'google_drive_client_id', 'Not set') if use_oauth else 'N/A',
"client_secret": mask_sensitive_value(getattr(settings, 'google_drive_client_secret', None)) if use_oauth else 'N/A',
"refresh_token": mask_sensitive_value(getattr(settings, 'google_drive_refresh_token', None)) if use_oauth else 'N/A',
"credentials_json": mask_sensitive_value(getattr(settings, 'google_drive_credentials_json', None)) if not use_oauth else 'N/A',
"folder_id": getattr(settings, 'google_drive_folder_id', 'Not set'),
"delegate": getattr(settings, 'google_drive_delegate_to', 'Not set') if not use_oauth else 'N/A'
}
"client_id": getattr(settings, "google_drive_client_id", "Not set") if use_oauth else "N/A",
"client_secret": (
mask_sensitive_value(getattr(settings, "google_drive_client_secret", None)) if use_oauth else "N/A"
),
"refresh_token": (
mask_sensitive_value(getattr(settings, "google_drive_refresh_token", None)) if use_oauth else "N/A"
),
"credentials_json": (
mask_sensitive_value(getattr(settings, "google_drive_credentials_json", None))
if not use_oauth
else "N/A"
),
"folder_id": getattr(settings, "google_drive_folder_id", "Not set"),
"delegate": getattr(settings, "google_drive_delegate_to", "Not set") if not use_oauth else "N/A",
},
}
# Check NextCloud configuration
nextcloud_url = getattr(settings, 'nextcloud_upload_url', 'Not set')
nextcloud_url = getattr(settings, "nextcloud_upload_url", "Not set")
# Extract base URL from WebDAV URL (remove the /remote.php part and everything after it)
nextcloud_base_url = nextcloud_url
if nextcloud_url != 'Not set' and nextcloud_url is not None and '/remote.php' in nextcloud_url:
nextcloud_base_url = nextcloud_url.split('/remote.php')[0]
if nextcloud_url != "Not set" and nextcloud_url is not None and "/remote.php" in nextcloud_url:
nextcloud_base_url = nextcloud_url.split("/remote.php")[0]
providers["NextCloud"] = {
"name": "NextCloud",
"icon": "fa-solid fa-cloud",
"configured": bool(getattr(settings, 'nextcloud_upload_url', None) and
getattr(settings, 'nextcloud_username', None) and
getattr(settings, 'nextcloud_password', None)),
"configured": bool(
getattr(settings, "nextcloud_upload_url", None)
and getattr(settings, "nextcloud_username", None)
and getattr(settings, "nextcloud_password", None)
),
"enabled": True,
"description": "Store documents in NextCloud",
"details": {
"url": getattr(settings, 'nextcloud_upload_url', 'Not set'),
"url": getattr(settings, "nextcloud_upload_url", "Not set"),
"base_url": nextcloud_base_url,
"username": getattr(settings, 'nextcloud_username', 'Not set'),
"password": mask_sensitive_value(getattr(settings, 'nextcloud_password', None)),
"folder": getattr(settings, 'nextcloud_folder', 'Not set')
}
"username": getattr(settings, "nextcloud_username", "Not set"),
"password": mask_sensitive_value(getattr(settings, "nextcloud_password", None)),
"folder": getattr(settings, "nextcloud_folder", "Not set"),
},
}
# Check OneDrive configuration
providers["OneDrive"] = {
"name": "OneDrive",
"icon": "fa-brands fa-microsoft",
"configured": bool(getattr(settings, 'onedrive_client_id', None) and
getattr(settings, 'onedrive_client_secret', None) and
getattr(settings, 'onedrive_refresh_token', None)),
"configured": bool(
getattr(settings, "onedrive_client_id", None)
and getattr(settings, "onedrive_client_secret", None)
and getattr(settings, "onedrive_refresh_token", None)
),
"enabled": True,
"description": "Store documents in Microsoft OneDrive",
"details": {
"client_id": getattr(settings, 'onedrive_client_id', 'Not set'),
"client_secret": mask_sensitive_value(getattr(settings, 'onedrive_client_secret', None)),
"tenant_id": getattr(settings, 'onedrive_tenant_id', 'Not set'),
"refresh_token": mask_sensitive_value(getattr(settings, 'onedrive_refresh_token', None)),
"folder": getattr(settings, 'onedrive_folder_path', 'Not set')
}
"client_id": getattr(settings, "onedrive_client_id", "Not set"),
"client_secret": mask_sensitive_value(getattr(settings, "onedrive_client_secret", None)),
"tenant_id": getattr(settings, "onedrive_tenant_id", "Not set"),
"refresh_token": mask_sensitive_value(getattr(settings, "onedrive_refresh_token", None)),
"folder": getattr(settings, "onedrive_folder_path", "Not set"),
},
}
# Check Paperless configuration
providers["Paperless-ngx"] = {
"name": "Paperless-ngx",
"icon": "fa-solid fa-file-lines",
"configured": bool(getattr(settings, 'paperless_host', None) and
getattr(settings, 'paperless_ngx_api_token', None)),
"configured": bool(
getattr(settings, "paperless_host", None) and getattr(settings, "paperless_ngx_api_token", None)
),
"enabled": True,
"description": "Document management system for digital archives",
"details": {
"host": getattr(settings, 'paperless_host', 'Not set'),
"api_token": mask_sensitive_value(getattr(settings, 'paperless_ngx_api_token', None))
}
"host": getattr(settings, "paperless_host", "Not set"),
"api_token": mask_sensitive_value(getattr(settings, "paperless_ngx_api_token", None)),
},
}
# Check S3 configuration
providers["S3 Storage"] = {
"name": "S3 Storage",
"icon": "fa-brands fa-aws",
"configured": bool(getattr(settings, 's3_bucket_name', None) and
getattr(settings, 'aws_access_key_id', None) and
getattr(settings, 'aws_secret_access_key', None)),
"configured": bool(
getattr(settings, "s3_bucket_name", None)
and getattr(settings, "aws_access_key_id", None)
and getattr(settings, "aws_secret_access_key", None)
),
"enabled": True,
"description": "Store documents in S3-compatible object storage",
"details": {
"bucket": getattr(settings, 's3_bucket_name', 'Not set'),
"region": getattr(settings, 'aws_region', 'Not set'),
"access_key_id": getattr(settings, 'aws_access_key_id', 'Not set'),
"secret_access_key": mask_sensitive_value(getattr(settings, 'aws_secret_access_key', None)),
"folder_prefix": getattr(settings, 's3_folder_prefix', 'Not set'),
"storage_class": getattr(settings, 's3_storage_class', 'Not set'),
"acl": getattr(settings, 's3_acl', 'Not set')
}
"bucket": getattr(settings, "s3_bucket_name", "Not set"),
"region": getattr(settings, "aws_region", "Not set"),
"access_key_id": getattr(settings, "aws_access_key_id", "Not set"),
"secret_access_key": mask_sensitive_value(getattr(settings, "aws_secret_access_key", None)),
"folder_prefix": getattr(settings, "s3_folder_prefix", "Not set"),
"storage_class": getattr(settings, "s3_storage_class", "Not set"),
"acl": getattr(settings, "s3_acl", "Not set"),
},
}
# Check SFTP configuration
providers["SFTP Storage"] = {
"name": "SFTP Storage",
"icon": "fa-solid fa-lock",
"configured": bool(getattr(settings, 'sftp_host', None) and
getattr(settings, 'sftp_username', None) and
(getattr(settings, 'sftp_password', None) or
getattr(settings, 'sftp_private_key', None))),
"configured": bool(
getattr(settings, "sftp_host", None)
and getattr(settings, "sftp_username", None)
and (getattr(settings, "sftp_password", None) or getattr(settings, "sftp_private_key", None))
),
"enabled": True,
"description": "Upload files to SFTP server",
"details": {
"host": getattr(settings, 'sftp_host', 'Not set'),
"port": getattr(settings, 'sftp_port', 'Not set'),
"username": getattr(settings, 'sftp_username', 'Not set'),
"password": mask_sensitive_value(getattr(settings, 'sftp_password', None)),
"private_key": getattr(settings, 'sftp_private_key', 'Not set'),
"private_key_passphrase": mask_sensitive_value(getattr(settings, 'sftp_private_key_passphrase', None)),
"folder": getattr(settings, 'sftp_folder', 'Not set')
}
"host": getattr(settings, "sftp_host", "Not set"),
"port": getattr(settings, "sftp_port", "Not set"),
"username": getattr(settings, "sftp_username", "Not set"),
"password": mask_sensitive_value(getattr(settings, "sftp_password", None)),
"private_key": getattr(settings, "sftp_private_key", "Not set"),
"private_key_passphrase": mask_sensitive_value(getattr(settings, "sftp_private_key_passphrase", None)),
"folder": getattr(settings, "sftp_folder", "Not set"),
},
}
# Add Uptime Kuma configuration
providers["Uptime Kuma"] = {
"name": "Uptime Kuma",
"icon": "fa-solid fa-heart-pulse",
"configured": bool(getattr(settings, 'uptime_kuma_url', None)),
"configured": bool(getattr(settings, "uptime_kuma_url", None)),
"enabled": True,
"description": "Server monitoring and status page",
"details": {
"url": getattr(settings, 'uptime_kuma_url', 'Not set'),
"ping_interval": getattr(settings, 'uptime_kuma_ping_interval', 'Not set')
}
"url": getattr(settings, "uptime_kuma_url", "Not set"),
"ping_interval": getattr(settings, "uptime_kuma_ping_interval", "Not set"),
},
}
# Check WebDAV configuration
providers["WebDAV"] = {
"name": "WebDAV",
"icon": "fa-solid fa-globe",
"configured": bool(getattr(settings, 'webdav_url', None) and
getattr(settings, 'webdav_username', None) and
getattr(settings, 'webdav_password', None)),
"configured": bool(
getattr(settings, "webdav_url", None)
and getattr(settings, "webdav_username", None)
and getattr(settings, "webdav_password", None)
),
"enabled": True,
"description": "Store documents on WebDAV servers",
"details": {
"url": getattr(settings, 'webdav_url', 'Not set'),
"username": getattr(settings, 'webdav_username', 'Not set'),
"password": mask_sensitive_value(getattr(settings, 'webdav_password', None)),
"folder": getattr(settings, 'webdav_folder', 'Not set'),
"verify_ssl": getattr(settings, 'webdav_verify_ssl', 'Not set')
}
"url": getattr(settings, "webdav_url", "Not set"),
"username": getattr(settings, "webdav_username", "Not set"),
"password": mask_sensitive_value(getattr(settings, "webdav_password", None)),
"folder": getattr(settings, "webdav_folder", "Not set"),
"verify_ssl": getattr(settings, "webdav_verify_ssl", "Not set"),
},
}
return providers
app/utils/config_validator/settings_display.py
+62 -72
View File
@@ -3,31 +3,47 @@ Module for displaying and organizing settings information
"""
import logging
from app.config import settings
from app.utils.config_validator.masking import mask_sensitive_value
logger = logging.getLogger(__name__)
def dump_all_settings():
"""Log all settings values for diagnostic purposes"""
logger.info("--- DUMPING ALL SETTINGS FOR DIAGNOSTIC PURPOSES ---")
for key in dir(settings):
if not key.startswith('_') and not callable(getattr(settings, key)):
if not key.startswith("_") and not callable(getattr(settings, key)):
value = getattr(settings, key)
# Mask sensitive values in logs
if key.lower().find('password') >= 0 or key.lower().find('secret') >= 0 or key.lower().find('token') >= 0 or key.lower().find('key') >= 0:
if (
key.lower().find("password") >= 0
or key.lower().find("secret") >= 0
or key.lower().find("token") >= 0
or key.lower().find("key") >= 0
):
if value:
if isinstance(value, str) and len(value) > 10:
visible_start = max(1, len(value) // 3)
visible_end = max(1, len(value) // 4)
value = f"{value[:visible_start]}{'*' * (len(value) - visible_start - visible_end)}{value[-visible_end:]}"
value = (
f"{value[:visible_start]}"
f"{'*' * (len(value) - visible_start - visible_end)}"
f"{value[-visible_end:]}"
)
else:
value = f"{value[:2]}{'*' * (len(value) - 4)}{value[-2:]}" if isinstance(value, str) and len(value) > 4 else "****"
value = (
f"{value[:2]}{'*' * (len(value) - 4)}{value[-2:]}"
if isinstance(value, str) and len(value) > 4
else "****"
)
# Special handling for notification URLs
if key == 'notification_urls' and value:
if key == "notification_urls" and value:
try:
from app.utils.notification import _mask_sensitive_url
if isinstance(value, list):
masked_urls = [_mask_sensitive_url(url) for url in value]
logger.info(f"{key}: {masked_urls}")
@@ -40,6 +56,7 @@ def dump_all_settings():
logger.info(f"{key}: {value}")
logger.info("--- END OF SETTINGS DUMP ---")
def get_settings_for_display(show_values=False):
"""
Group settings into logical categories and check if they are configured.
@@ -51,29 +68,21 @@ def get_settings_for_display(show_values=False):
# First include system info with version in result
result = {
"System Info": [
{
"name": "App Version",
"value": settings.version,
"is_configured": True
},
{
"name": "Build Date",
"value": settings.build_date,
"is_configured": True
}
{"name": "App Version", "value": settings.version, "is_configured": True},
{"name": "Build Date", "value": settings.build_date, "is_configured": True},
]
}
# Define categories and their settings
categories = {
"Core": [
"debug", # Explicitly include debug setting
"debug", # Explicitly include debug setting
"external_hostname",
"workdir",
"database_url",
"redis_url",
"gotenberg_url",
"allow_file_delete" # Added allow_file_delete to Core settings
"allow_file_delete", # Added allow_file_delete to Core settings
],
"Authentication": [
"auth_enabled",
@@ -83,7 +92,7 @@ def get_settings_for_display(show_values=False):
"authentik_client_id",
"authentik_client_secret",
"authentik_config_url",
"oauth_provider_name"
"oauth_provider_name",
],
"Email": [
"email_host",
@@ -92,7 +101,7 @@ def get_settings_for_display(show_values=False):
"email_password",
"email_use_tls",
"email_sender",
"email_default_recipient"
"email_default_recipient",
],
"IMAP": [
"imap1_host",
@@ -108,24 +117,11 @@ def get_settings_for_display(show_values=False):
"imap2_password",
"imap2_ssl",
"imap2_poll_interval_minutes",
"imap2_delete_after_process"
],
"Dropbox": [
"dropbox_app_key",
"dropbox_app_secret",
"dropbox_folder",
"dropbox_refresh_token"
],
"NextCloud": [
"nextcloud_upload_url",
"nextcloud_username",
"nextcloud_password",
"nextcloud_folder"
],
"Paperless": [
"paperless_host",
"paperless_ngx_api_token"
"imap2_delete_after_process",
],
"Dropbox": ["dropbox_app_key", "dropbox_app_secret", "dropbox_folder", "dropbox_refresh_token"],
"NextCloud": ["nextcloud_upload_url", "nextcloud_username", "nextcloud_password", "nextcloud_folder"],
"Paperless": ["paperless_host", "paperless_ngx_api_token"],
"Google Drive": [
"google_drive_use_oauth",
"google_drive_client_id",
@@ -133,22 +129,16 @@ def get_settings_for_display(show_values=False):
"google_drive_refresh_token",
"google_drive_credentials_json",
"google_drive_folder_id",
"google_drive_delegate_to"
"google_drive_delegate_to",
],
"OneDrive": [
"onedrive_client_id",
"onedrive_client_secret",
"onedrive_tenant_id",
"onedrive_refresh_token",
"onedrive_folder_path"
],
"WebDAV": [
"webdav_url",
"webdav_username",
"webdav_password",
"webdav_folder",
"webdav_verify_ssl"
"onedrive_folder_path",
],
"WebDAV": ["webdav_url", "webdav_username", "webdav_password", "webdav_folder", "webdav_verify_ssl"],
"SFTP": [
"sftp_host",
"sftp_port",
@@ -156,7 +146,7 @@ def get_settings_for_display(show_values=False):
"sftp_password",
"sftp_folder",
"sftp_private_key",
"sftp_private_key_passphrase"
"sftp_private_key_passphrase",
],
"FTP": [
"ftp_host",
@@ -165,7 +155,7 @@ def get_settings_for_display(show_values=False):
"ftp_password",
"ftp_folder",
"ftp_use_tls",
"ftp_allow_plaintext"
"ftp_allow_plaintext",
],
"S3/AWS": [
"aws_access_key_id",
@@ -174,7 +164,7 @@ def get_settings_for_display(show_values=False):
"s3_bucket_name",
"s3_folder_prefix",
"s3_storage_class",
"s3_acl"
"s3_acl",
],
"AI Services": [
"openai_api_key",
@@ -182,28 +172,28 @@ def get_settings_for_display(show_values=False):
"openai_model",
"azure_ai_key",
"azure_endpoint",
"azure_region"
],
"Monitoring": [
"uptime_kuma_url",
"uptime_kuma_ping_interval"
"azure_region",
],
"Monitoring": ["uptime_kuma_url", "uptime_kuma_ping_interval"],
"Notifications": [
"notification_urls",
"notify_on_task_failure",
"notify_on_credential_failure",
"notify_on_startup",
"notify_on_shutdown"
]
"notify_on_shutdown",
],
}
# Handle any settings that don't fit into the predefined categories
all_settings = set([key for key in dir(settings)
if not key.startswith('_') and
not callable(getattr(settings, key)) and
key not in ["model_computed_fields", "model_config",
"model_extra", "model_fields",
"model_fields_set"]])
all_settings = set(
[
key
for key in dir(settings)
if not key.startswith("_")
and not callable(getattr(settings, key))
and key not in ["model_computed_fields", "model_config", "model_extra", "model_fields", "model_fields_set"]
]
)
# Ensure 'version' is excluded since we display it separately
all_settings.discard("version")
@@ -225,20 +215,24 @@ def get_settings_for_display(show_values=False):
# List of patterns that indicate sensitive values
sensitive_patterns = [
'password', 'secret', 'token', 'api_key', 'private_key',
'credentials', 'access_key', 'ai_key'
"password",
"secret",
"token",
"api_key",
"private_key",
"credentials",
"access_key",
"ai_key",
]
# Check if this is a sensitive value that should be masked
is_sensitive = any(
pattern in key.lower() for pattern in sensitive_patterns
)
is_sensitive = any(pattern in key.lower() for pattern in sensitive_patterns)
# Special handling for "auth" to avoid matching prefixes like "authentik"
if not is_sensitive and "auth" in key.lower():
# Only mark as sensitive if "auth" is a standalone word or at the end
# This avoids matching "authentik" as sensitive
parts = key.lower().split('_')
parts = key.lower().split("_")
is_sensitive = any(part == "auth" for part in parts) or key.lower().endswith("auth")
# Mask sensitive values regardless of debug mode
@@ -253,11 +247,7 @@ def get_settings_for_display(show_values=False):
if is_configured and isinstance(value, str):
is_configured = len(value) > 0
items.append({
"name": key,
"value": value,
"is_configured": is_configured
})
items.append({"name": key, "value": value, "is_configured": is_configured})
if items: # Only add categories that have items
result[category] = items
app/utils/config_validator/validators.py
+72 -62
View File
@@ -1,24 +1,26 @@
#!/usr/bin/env python3
import logging
import os
import socket
import logging
from app.config import settings
logger = logging.getLogger(__name__)
def validate_email_config():
"""Validates email configuration settings"""
issues = []
# Check for required email settings
if not getattr(settings, 'email_host', None):
if not getattr(settings, "email_host", None):
issues.append("EMAIL_HOST is not configured")
if not getattr(settings, 'email_port', None):
if not getattr(settings, "email_port", None):
issues.append("EMAIL_PORT is not configured")
# Test SMTP server connectivity if host is configured
if getattr(settings, 'email_host', None) and getattr(settings, 'email_port', None):
if getattr(settings, "email_host", None) and getattr(settings, "email_port", None):
try:
# Attempt to resolve the hostname
socket.gethostbyname(settings.email_host)
@@ -26,156 +28,168 @@ def validate_email_config():
issues.append(f"Cannot resolve email host: {settings.email_host}")
# Check for authentication settings
if not getattr(settings, 'email_username', None):
if not getattr(settings, "email_username", None):
issues.append("EMAIL_USERNAME is not configured")
if not getattr(settings, 'email_password', None):
if not getattr(settings, "email_password", None):
issues.append("EMAIL_PASSWORD is not configured")
return issues
def validate_auth_config():
"""Validates authentication configuration settings"""
issues = []
# If auth is enabled, check for required settings
if getattr(settings, 'auth_enabled', False):
if getattr(settings, "auth_enabled", False):
# Check for session secret
if not getattr(settings, 'session_secret', None):
if not getattr(settings, "session_secret", None):
issues.append("SESSION_SECRET is not configured but AUTH_ENABLED is True")
elif len(getattr(settings, 'session_secret', '')) < 32:
elif len(getattr(settings, "session_secret", "")) < 32:
issues.append("SESSION_SECRET must be at least 32 characters long")
# Check if using simple authentication or OIDC
using_simple_auth = bool(getattr(settings, 'admin_username', None) and
getattr(settings, 'admin_password', None))
using_simple_auth = bool(
getattr(settings, "admin_username", None) and getattr(settings, "admin_password", None)
)
using_oidc = bool(getattr(settings, 'authentik_client_id', None) and
getattr(settings, 'authentik_client_secret', None) and
getattr(settings, 'authentik_config_url', None))
using_oidc = bool(
getattr(settings, "authentik_client_id", None)
and getattr(settings, "authentik_client_secret", None)
and getattr(settings, "authentik_config_url", None)
)
if not using_simple_auth and not using_oidc:
issues.append("Neither simple authentication nor OIDC are properly configured")
# If using OIDC, check for provider name
if using_oidc and not getattr(settings, 'oauth_provider_name', None):
if using_oidc and not getattr(settings, "oauth_provider_name", None):
issues.append("OAUTH_PROVIDER_NAME is not configured but OIDC is enabled")
return issues
def validate_storage_configs():
"""Validates configuration for all storage providers"""
issues = {}
# Validate Dropbox config
dropbox_issues = []
if not (getattr(settings, 'dropbox_app_key', None) and
getattr(settings, 'dropbox_app_secret', None) and
getattr(settings, 'dropbox_refresh_token', None)):
if not (
getattr(settings, "dropbox_app_key", None)
and getattr(settings, "dropbox_app_secret", None)
and getattr(settings, "dropbox_refresh_token", None)
):
dropbox_issues.append("Dropbox credentials are not fully configured")
issues['dropbox'] = dropbox_issues
issues["dropbox"] = dropbox_issues
# Validate Nextcloud config
nextcloud_issues = []
if not (getattr(settings, 'nextcloud_upload_url', None) and
getattr(settings, 'nextcloud_username', None) and
getattr(settings, 'nextcloud_password', None)):
if not (
getattr(settings, "nextcloud_upload_url", None)
and getattr(settings, "nextcloud_username", None)
and getattr(settings, "nextcloud_password", None)
):
nextcloud_issues.append("Nextcloud credentials are not fully configured")
issues['nextcloud'] = nextcloud_issues
issues["nextcloud"] = nextcloud_issues
# Validate SFTP config
sftp_issues = []
if not getattr(settings, 'sftp_host', None):
if not getattr(settings, "sftp_host", None):
sftp_issues.append("SFTP_HOST is not configured")
sftp_key_path = getattr(settings, 'sftp_private_key', None)
sftp_key_path = getattr(settings, "sftp_private_key", None)
if sftp_key_path and not os.path.exists(sftp_key_path):
sftp_issues.append(f"SFTP_KEY_PATH file not found: {sftp_key_path}")
if not sftp_key_path and not getattr(settings, 'sftp_password', None):
if not sftp_key_path and not getattr(settings, "sftp_password", None):
sftp_issues.append("Neither SFTP_KEY_PATH nor SFTP_PASSWORD is configured")
issues['sftp'] = sftp_issues
issues["sftp"] = sftp_issues
# Validate Email sending
email_issues = []
if not getattr(settings, 'email_host', None):
if not getattr(settings, "email_host", None):
email_issues.append("EMAIL_HOST is not configured")
if not getattr(settings, 'email_default_recipient', None):
if not getattr(settings, "email_default_recipient", None):
email_issues.append("EMAIL_DEFAULT_RECIPIENT is not configured")
issues['email'] = email_issues
issues["email"] = email_issues
# Validate S3
s3_issues = []
if not getattr(settings, 's3_bucket_name', None):
if not getattr(settings, "s3_bucket_name", None):
s3_issues.append("S3_BUCKET_NAME is not configured")
if not (getattr(settings, 'aws_access_key_id', None) and
getattr(settings, 'aws_secret_access_key', None)):
if not (getattr(settings, "aws_access_key_id", None) and getattr(settings, "aws_secret_access_key", None)):
s3_issues.append("AWS credentials are not configured")
issues['s3'] = s3_issues
issues["s3"] = s3_issues
# Validate FTP
ftp_issues = []
if not getattr(settings, 'ftp_host', None):
if not getattr(settings, "ftp_host", None):
ftp_issues.append("FTP_HOST is not configured")
if not getattr(settings, 'ftp_username', None):
if not getattr(settings, "ftp_username", None):
ftp_issues.append("FTP_USERNAME is not configured")
if not getattr(settings, 'ftp_password', None):
if not getattr(settings, "ftp_password", None):
ftp_issues.append("FTP_PASSWORD is not configured")
issues['ftp'] = ftp_issues
issues["ftp"] = ftp_issues
# Validate WebDAV
webdav_issues = []
if not getattr(settings, 'webdav_url', None):
if not getattr(settings, "webdav_url", None):
webdav_issues.append("WEBDAV_URL is not configured")
if not getattr(settings, 'webdav_username', None):
if not getattr(settings, "webdav_username", None):
webdav_issues.append("WEBDAV_USERNAME is not configured")
if not getattr(settings, 'webdav_password', None):
if not getattr(settings, "webdav_password", None):
webdav_issues.append("WEBDAV_PASSWORD is not configured")
issues['webdav'] = webdav_issues
issues["webdav"] = webdav_issues
# Validate Google Drive
gdrive_issues = []
if not getattr(settings, 'google_drive_credentials_json', None):
if not getattr(settings, "google_drive_credentials_json", None):
gdrive_issues.append("GOOGLE_DRIVE_CREDENTIALS_JSON is not configured")
if not getattr(settings, 'google_drive_folder_id', None):
if not getattr(settings, "google_drive_folder_id", None):
gdrive_issues.append("GOOGLE_DRIVE_FOLDER_ID is not configured")
issues['google_drive'] = gdrive_issues
issues["google_drive"] = gdrive_issues
# Validate Paperless
paperless_issues = []
if not getattr(settings, 'paperless_host', None):
if not getattr(settings, "paperless_host", None):
paperless_issues.append("PAPERLESS_HOST is not configured")
if not getattr(settings, 'paperless_ngx_api_token', None):
if not getattr(settings, "paperless_ngx_api_token", None):
paperless_issues.append("PAPERLESS_NGX_API_TOKEN is not configured")
issues['paperless'] = paperless_issues
issues["paperless"] = paperless_issues
# Validate OneDrive
onedrive_issues = []
if not (getattr(settings, 'onedrive_client_id', None) and
getattr(settings, 'onedrive_client_secret', None) and
getattr(settings, 'onedrive_refresh_token', None)):
if not (
getattr(settings, "onedrive_client_id", None)
and getattr(settings, "onedrive_client_secret", None)
and getattr(settings, "onedrive_refresh_token", None)
):
onedrive_issues.append("OneDrive credentials are not fully configured")
issues['onedrive'] = onedrive_issues
issues["onedrive"] = onedrive_issues
# Validate Uptime Kuma
uptime_kuma_issues = []
if not getattr(settings, 'uptime_kuma_url', None):
if not getattr(settings, "uptime_kuma_url", None):
uptime_kuma_issues.append("UPTIME_KUMA_URL is not configured")
issues['uptime_kuma'] = uptime_kuma_issues
issues["uptime_kuma"] = uptime_kuma_issues
return issues
def validate_notification_config():
"""Check notification configuration"""
issues = []
# Check if any notification URLs are configured
if not getattr(settings, 'notification_urls', None):
if not getattr(settings, "notification_urls", None):
issues.append("No notification URLs configured")
else:
try:
# Try initializing Apprise to validate URLs
import apprise
a = apprise.Apprise()
for url in settings.notification_urls:
@@ -195,6 +209,7 @@ def validate_notification_config():
return issues
def check_all_configs():
"""Run all configuration validations and log results"""
from app.utils.config_validator.settings_display import dump_all_settings
@@ -202,7 +217,7 @@ def check_all_configs():
logger.info("Validating application configuration...")
# Check if debug is enabled and dump all settings if it is
if hasattr(settings, 'debug') and settings.debug:
if hasattr(settings, "debug") and settings.debug:
dump_all_settings()
# Check auth config
@@ -235,9 +250,4 @@ def check_all_configs():
logger.info("Notification configuration OK")
# Return all identified issues
return {
'auth': auth_issues,
'email': email_issues,
'storage': storage_issues,
'notification': notification_issues
}
return {"auth": auth_issues, "email": email_issues, "storage": storage_issues, "notification": notification_issues}
app/utils/encryption.py
+7 -6
View File
@@ -5,9 +5,9 @@ Uses Fernet symmetric encryption with a key derived from SESSION_SECRET.
This provides encryption at rest for sensitive configuration values.
"""
import logging
import base64
import hashlib
import logging
from typing import Optional
logger = logging.getLogger(__name__)
@@ -33,11 +33,12 @@ def _get_cipher_suite():
if _cipher_suite is None:
try:
from cryptography.fernet import Fernet
from app.config import settings
# Derive a Fernet-compatible key from SESSION_SECRET
# Fernet requires a 32-byte base64-encoded key
secret = settings.session_secret.encode('utf-8')
secret = settings.session_secret.encode("utf-8")
# Use SHA256 to get exactly 32 bytes, then base64 encode
key_bytes = hashlib.sha256(secret).digest()
@@ -81,9 +82,9 @@ def encrypt_value(plaintext: Optional[str]) -> Optional[str]:
return plaintext
try:
encrypted_bytes = cipher.encrypt(plaintext.encode('utf-8'))
encrypted_bytes = cipher.encrypt(plaintext.encode("utf-8"))
# Prefix with "enc:" to identify encrypted values
return "enc:" + encrypted_bytes.decode('utf-8')
return "enc:" + encrypted_bytes.decode("utf-8")
except Exception as e:
logger.error(f"Encryption failed: {e}")
# Fall back to plaintext
@@ -116,9 +117,9 @@ def decrypt_value(ciphertext: Optional[str]) -> Optional[str]:
try:
# Remove "enc:" prefix and decrypt
encrypted_bytes = ciphertext[4:].encode('utf-8')
encrypted_bytes = ciphertext[4:].encode("utf-8")
plaintext_bytes = cipher.decrypt(encrypted_bytes)
return plaintext_bytes.decode('utf-8')
return plaintext_bytes.decode("utf-8")
except Exception as e:
logger.error(f"Decryption failed: {e}")
return "[DECRYPTION FAILED]"
app/utils/file_operations.py
+1
View File
@@ -1,5 +1,6 @@
import hashlib
def hash_file(filepath, chunk_size=65536):
"""
Returns the SHA-256 hash of the file at 'filepath'.
app/utils/file_status.py
+14 -18
View File
@@ -1,8 +1,11 @@
"""
Utility functions for file processing status determination.
"""
from typing import Dict, List
from sqlalchemy.orm import Session
from app.models import ProcessingLog
@@ -18,9 +21,9 @@ def get_file_processing_status(db: Session, file_id: int) -> Dict:
dict with status, last_step, and has_errors
"""
# Get all logs for this file
logs = db.query(ProcessingLog).filter(
ProcessingLog.file_id == file_id
).order_by(ProcessingLog.timestamp.desc()).all()
logs = (
db.query(ProcessingLog).filter(ProcessingLog.file_id == file_id).order_by(ProcessingLog.timestamp.desc()).all()
)
return _compute_status_from_logs(logs)
@@ -37,9 +40,12 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
dict mapping file_id to status dict
"""
# Get all logs for these files in one query
logs = db.query(ProcessingLog).filter(
ProcessingLog.file_id.in_(file_ids)
).order_by(ProcessingLog.file_id, ProcessingLog.timestamp.desc()).all()
logs = (
db.query(ProcessingLog)
.filter(ProcessingLog.file_id.in_(file_ids))
.order_by(ProcessingLog.file_id, ProcessingLog.timestamp.desc())
.all()
)
# Group logs by file_id
logs_by_file = {}
@@ -68,12 +74,7 @@ def _compute_status_from_logs(logs: List[ProcessingLog]) -> Dict:
dict with status, last_step, has_errors, and total_steps
"""
if not logs:
return {
"status": "pending",
"last_step": None,
"has_errors": False,
"total_steps": 0
}
return {"status": "pending", "last_step": None, "has_errors": False, "total_steps": 0}
# Check for failures
has_errors = any(log.status == "failure" for log in logs)
@@ -94,9 +95,4 @@ def _compute_status_from_logs(logs: List[ProcessingLog]) -> Dict:
else:
status = "pending"
return {
"status": status,
"last_step": latest_log.step_name,
"has_errors": has_errors,
"total_steps": len(logs)
}
return {"status": status, "last_step": latest_log.step_name, "has_errors": has_errors, "total_steps": len(logs)}
app/utils/filename_utils.py
+13 -10
View File
@@ -1,12 +1,13 @@
import logging
import os
import re
import uuid
import logging
from datetime import datetime
from pathlib import Path
logger = logging.getLogger(__name__)
def get_unique_filename(original_path, check_exists_func=None):
"""
Generates a unique filename by appending a timestamp or counter when a collision occurs.
@@ -67,6 +68,7 @@ def get_unique_filename(original_path, check_exists_func=None):
return new_path
def sanitize_filename(filename):
"""
Sanitize a filename to ensure it's valid across different file systems.
@@ -79,21 +81,22 @@ def sanitize_filename(filename):
"""
# Replace characters that are problematic in various filesystems
# Keep only alphanumeric, dash, underscore, period, and space
sanitized = re.sub(r'[^\w\-\. ]', '_', filename)
sanitized = re.sub(r"[^\w\-\. ]", "_", filename)
# Replace multiple spaces/underscores with single ones
sanitized = re.sub(r'__+', '_', sanitized)
sanitized = re.sub(r' +', ' ', sanitized)
sanitized = re.sub(r"__+", "_", sanitized)
sanitized = re.sub(r" +", " ", sanitized)
# Trim leading/trailing spaces and periods which cause issues in Windows
sanitized = sanitized.strip('. ')
sanitized = sanitized.strip(". ")
# Ensure the filename isn't empty after sanitization
if not sanitized or sanitized == '.':
if not sanitized or sanitized == ".":
sanitized = f"document_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
return sanitized
def extract_remote_path(file_path, base_dir, remote_base=""):
"""
Extract a remote path for a file by preserving its directory structure
@@ -114,14 +117,14 @@ def extract_remote_path(file_path, base_dir, remote_base=""):
# Skip 'processed' directory if it's in the path
path_parts = rel_path.split(os.sep)
if 'processed' in path_parts:
if "processed" in path_parts:
# Remove 'processed' from the path
path_parts.remove('processed')
path_parts.remove("processed")
rel_path = os.path.join(*path_parts)
# Combine with remote base path
if remote_base:
if remote_base.startswith('/'):
if remote_base.startswith("/"):
# Handle absolute path for services like Dropbox
remote_path = os.path.join(remote_base[1:], rel_path)
else:
@@ -130,6 +133,6 @@ def extract_remote_path(file_path, base_dir, remote_base=""):
remote_path = rel_path
# Convert to forward slashes for compatibility with most cloud services
remote_path = remote_path.replace(os.sep, '/')
remote_path = remote_path.replace(os.sep, "/")
return remote_path
app/utils/logging.py
+1
View File
@@ -1,6 +1,7 @@
from app.database import SessionLocal
from app.models import ProcessingLog
def log_task_progress(task_id, step_name, status, message=None, file_id=None):
"""
Logs the progress of a Celery task to the database.
app/utils/notification.py
+29 -43
View File
@@ -1,6 +1,7 @@
import apprise
import logging
from typing import List, Optional, Dict, Any, Union
from typing import Any, Dict, List, Optional
import apprise
from app.config import settings
@@ -9,6 +10,7 @@ logger = logging.getLogger(__name__)
# Global Apprise instance
_apprise = None
def init_apprise() -> apprise.Apprise:
"""Initialize the Apprise instance with configured notification services"""
global _apprise
@@ -29,24 +31,27 @@ def init_apprise() -> apprise.Apprise:
return _apprise
def _mask_sensitive_url(url: str) -> str:
"""Mask sensitive parts of notification URLs for logging"""
# Simple masking for common URL formats with credentials
import re
# Match patterns like user:pass@host or token in URL parameters
masked = re.sub(r'://([^:]+):([^@]+)@', r'://\1:****@', url)
masked = re.sub(r'(discord://)[^/]+/[^/]+', r'\1webhook_id/****', masked)
masked = re.sub(r'(tgram://)[^/]+/[^/]+', r'\1bot_token/****', masked)
masked = re.sub(r'([?&](token|key|api_key|password|secret)=)([^&]+)', r'\1****', masked)
masked = re.sub(r"://([^:]+):([^@]+)@", r"://\1:****@", url)
masked = re.sub(r"(discord://)[^/]+/[^/]+", r"\1webhook_id/****", masked)
masked = re.sub(r"(tgram://)[^/]+/[^/]+", r"\1bot_token/****", masked)
masked = re.sub(r"([?&](token|key|api_key|password|secret)=)([^&]+)", r"\1****", masked)
return masked
def send_notification(
title: str,
message: str,
notification_type: str = "info",
tags: Optional[List[str]] = None,
attachments: Optional[List[str]] = None,
data: Optional[Dict[str, Any]] = None
data: Optional[Dict[str, Any]] = None,
) -> bool:
"""
Send a notification through all configured channels
@@ -89,12 +94,7 @@ def send_notification(
for server in apprise_obj.servers: # Iterate through the list directly
try:
service_name = str(server).split("://")[0] if "://" in str(server) else str(server)
service_result = server.notify(
title=title,
body=message,
notify_type=notify_type,
attach=attachments
)
service_result = server.notify(title=title, body=message, notify_type=notify_type, attach=attachments)
if service_result:
successful_services += 1
@@ -117,6 +117,7 @@ def send_notification(
logger.exception(f"Error sending notification: {e}")
return False
def notify_celery_failure(task_name: str, task_id: str, exc: Exception, args: list, kwargs: dict) -> bool:
"""Send a notification about a failed Celery task"""
if not settings.notify_on_task_failure:
@@ -131,12 +132,10 @@ Arguments: {args}
Keyword arguments: {kwargs}
"""
return send_notification(
title=title,
message=message,
notification_type="failure",
tags=["celery", "failure", task_name]
title=title, message=message, notification_type="failure", tags=["celery", "failure", task_name]
)
def notify_credential_failure(service_name: str, error: str) -> bool:
"""Send a notification about a credential failure"""
if not settings.notify_on_credential_failure:
@@ -150,39 +149,29 @@ The credentials for {service_name} have failed:
Please check and update the credentials in the system settings.
"""
return send_notification(
title=title,
message=message,
notification_type="warning",
tags=["credentials", "warning", service_name]
title=title, message=message, notification_type="warning", tags=["credentials", "warning", service_name]
)
def notify_startup() -> bool:
"""Send a notification that the application has started"""
if not settings.notify_on_startup:
return False
title = f"DocuElevate Started"
title = "DocuElevate Started"
message = f"DocuElevate has been started successfully on {settings.external_hostname}"
return send_notification(
title=title,
message=message,
notification_type="success",
tags=["system", "startup"]
)
return send_notification(title=title, message=message, notification_type="success", tags=["system", "startup"])
def notify_shutdown() -> bool:
"""Send a notification that the application is shutting down"""
if not settings.notify_on_shutdown:
return False
title = f"DocuElevate Shutting Down"
title = "DocuElevate Shutting Down"
message = f"DocuElevate on {settings.external_hostname} is shutting down"
return send_notification(
title=title,
message=message,
notification_type="info",
tags=["system", "shutdown"]
)
return send_notification(title=title, message=message, notification_type="info", tags=["system", "shutdown"])
def notify_file_processed(filename: str, file_size: int, metadata: dict, destinations: list) -> bool:
"""Send a notification that a file has been successfully processed"""
@@ -194,12 +183,12 @@ def notify_file_processed(filename: str, file_size: int, metadata: dict, destina
size_str = f"{size_mb:.2f} MB" if size_mb >= 1 else f"{file_size / 1024:.2f} KB"
# Extract key metadata fields
doc_type = metadata.get('document_type', 'Unknown')
tags = metadata.get('tags', [])
tags_str = ', '.join(tags) if tags else 'None'
doc_type = metadata.get("document_type", "Unknown")
tags = metadata.get("tags", [])
tags_str = ", ".join(tags) if tags else "None"
# Format destinations
destinations_str = ', '.join(destinations) if destinations else 'None configured'
destinations_str = ", ".join(destinations) if destinations else "None configured"
title = f"File Processed: {filename}"
message = f"""
@@ -213,8 +202,5 @@ The file has been successfully processed and is being uploaded to all configured
"""
return send_notification(
title=title,
message=message.strip(),
notification_type="success",
tags=["document", "processed", "success"]
title=title, message=message.strip(), notification_type="success", tags=["document", "processed", "success"]
)
app/utils/oauth_helper.py
+2 -1
View File
@@ -4,7 +4,8 @@ Shared across multiple OAuth providers to reduce code duplication.
"""
import logging
from typing import Dict, Any, Optional
from typing import Any, Dict, Optional
import requests
from fastapi import HTTPException, status
app/utils/settings_service.py
+15 -27
View File
@@ -9,8 +9,9 @@ This module provides functionality to:
import logging
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.orm import Session
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import Session
from app.models import ApplicationSettings
@@ -67,7 +68,6 @@ SETTING_METADATA = {
"required": True,
"restart_required": True,
},
# Authentication Settings
"auth_enabled": {
"category": "Authentication",
@@ -133,7 +133,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": True,
},
# AI Services
"openai_api_key": {
"category": "AI Services",
@@ -183,7 +182,6 @@ SETTING_METADATA = {
"required": True,
"restart_required": False,
},
# Storage Providers - Dropbox
"dropbox_app_key": {
"category": "Storage Providers",
@@ -217,7 +215,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - Nextcloud
"nextcloud_upload_url": {
"category": "Storage Providers",
@@ -251,7 +248,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - Paperless-ngx
"paperless_ngx_api_token": {
"category": "Storage Providers",
@@ -269,7 +265,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - Google Drive
"google_drive_credentials_json": {
"category": "Storage Providers",
@@ -327,7 +322,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - OneDrive
"onedrive_client_id": {
"category": "Storage Providers",
@@ -369,7 +363,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - WebDAV
"webdav_url": {
"category": "Storage Providers",
@@ -411,7 +404,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - FTP
"ftp_host": {
"category": "Storage Providers",
@@ -469,7 +461,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - SFTP
"sftp_host": {
"category": "Storage Providers",
@@ -535,7 +526,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Storage Providers - AWS S3
"aws_access_key_id": {
"category": "Storage Providers",
@@ -593,7 +583,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Email Settings
"email_host": {
"category": "Email",
@@ -651,7 +640,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# IMAP Settings - Account 1
"imap1_host": {
"category": "IMAP",
@@ -709,7 +697,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# IMAP Settings - Account 2
"imap2_host": {
"category": "IMAP",
@@ -767,7 +754,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Monitoring - Uptime Kuma
"uptime_kuma_url": {
"category": "Monitoring",
@@ -785,7 +771,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Processing Settings
"http_request_timeout": {
"category": "Processing",
@@ -811,7 +796,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Notifications Settings
"notification_urls": {
"category": "Notifications",
@@ -861,7 +845,6 @@ SETTING_METADATA = {
"required": False,
"restart_required": False,
},
# Feature Flags
"allow_file_delete": {
"category": "Feature Flags",
@@ -896,6 +879,7 @@ def get_setting_from_db(db: Session, key: str) -> Optional[str]:
metadata = get_setting_metadata(key)
if metadata.get("sensitive", False):
from app.utils.encryption import decrypt_value
return decrypt_value(setting.value)
return setting.value
@@ -968,6 +952,7 @@ def get_all_settings_from_db(db: Session) -> Dict[str, str]:
metadata = get_setting_metadata(setting.key)
if metadata.get("sensitive", False):
from app.utils.encryption import decrypt_value
result[setting.key] = decrypt_value(setting.value)
else:
result[setting.key] = setting.value
@@ -1013,14 +998,17 @@ def get_setting_metadata(key: str) -> Dict[str, Any]:
Returns:
Dictionary containing setting metadata
"""
return SETTING_METADATA.get(key, {
"category": "Other",
"description": f"Setting: {key}",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
})
return SETTING_METADATA.get(
key,
{
"category": "Other",
"description": f"Setting: {key}",
"type": "string",
"sensitive": False,
"required": False,
"restart_required": False,
},
)
def get_settings_by_category() -> Dict[str, List[str]]:
app/utils/setup_wizard.py
+16 -14
View File
@@ -5,7 +5,8 @@ Detects if the system needs initial setup and provides required settings list.
"""
import logging
from typing import List, Dict, Any
from typing import Any, Dict, List
from app.config import settings
logger = logging.getLogger(__name__)
@@ -27,7 +28,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": "sqlite:///./app/database.db",
"wizard_step": 1,
"wizard_category": "Core Infrastructure"
"wizard_category": "Core Infrastructure",
},
{
"key": "redis_url",
@@ -37,7 +38,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": "redis://localhost:6379/0",
"wizard_step": 1,
"wizard_category": "Core Infrastructure"
"wizard_category": "Core Infrastructure",
},
{
"key": "workdir",
@@ -47,7 +48,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": "/workdir",
"wizard_step": 1,
"wizard_category": "Core Infrastructure"
"wizard_category": "Core Infrastructure",
},
{
"key": "gotenberg_url",
@@ -57,7 +58,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": "http://gotenberg:3000",
"wizard_step": 1,
"wizard_category": "Core Infrastructure"
"wizard_category": "Core Infrastructure",
},
{
"key": "session_secret",
@@ -67,7 +68,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": True,
"default": None, # Should be generated
"wizard_step": 2,
"wizard_category": "Security"
"wizard_category": "Security",
},
{
"key": "admin_username",
@@ -77,7 +78,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": "admin",
"wizard_step": 2,
"wizard_category": "Security"
"wizard_category": "Security",
},
{
"key": "admin_password",
@@ -87,7 +88,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": True,
"default": None, # Must be set
"wizard_step": 2,
"wizard_category": "Security"
"wizard_category": "Security",
},
{
"key": "openai_api_key",
@@ -97,7 +98,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": True,
"default": None,
"wizard_step": 3,
"wizard_category": "AI Services"
"wizard_category": "AI Services",
},
{
"key": "azure_ai_key",
@@ -107,7 +108,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": True,
"default": None,
"wizard_step": 3,
"wizard_category": "AI Services"
"wizard_category": "AI Services",
},
{
"key": "azure_region",
@@ -117,7 +118,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": "eastus",
"wizard_step": 3,
"wizard_category": "AI Services"
"wizard_category": "AI Services",
},
{
"key": "azure_endpoint",
@@ -127,7 +128,7 @@ def get_required_settings() -> List[Dict[str, Any]]:
"sensitive": False,
"default": None,
"wizard_step": 3,
"wizard_category": "AI Services"
"wizard_category": "AI Services",
},
]
@@ -180,12 +181,13 @@ def get_missing_required_settings() -> List[str]:
# Check if value is missing or is a placeholder
placeholder_values = [
None, "",
None,
"",
f"<{key.upper()}>",
"test-key",
"your_secure_password",
"changeme",
"INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS"
"INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS",
]
if value in placeholder_values:
app/views/__init__.py
+5 -3
View File
@@ -1,16 +1,18 @@
"""
Aggregated view routers for the application.
"""
from fastapi import APIRouter
from app.views.dropbox import router as dropbox_router
# Import all the view routers
from app.views.general import router as general_router
from app.views.status import router as status_router
from app.views.onedrive import router as onedrive_router
from app.views.dropbox import router as dropbox_router
from app.views.google_drive import router as google_drive_router
from app.views.license_routes import router as license_router # Add the license router
from app.views.onedrive import router as onedrive_router
from app.views.settings import router as settings_router
from app.views.status import router as status_router
from app.views.wizard import router as wizard_router
# Create a main router that includes all the view routers
app/views/base.py
+12 -7
View File
@@ -1,15 +1,17 @@
"""
Base setup for views, containing shared functionality and imports.
"""
from fastapi import APIRouter, Request, Depends, HTTPException
from fastapi.templating import Jinja2Templates
from pathlib import Path
from sqlalchemy.orm import Session
import logging
from app.auth import require_login
from app.database import SessionLocal
import logging
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException, Request # noqa: F401
from fastapi.templating import Jinja2Templates
from sqlalchemy.orm import Session # noqa: F401
from app.auth import require_login # noqa: F401
from app.config import settings
from app.database import SessionLocal
# Set up Jinja2 templates
templates_dir = Path(__file__).parent.parent.parent / "frontend" / "templates"
@@ -22,6 +24,7 @@ templates.env.globals["max"] = max
# Customize Jinja2Templates to include app_version in all templates
original_template_response = templates.TemplateResponse
def template_response_with_version(*args, **kwargs):
"""Wrapper for TemplateResponse to include version in all templates"""
# If context dict is provided, add version to it
@@ -31,11 +34,13 @@ def template_response_with_version(*args, **kwargs):
kwargs["context"].setdefault("version", settings.version)
return original_template_response(*args, **kwargs)
templates.TemplateResponse = template_response_with_version
# Set up logging
logger = logging.getLogger(__name__)
def get_db():
"""
Dependency to get a database session.
app/views/dropbox.py
+11 -14
View File
@@ -1,12 +1,14 @@
"""
Dropbox integration views for setup and OAuth callback.
"""
from fastapi import Request
from app.views.base import APIRouter, templates, require_login, settings
from app.views.base import APIRouter, require_login, settings, templates
router = APIRouter()
@router.get("/dropbox-setup")
@require_login
async def dropbox_setup_page(request: Request):
@@ -15,9 +17,7 @@ async def dropbox_setup_page(request: Request):
Shows configuration status and setup instructions.
"""
# Check Dropbox configuration
is_configured = bool(settings.dropbox_app_key and
settings.dropbox_app_secret and
settings.dropbox_refresh_token)
is_configured = bool(settings.dropbox_app_key and settings.dropbox_app_secret and settings.dropbox_refresh_token)
return templates.TemplateResponse(
"dropbox.html",
@@ -27,10 +27,11 @@ async def dropbox_setup_page(request: Request):
"app_key_value": settings.dropbox_app_key or "",
"app_secret_value": settings.dropbox_app_secret if settings.dropbox_app_secret else "",
"refresh_token_value": settings.dropbox_refresh_token if settings.dropbox_refresh_token else "",
"folder_path": settings.dropbox_folder or "/Documents/Uploads" # Default folder path
}
"folder_path": settings.dropbox_folder or "/Documents/Uploads", # Default folder path
},
)
@router.get("/dropbox-callback")
@require_login
async def dropbox_callback(request: Request, code: str = None, error: str = None):
@@ -39,15 +40,11 @@ async def dropbox_callback(request: Request, code: str = None, error: str = None
Automatically exchanges the code for a token and saves it to the configuration.
"""
if error:
return templates.TemplateResponse(
"dropbox_callback_error.html",
{"request": request, "error": error}
)
return templates.TemplateResponse("dropbox_callback_error.html", {"request": request, "error": error})
if not code:
return templates.TemplateResponse(
"dropbox_callback_error.html",
{"request": request, "error": "No authorization code received from Dropbox"}
"dropbox_callback_error.html", {"request": request, "error": "No authorization code received from Dropbox"}
)
# Display the processing page with automatic token exchange
@@ -60,6 +57,6 @@ async def dropbox_callback(request: Request, code: str = None, error: str = None
"code": code,
"app_key_value": "", # The callback will prioritize sessionStorage values
"app_secret_value": "", # The callback will prioritize sessionStorage values
"folder_path": "" # The callback will prioritize sessionStorage values
}
"folder_path": "", # The callback will prioritize sessionStorage values
},
)
app/views/files.py
+9 -6
View File
@@ -2,13 +2,14 @@
File management views for displaying and managing files.
"""
from fastapi import Request, Depends, Query
from sqlalchemy.orm import Session
from typing import Optional
from app.views.base import APIRouter, templates, require_login, get_db, logger
from app.utils.file_status import get_files_processing_status
from fastapi import Depends, Query, Request
from sqlalchemy.orm import Session
from app.config import settings
from app.utils.file_status import get_files_processing_status
from app.views.base import APIRouter, get_db, logger, require_login, templates
router = APIRouter()
@@ -31,8 +32,9 @@ def files_page(
"""
try:
# Import the model here to avoid circular imports
from sqlalchemy import asc, desc, or_
from app.models import FileRecord, ProcessingLog
from sqlalchemy import desc, asc, or_
# Start with base query
query = db.query(FileRecord)
@@ -159,9 +161,10 @@ def file_detail_page(request: Request, file_id: int, db: Session = Depends(get_d
Return the file detail page showing processing history and file information
"""
try:
from app.models import FileRecord, ProcessingLog
import os
from app.models import FileRecord, ProcessingLog
# Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
app/views/general.py
+26 -18
View File
@@ -1,17 +1,20 @@
"""
General routes for the application homepage and basic pages.
"""
from fastapi import Request, HTTPException, Depends
from fastapi.responses import FileResponse, RedirectResponse
from pathlib import Path
from datetime import date
from pathlib import Path
from fastapi import Depends, HTTPException, Request
from fastapi.responses import FileResponse, RedirectResponse
from sqlalchemy.orm import Session
from app.views.base import APIRouter, templates, require_login, get_db, logger
from app.utils.config_validator import get_provider_status, validate_storage_configs
from app.views.base import APIRouter, get_db, logger, require_login, templates
router = APIRouter()
@router.get("/", include_in_schema=False)
async def serve_index(request: Request, db: Session = Depends(get_db)):
"""
@@ -20,8 +23,8 @@ async def serve_index(request: Request, db: Session = Depends(get_db)):
If the system requires initial setup, redirect to the setup wizard.
"""
# Check if setup wizard is needed
from app.utils.setup_wizard import is_setup_required
from app.utils.settings_service import get_setting_from_db
from app.utils.setup_wizard import is_setup_required
# Check if setup was explicitly skipped
setup_skipped = get_setting_from_db(db, "_setup_wizard_skipped")
@@ -37,20 +40,23 @@ async def serve_index(request: Request, db: Session = Depends(get_db)):
providers = get_provider_status()
# Count configured providers
configured_providers = sum(1 for provider in providers.values() if provider['configured'])
configured_providers = sum(1 for provider in providers.values() if provider["configured"])
# Count different types of storage targets
storage_issues = validate_storage_configs()
configured_storage_targets = sum(1 for provider, issues in storage_issues.items()
if not issues and provider in ['dropbox', 'nextcloud', 'sftp',
's3', 'ftp', 'webdav',
'google_drive', 'onedrive'])
configured_storage_targets = sum(
1
for provider, issues in storage_issues.items()
if not issues
and provider in ["dropbox", "nextcloud", "sftp", "s3", "ftp", "webdav", "google_drive", "onedrive"]
)
# Query the actual file count from the database
processed_files = 0
try:
# Import the model here to avoid circular imports
from app.models import FileRecord
processed_files = db.query(FileRecord).count()
except Exception as e:
# Log error but continue (don't break the page if DB query fails)
@@ -60,16 +66,18 @@ async def serve_index(request: Request, db: Session = Depends(get_db)):
stats = {
"processed_files": processed_files,
"active_integrations": configured_providers,
"storage_targets": configured_storage_targets
"storage_targets": configured_storage_targets,
}
return templates.TemplateResponse("index.html", {"request": request, "stats": stats})
@router.get("/about", include_in_schema=False)
async def serve_about(request: Request):
"""Serve the about page."""
return templates.TemplateResponse("about.html", {"request": request})
@router.get("/privacy", include_in_schema=False)
async def serve_privacy(request: Request):
"""Serve the privacy policy page."""
@@ -77,17 +85,20 @@ async def serve_privacy(request: Request):
current_date = date.today().strftime("%B %d, %Y")
return templates.TemplateResponse("privacy.html", {"request": request, "current_date": current_date})
@router.get("/imprint", include_in_schema=False)
async def serve_imprint(request: Request):
"""Serve the imprint/impressum page."""
return templates.TemplateResponse("imprint.html", {"request": request})
@router.get("/upload", include_in_schema=False)
@require_login
async def serve_upload(request: Request):
"""Serve the upload page."""
return templates.TemplateResponse("upload.html", {"request": request})
@router.get("/favicon.ico", include_in_schema=False)
def favicon():
"""Serve the favicon."""
@@ -97,6 +108,7 @@ def favicon():
raise HTTPException(status_code=404, detail="Favicon not found")
return FileResponse(favicon_path)
@router.get("/license", include_in_schema=False)
async def serve_license(request: Request):
"""Serve the license page."""
@@ -130,13 +142,8 @@ The full license text could not be located on this system.
Please visit http://www.apache.org/licenses/LICENSE-2.0 for the complete license text.
"""
return templates.TemplateResponse(
"license.html",
{
"request": request,
"license_text": license_text
}
)
return templates.TemplateResponse("license.html", {"request": request, "license_text": license_text})
@router.get("/cookies", include_in_schema=False)
async def serve_cookies(request: Request):
@@ -144,6 +151,7 @@ async def serve_cookies(request: Request):
current_date = date.today().strftime("%B %d, %Y")
return templates.TemplateResponse("cookies.html", {"request": request, "current_date": current_date})
@router.get("/terms", include_in_schema=False)
async def serve_terms(request: Request):
"""Serve the terms of service page."""
app/views/google_drive.py
+20 -31
View File
@@ -1,14 +1,17 @@
"""
Google Drive integration views for setup and OAuth callback.
"""
from fastapi import Request
from fastapi.responses import RedirectResponse
import urllib.parse
from app.views.base import APIRouter, templates, require_login, settings
from fastapi import Request
from fastapi.responses import RedirectResponse
from app.views.base import APIRouter, require_login, settings, templates
router = APIRouter()
@router.get("/google-drive-setup")
@require_login
async def google_drive_setup_page(request: Request):
@@ -17,12 +20,12 @@ async def google_drive_setup_page(request: Request):
Shows configuration status and setup instructions.
"""
# Check if using OAuth
use_oauth = getattr(settings, 'google_drive_use_oauth', False)
use_oauth = getattr(settings, "google_drive_use_oauth", False)
# Check Google Drive OAuth configuration
oauth_configured = bool(settings.google_drive_client_id and
settings.google_drive_client_secret and
settings.google_drive_refresh_token)
oauth_configured = bool(
settings.google_drive_client_id and settings.google_drive_client_secret and settings.google_drive_refresh_token
)
# Check Google Drive service account configuration
sa_configured = bool(settings.google_drive_credentials_json)
@@ -51,10 +54,11 @@ async def google_drive_setup_page(request: Request):
"refresh_token": bool(settings.google_drive_refresh_token),
"refresh_token_value": settings.google_drive_refresh_token or "",
"folder_id": settings.google_drive_folder_id or "",
"has_credentials_json": bool(settings.google_drive_credentials_json)
}
"has_credentials_json": bool(settings.google_drive_credentials_json),
},
)
@router.get("/google-drive-callback")
@require_login
async def google_drive_callback(request: Request, code: str = None, error: str = None, state: str = None):
@@ -63,34 +67,21 @@ async def google_drive_callback(request: Request, code: str = None, error: str =
Now automatically exchanges the code for a token and saves it to the configuration.
"""
if error:
return templates.TemplateResponse(
"google_drive_callback_error.html",
{"request": request, "error": error}
)
return templates.TemplateResponse("google_drive_callback_error.html", {"request": request, "error": error})
if not code:
return templates.TemplateResponse(
"google_drive_callback_error.html",
{"request": request, "error": "No authorization code received from Google"}
{"request": request, "error": "No authorization code received from Google"},
)
# Display the processing page with automatic token exchange
return templates.TemplateResponse(
"google_drive_callback.html",
{
"request": request,
"code": code,
"state": state
}
)
return templates.TemplateResponse("google_drive_callback.html", {"request": request, "code": code, "state": state})
@router.get("/google-drive-auth-start")
@require_login
async def google_drive_auth_start(
request: Request,
client_id: str,
redirect_uri: str = None
):
async def google_drive_auth_start(request: Request, client_id: str, redirect_uri: str = None):
"""
Start the Google Drive OAuth flow by redirecting to Google's authorization page.
"""
@@ -99,11 +90,9 @@ async def google_drive_auth_start(
# Create the authorization URL with required scopes
# Use only drive.file scope to minimize required permissions
scopes = [
"https://www.googleapis.com/auth/drive.file" # Access to files created or opened by the app
]
scopes = ["https://www.googleapis.com/auth/drive.file"] # Access to files created or opened by the app
scope_str = urllib.parse.quote(' '.join(scopes))
scope_str = urllib.parse.quote(" ".join(scopes))
auth_url = (
f"https://accounts.google.com/o/oauth2/auth"
app/views/license_routes.py
+6 -4
View File
@@ -1,12 +1,13 @@
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import PlainTextResponse, HTMLResponse
from pathlib import Path
import os
from app.views.base import templates, require_login
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import HTMLResponse, PlainTextResponse
from app.views.base import templates
router = APIRouter()
@router.get("/licenses/lgpl.txt", response_class=PlainTextResponse)
async def get_lgpl_license():
"""
@@ -19,6 +20,7 @@ async def get_lgpl_license():
with open(license_path, "r") as f:
return f.read()
@router.get("/attribution", response_class=HTMLResponse, include_in_schema=False)
async def serve_attribution(request: Request):
"""
app/views/onedrive.py
+13 -13
View File
@@ -1,12 +1,14 @@
"""
OneDrive integration views for setup and OAuth callback.
"""
from fastapi import Request
from app.views.base import APIRouter, templates, require_login, settings
from app.views.base import APIRouter, require_login, settings, templates
router = APIRouter()
@router.get("/onedrive-setup")
@require_login
async def onedrive_setup_page(request: Request):
@@ -15,9 +17,9 @@ async def onedrive_setup_page(request: Request):
Shows configuration status and setup instructions.
"""
# Check OneDrive configuration
is_configured = bool(settings.onedrive_client_id and
settings.onedrive_client_secret and
settings.onedrive_refresh_token)
is_configured = bool(
settings.onedrive_client_id and settings.onedrive_client_secret and settings.onedrive_refresh_token
)
# Get configuration values to display status (hide sensitive values)
return templates.TemplateResponse(
@@ -32,10 +34,11 @@ async def onedrive_setup_page(request: Request):
"tenant_id": settings.onedrive_tenant_id,
"refresh_token": bool(settings.onedrive_refresh_token),
"refresh_token_value": settings.onedrive_refresh_token if settings.onedrive_refresh_token else "",
"folder_path": settings.onedrive_folder_path or "Documents/Uploads" # Default folder path
}
"folder_path": settings.onedrive_folder_path or "Documents/Uploads", # Default folder path
},
)
@router.get("/onedrive-callback")
@require_login
async def onedrive_callback(request: Request, code: str = None, error: str = None):
@@ -44,15 +47,12 @@ async def onedrive_callback(request: Request, code: str = None, error: str = Non
Now automatically exchanges the code for a token and saves it to the configuration.
"""
if error:
return templates.TemplateResponse(
"onedrive_callback_error.html",
{"request": request, "error": error}
)
return templates.TemplateResponse("onedrive_callback_error.html", {"request": request, "error": error})
if not code:
return templates.TemplateResponse(
"onedrive_callback_error.html",
{"request": request, "error": "No authorization code received from Microsoft"}
{"request": request, "error": "No authorization code received from Microsoft"},
)
# Display the processing page with automatic token exchange
@@ -63,6 +63,6 @@ async def onedrive_callback(request: Request, code: str = None, error: str = Non
"code": code,
"client_id_value": settings.onedrive_client_id or "",
"client_secret_value": settings.onedrive_client_secret or "",
"tenant_id": settings.onedrive_tenant_id or "common"
}
"tenant_id": settings.onedrive_tenant_id or "common",
},
)
app/views/settings.py
+22 -24
View File
@@ -2,17 +2,18 @@
Settings management views for the application.
"""
import os
import logging
import inspect
import logging
import os
from functools import wraps
from fastapi import Request, Depends, HTTPException, status
from fastapi import Depends, HTTPException, Request, status
from fastapi.responses import RedirectResponse
from sqlalchemy.orm import Session
from app.views.base import APIRouter, templates, require_login, settings, get_db
from app.utils.settings_service import get_settings_by_category, get_setting_metadata, SETTING_METADATA
from app.utils.config_validator.masking import mask_sensitive_value
from app.utils.settings_service import get_setting_metadata, get_settings_by_category
from app.views.base import APIRouter, get_db, require_login, settings, templates
logger = logging.getLogger(__name__)
router = APIRouter()
@@ -26,11 +27,12 @@ def require_admin_access(func):
If not, redirects to the home page. Works with both sync and async functions,
though FastAPI route handlers should always be async.
"""
@wraps(func)
async def wrapper(request: Request, *args, **kwargs):
user = request.session.get("user")
if not user or not user.get("is_admin"):
logger.warning(f"Non-admin user attempted to access admin-only route")
logger.warning("Non-admin user attempted to access admin-only route")
return RedirectResponse(url="/", status_code=status.HTTP_302_FOUND)
# FastAPI route handlers are async, but we support sync for flexibility
@@ -38,6 +40,7 @@ def require_admin_access(func):
return await func(request, *args, **kwargs)
else:
return func(request, *args, **kwargs)
return wrapper
@@ -55,6 +58,7 @@ async def settings_page(request: Request, db: Session = Depends(get_db)):
try:
# Get settings from database
from app.utils.settings_service import get_all_settings_from_db
db_settings = get_all_settings_from_db(db)
# Get settings organized by category
@@ -93,26 +97,20 @@ async def settings_page(request: Request, db: Session = Depends(get_db)):
if metadata.get("sensitive") and value:
display_value = mask_sensitive_value(value)
settings_data[category].append({
"key": key,
"display_value": display_value if display_value is not None else "",
"metadata": metadata,
"source": source,
"source_label": source_label,
"source_color": source_color
})
settings_data[category].append(
{
"key": key,
"display_value": display_value if display_value is not None else "",
"metadata": metadata,
"source": source,
"source_label": source_label,
"source_color": source_color,
}
)
return templates.TemplateResponse(
"settings.html",
{
"request": request,
"settings_data": settings_data,
"app_version": settings.version
}
"settings.html", {"request": request, "settings_data": settings_data, "app_version": settings.version}
)
except Exception as e:
logger.error(f"Error loading settings page: {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to load settings page"
)
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to load settings page")
app/views/status.py
+31 -29
View File
@@ -1,17 +1,19 @@
"""
Status and configuration views for the application.
"""
from fastapi import Request
from fastapi.responses import JSONResponse
from datetime import datetime
import os
import logging
from app.views.base import APIRouter, templates, require_login, settings
import logging
import os
from datetime import datetime
from fastapi import Request
from app.views.base import APIRouter, require_login, settings, templates
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/status")
@require_login
async def status_dashboard(request: Request):
@@ -24,53 +26,53 @@ async def status_dashboard(request: Request):
providers = get_provider_status()
# Get build date from settings
build_date = getattr(settings, 'build_date', 'Unknown')
build_date = getattr(settings, "build_date", "Unknown")
# Try to get container information
container_info = {}
try:
# Check for Docker environment
if os.path.exists('/.dockerenv'):
if os.path.exists("/.dockerenv"):
# We're inside a Docker container
container_info['is_docker'] = True
container_info["is_docker"] = True
# Try to get container ID
try:
with open('/proc/self/cgroup', 'r') as f:
with open("/proc/self/cgroup", "r") as f:
for line in f:
if 'docker' in line:
container_id = line.split('/')[-1].strip()
container_info['id'] = container_id[:12] # Short ID format
if "docker" in line:
container_id = line.split("/")[-1].strip()
container_info["id"] = container_id[:12] # Short ID format
break
except Exception:
container_info['id'] = 'Unknown'
container_info["id"] = "Unknown"
# Get Git commit SHA from settings
try:
git_sha = settings.git_sha
container_info['git_sha'] = git_sha[:7] if git_sha and git_sha != 'unknown' else 'Unknown'
container_info["git_sha"] = git_sha[:7] if git_sha and git_sha != "unknown" else "Unknown"
except Exception:
container_info['git_sha'] = 'Unknown'
container_info["git_sha"] = "Unknown"
# Try to get runtime information
try:
container_info['runtime_info'] = settings.runtime_info
container_info["runtime_info"] = settings.runtime_info
except Exception:
pass
else:
container_info['is_docker'] = False
container_info["is_docker"] = False
# If not in Docker, get Git info from settings
try:
git_sha = settings.git_sha
container_info['git_sha'] = git_sha[:7] if git_sha and git_sha != 'unknown' else 'Unknown'
container_info["git_sha"] = git_sha[:7] if git_sha and git_sha != "unknown" else "Unknown"
except Exception:
container_info['git_sha'] = 'Unknown'
container_info["git_sha"] = "Unknown"
except Exception:
container_info = {'is_docker': False, 'id': 'Unknown', 'git_sha': 'Unknown'}
container_info = {"is_docker": False, "id": "Unknown", "git_sha": "Unknown"}
# Get notification URLs for the notification box
notification_urls = getattr(settings, 'notification_urls', [])
notification_urls = getattr(settings, "notification_urls", [])
return templates.TemplateResponse(
"status_dashboard.html",
@@ -79,15 +81,14 @@ async def status_dashboard(request: Request):
"providers": providers,
"app_version": settings.version,
"build_date": build_date,
"debug_enabled": getattr(settings, 'debug', False),
"debug_enabled": getattr(settings, "debug", False),
"last_check": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"container_info": container_info,
"settings": {
"notification_urls": notification_urls
}
}
"settings": {"notification_urls": notification_urls},
},
)
@router.get("/env")
@require_login
async def env_debug(request: Request):
@@ -100,6 +101,7 @@ async def env_debug(request: Request):
# Get settings data
from app.utils.config_validator import get_settings_for_display
settings_data = get_settings_for_display(show_values=debug_enabled)
return templates.TemplateResponse(
@@ -108,6 +110,6 @@ async def env_debug(request: Request):
"request": request,
"settings": settings_data,
"debug_enabled": debug_enabled,
"app_version": settings.version
}
"app_version": settings.version,
},
)
app/views/wizard.py
+7 -16
View File
@@ -2,21 +2,16 @@
Setup wizard views for initial system configuration.
"""
import os
import logging
import secrets
from fastapi import Request, Depends, Form
from fastapi import Depends, Form, Request
from fastapi.responses import RedirectResponse
from sqlalchemy.orm import Session
from app.views.base import APIRouter, templates, get_db
from app.utils.setup_wizard import (
is_setup_required,
get_required_settings,
get_wizard_steps,
get_missing_required_settings
)
from app.utils.settings_service import save_setting_to_db
from app.utils.setup_wizard import get_wizard_steps
from app.views.base import APIRouter, get_db, templates
logger = logging.getLogger(__name__)
router = APIRouter()
@@ -54,17 +49,13 @@ async def setup_wizard(request: Request, step: int = 1):
"max_step": max_step,
"settings": current_settings,
"step_category": step_category,
"progress_percent": int((step / max_step) * 100)
}
"progress_percent": int((step / max_step) * 100),
},
)
@router.post("/setup")
async def setup_wizard_save(
request: Request,
step: int = Form(...),
db: Session = Depends(get_db)
):
async def setup_wizard_save(request: Request, step: int = Form(...), db: Session = Depends(get_db)):
"""
Save settings from the current wizard step.
"""