gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enhance FTP configuration with TLS options and update documentation

Browse Source 
Enhanced OneDrive workflow and tested
This commit is contained in:
Christian Krakau-Louis
2025-04-02 02:57:30 +02:00
parent 81296b93f8
commit c3df8107d8
15 changed files with 1242 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api.py
+370 -3
View File
@@ -1,9 +1,11 @@
# app/api.py
from fastapi import APIRouter, Request, HTTPException, status, Depends, UploadFile, File
from fastapi import APIRouter, Request, HTTPException, status, Depends, UploadFile, File, Form
from hashlib import md5
from sqlalchemy.orm import Session
from typing import List
import os
import requests
import logging
from app.auth import require_login, get_current_user
from app.database import SessionLocal
@@ -14,8 +16,12 @@ from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.send_to_all import send_to_all_destinations
# Set up logging
logger = logging.getLogger(__name__)
router = APIRouter()
def get_db():
@@ -186,6 +192,19 @@ def send_to_google_drive(file_path: str):
task = upload_to_google_drive.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_onedrive/")
@require_login
def send_to_onedrive_endpoint(file_path: str):
"""Send a document to OneDrive."""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = upload_to_onedrive.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_all_destinations/")
@require_login
def send_to_all_destinations_endpoint(file_path: str):
@@ -276,5 +295,353 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
"stored_filename": target_filename
}
# Note: The api/router.py is now a submodule organization,
# but we're keeping this file for compatibility until we've fully migrated
@router.post("/onedrive/exchange-token")
@require_login
async def exchange_onedrive_token(
request: Request,
client_id: str = Form(...),
client_secret: str = Form(...),
redirect_uri: str = Form(...),
code: str = Form(...),
tenant_id: str = Form(...)
):
"""
Exchange an authorization code for a refresh token.
This is done on the server to avoid exposing client secret in the browser.
"""
try:
logger.info(f"Starting OneDrive token exchange process with tenant_id: {tenant_id}")
# Prepare the token request
token_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"
logger.info(f"Using token URL: {token_url}")
payload = {
'client_id': client_id,
'scope': 'https://graph.microsoft.com/.default offline_access',
'code': code,
'redirect_uri': redirect_uri,
'grant_type': 'authorization_code',
'client_secret': client_secret
}
# Log request details (excluding secret)
safe_payload = payload.copy()
safe_payload['client_secret'] = '[REDACTED]'
safe_payload['code'] = f"{code[:5]}...{code[-5:]}" if len(code) > 10 else '[REDACTED]'
logger.info(f"Token exchange request payload: {safe_payload}")
# Make the token request
logger.info("Sending POST request to Microsoft for token exchange")
response = requests.post(token_url, data=payload)
# Check if the request was successful
logger.info(f"Token exchange response status: {response.status_code}")
if response.status_code != 200:
# Log the error response for debugging
try:
error_json = response.json()
logger.error(f"Token exchange failed with status {response.status_code}: {error_json}")
error_detail = error_json
except Exception as json_err:
logger.error(f"Failed to parse error response as JSON: {str(json_err)}")
logger.error(f"Raw response content: {response.content[:500]}") # Limit log size
error_detail = {"error": "Unknown error", "raw_content_snippet": str(response.content[:100])}
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Token exchange failed: {error_detail}"
)
# Return the token response
token_data = response.json()
# Validate the token response
if "refresh_token" not in token_data:
logger.error(f"Microsoft returned success but no refresh token found in response: {token_data.keys()}")
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail="Microsoft OAuth server returned success but no refresh token was included"
)
# Calculate token length for logging
refresh_token_length = len(token_data.get("refresh_token", ""))
access_token_length = len(token_data.get("access_token", ""))
logger.info(f"Successfully exchanged authorization code for OneDrive tokens. "
f"Refresh token length: {refresh_token_length}, "
f"Access token length: {access_token_length}")
# Return just what's needed by the frontend
return {
"refresh_token": token_data["refresh_token"],
"expires_in": token_data.get("expires_in", 3600)
}
except HTTPException:
# Re-raise HTTP exceptions as they already have appropriate status codes
raise
except Exception as e:
logger.exception(f"Unexpected error during OneDrive token exchange: {str(e)}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to exchange token: {str(e)}"
)
@router.get("/onedrive/test-token")
@require_login
async def test_onedrive_token(request: Request):
"""
Test if the configured OneDrive refresh token is valid.
Provides detailed error information if token is invalid.
"""
try:
from app.tasks.upload_to_onedrive import get_onedrive_token
logger.info("Testing OneDrive token validity")
if not settings.onedrive_refresh_token:
logger.warning("No OneDrive refresh token configured")
return {
"status": "error",
"message": "No OneDrive refresh token is configured"
}
# Check if client ID and client secret are configured
if not settings.onedrive_client_id or not settings.onedrive_client_secret:
logger.warning("OneDrive client ID or client secret is missing")
return {
"status": "error",
"message": "OneDrive client ID or client secret is missing",
"missing_config": True
}
# Try to get an access token using the configured refresh token
try:
access_token = get_onedrive_token()
# If we got here, token is valid
logger.info("OneDrive token is valid")
return {
"status": "success",
"message": "OneDrive token is valid",
}
except Exception as e:
error_msg = str(e)
logger.error(f"OneDrive token test failed: {error_msg}")
# Determine if this is an invalid_grant error (expired token)
is_expired = "invalid_grant" in error_msg.lower()
return {
"status": "error",
"message": f"Token validation failed: {error_msg}",
"is_expired": is_expired,
"needs_reauth": True
}
except Exception as e:
logger.exception("Unexpected error testing OneDrive token")
return {
"status": "error",
"message": f"Unexpected error: {str(e)}"
}
@router.post("/onedrive/save-settings")
@require_login
async def save_onedrive_settings(
request: Request,
client_id: str = Form(None),
client_secret: str = Form(None),
refresh_token: str = Form(...),
tenant_id: str = Form("common"),
folder_path: str = Form(None)
):
"""
Save OneDrive settings to the .env file
"""
try:
# Get the path to the .env file
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
if not os.path.exists(env_path):
logger.error(f".env file not found at {env_path}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Could not find .env file to update"
)
logger.info(f"Updating OneDrive settings in {env_path}")
# Read the current .env file
with open(env_path, "r") as f:
env_lines = f.readlines()
# Define settings to update
onedrive_settings = {
"ONEDRIVE_REFRESH_TOKEN": refresh_token,
}
# Only update these if provided
if client_id:
onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id
if client_secret:
onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret
if tenant_id:
onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id
if folder_path:
onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path
# Process each line and update or add settings
updated = set()
new_env_lines = []
for line in env_lines:
line = line.rstrip()
is_updated = False
for key, value in onedrive_settings.items():
if line.startswith(f"{key}=") or line.startswith(f"# {key}="):
if line.startswith("# "): # Uncomment if commented out
line = line[2:]
new_env_lines.append(f"{key}={value}")
updated.add(key)
is_updated = True
break
if not is_updated:
new_env_lines.append(line)
# Add any settings that weren't updated (they weren't in the file)
for key, value in onedrive_settings.items():
if key not in updated:
new_env_lines.append(f"{key}={value}")
# Write the updated .env file
with open(env_path, "w") as f:
f.write("\n".join(new_env_lines) + "\n")
# Update the settings in memory
if refresh_token:
settings.onedrive_refresh_token = refresh_token
if client_id:
settings.onedrive_client_id = client_id
if client_secret:
settings.onedrive_client_secret = client_secret
if tenant_id:
settings.onedrive_tenant_id = tenant_id
if folder_path:
settings.onedrive_folder_path = folder_path
logger.info("Successfully updated OneDrive settings")
return {
"status": "success",
"message": "OneDrive settings have been saved"
}
except HTTPException:
raise
except Exception as e:
logger.exception(f"Unexpected error saving OneDrive settings: {str(e)}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to save OneDrive settings: {str(e)}"
)
@router.post("/onedrive/update-settings")
@require_login
async def update_onedrive_settings(
request: Request,
client_id: str = Form(None),
client_secret: str = Form(None),
refresh_token: str = Form(...),
tenant_id: str = Form("common"),
folder_path: str = Form(None)
):
"""
Update OneDrive settings in memory (without modifying .env file)
"""
try:
logger.info("Updating OneDrive settings in memory")
# Update settings in memory
if refresh_token:
settings.onedrive_refresh_token = refresh_token
logger.info("Updated ONEDRIVE_REFRESH_TOKEN in memory")
if client_id:
settings.onedrive_client_id = client_id
logger.info("Updated ONEDRIVE_CLIENT_ID in memory")
if client_secret:
settings.onedrive_client_secret = client_secret
logger.info("Updated ONEDRIVE_CLIENT_SECRET in memory")
if tenant_id:
settings.onedrive_tenant_id = tenant_id
logger.info("Updated ONEDRIVE_TENANT_ID in memory")
if folder_path:
settings.onedrive_folder_path = folder_path
logger.info("Updated ONEDRIVE_FOLDER_PATH in memory")
# Test the token to make sure it works
try:
from app.tasks.upload_to_onedrive import get_onedrive_token
access_token = get_onedrive_token()
logger.info("Successfully tested OneDrive token")
except Exception as e:
logger.error(f"Token test failed after updating settings: {str(e)}")
return {
"status": "warning",
"message": "Settings updated but token test failed: " + str(e)
}
return {
"status": "success",
"message": "OneDrive settings have been updated in memory"
}
except Exception as e:
logger.exception(f"Unexpected error updating OneDrive settings: {str(e)}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to update OneDrive settings: {str(e)}"
)
@router.get("/onedrive/get-full-config")
@require_login
async def get_onedrive_full_config(request: Request):
"""
Get the full OneDrive configuration for sharing with worker nodes
"""
try:
# Create a configuration object with all OneDrive settings
config = {
"client_id": settings.onedrive_client_id or "",
"client_secret": settings.onedrive_client_secret or "",
"tenant_id": settings.onedrive_tenant_id or "common",
"refresh_token": settings.onedrive_refresh_token or "",
"folder_path": settings.onedrive_folder_path or "Documents/Uploads"
}
# Generate environment variable format
env_format = "\n".join([
f"ONEDRIVE_CLIENT_ID={config['client_id']}",
f"ONEDRIVE_CLIENT_SECRET={config['client_secret']}",
f"ONEDRIVE_TENANT_ID={config['tenant_id']}",
f"ONEDRIVE_REFRESH_TOKEN={config['refresh_token']}",
f"ONEDRIVE_FOLDER_PATH={config['folder_path']}"
])
return {
"status": "success",
"config": config,
"env_format": env_format
}
except Exception as e:
logger.exception("Error getting OneDrive configuration")
return {
"status": "error",
"message": str(e)
}
app/config.py
+2
View File
@@ -77,6 +77,8 @@ class Settings(BaseSettings):
ftp_username: Optional[str] = None
ftp_password: Optional[str] = None
ftp_folder: Optional[str] = None
ftp_use_tls: bool = True # Default to attempting TLS connection first
ftp_allow_plaintext: bool = True # Default to allowing plaintext fallback
# SFTP settings
sftp_host: Optional[str] = None
app/frontend.py
+64 -2
View File
@@ -4,6 +4,7 @@ from fastapi.templating import Jinja2Templates
from pathlib import Path
from sqlalchemy.orm import Session
from datetime import datetime
import logging
from app.auth import require_login
from app.database import SessionLocal
@@ -15,6 +16,9 @@ router = APIRouter()
templates_dir = Path(__file__).parent.parent / "frontend" / "templates"
templates = Jinja2Templates(directory=str(templates_dir))
# Set up logging
logger = logging.getLogger(__name__)
def get_db():
db = SessionLocal()
try:
@@ -95,5 +99,63 @@ async def env_debug(request: Request):
}
)
# Note: The frontend/router.py is now a submodule organization,
# but we're keeping this file for compatibility until we've fully migrated
@router.get("/onedrive-setup")
@require_login
async def onedrive_setup_page(request: Request):
"""
Setup page for the OneDrive integration.
Shows configuration status and setup instructions.
"""
# Check OneDrive configuration
is_configured = bool(settings.onedrive_client_id and
settings.onedrive_client_secret and
settings.onedrive_refresh_token)
# Get configuration values to display status (hide sensitive values)
return templates.TemplateResponse(
"onedrive.html",
{
"request": request,
"is_configured": is_configured,
"client_id": bool(settings.onedrive_client_id),
"client_id_value": settings.onedrive_client_id or "", # Pass the actual value for the form
"client_secret": bool(settings.onedrive_client_secret),
"client_secret_value": settings.onedrive_client_secret if settings.onedrive_client_secret else "",
"tenant_id": settings.onedrive_tenant_id,
"refresh_token": bool(settings.onedrive_refresh_token),
"refresh_token_value": settings.onedrive_refresh_token if settings.onedrive_refresh_token else "",
"folder_path": settings.onedrive_folder_path or "Documents/Uploads" # Default folder path
}
)
@router.get("/onedrive-callback")
@require_login
async def onedrive_callback(request: Request, code: str = None, error: str = None):
"""
Callback endpoint for OneDrive OAuth flow.
Now automatically exchanges the code for a token and saves it to the configuration.
"""
if error:
return templates.TemplateResponse(
"onedrive_callback_error.html",
{"request": request, "error": error}
)
if not code:
return templates.TemplateResponse(
"onedrive_callback_error.html",
{"request": request, "error": "No authorization code received from Microsoft"}
)
# Display the processing page with automatic token exchange
return templates.TemplateResponse(
"onedrive_callback.html",
{
"request": request,
"code": code,
"client_id_value": settings.onedrive_client_id or "",
"client_secret_value": settings.onedrive_client_secret or "",
"tenant_id": settings.onedrive_tenant_id or "common"
}
)
app/tasks/upload_to_ftp.py
+61 -12
View File
@@ -26,18 +26,66 @@ def upload_to_ftp(file_path: str):
raise ValueError(error_msg)
try:
# Connect to FTP server
ftp = ftplib.FTP()
ftp.connect(
host=settings.ftp_host,
port=settings.ftp_port or 21
)
# First attempt FTPS (FTP with TLS)
use_tls = getattr(settings, 'ftp_use_tls', True) # Default to try TLS
allow_plaintext = getattr(settings, 'ftp_allow_plaintext', True) # Default to allow plaintext fallback
# Login with credentials
ftp.login(
user=settings.ftp_username,
passwd=settings.ftp_password
)
if use_tls:
try:
logger.info(f"Attempting FTPS connection to {settings.ftp_host}")
ftp = ftplib.FTP_TLS()
ftp.connect(
host=settings.ftp_host,
port=settings.ftp_port or 21
)
# Login with credentials
ftp.login(
user=settings.ftp_username,
passwd=settings.ftp_password
)
# Enable data protection - encrypt the data channel
ftp.prot_p()
logger.info("Successfully established FTPS connection with TLS")
except Exception as e:
if not allow_plaintext:
error_msg = f"FTPS connection failed and plaintext FTP is forbidden: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
else:
logger.warning(f"FTPS connection failed, falling back to regular FTP: {str(e)}")
# Fall back to regular FTP
ftp = ftplib.FTP()
ftp.connect(
host=settings.ftp_host,
port=settings.ftp_port or 21
)
# Login with credentials
ftp.login(
user=settings.ftp_username,
passwd=settings.ftp_password
)
else:
# Check if plaintext is allowed when TLS is explicitly disabled
if not allow_plaintext:
error_msg = "Plaintext FTP is forbidden by configuration"
logger.error(error_msg)
raise Exception(error_msg)
# Directly use regular FTP if TLS is explicitly disabled
ftp = ftplib.FTP()
ftp.connect(
host=settings.ftp_host,
port=settings.ftp_port or 21
)
# Login with credentials
ftp.login(
user=settings.ftp_username,
passwd=settings.ftp_password
)
# Change to target directory if specified
if settings.ftp_folder:
@@ -80,7 +128,8 @@ def upload_to_ftp(file_path: str):
"status": "Completed",
"file": file_path,
"ftp_host": settings.ftp_host,
"ftp_path": f"{settings.ftp_folder}/{filename}" if settings.ftp_folder else filename
"ftp_path": f"{settings.ftp_folder}/{filename}" if settings.ftp_folder else filename,
"used_tls": isinstance(ftp, ftplib.FTP_TLS)
}
except Exception as e:
app/tasks/upload_to_onedrive.py
+35 -4
View File
@@ -21,21 +21,52 @@ def get_onedrive_token():
if not settings.onedrive_client_id or not settings.onedrive_client_secret:
raise ValueError("OneDrive client ID and client secret must be configured")
# Log more details about the configuration
tenant = settings.onedrive_tenant_id or "common"
logger.info(f"Using OneDrive tenant: {tenant}")
# Define scopes consistently
scopes = ["https://graph.microsoft.com/.default"]
# Use refresh token flow (works for both personal and org accounts)
if settings.onedrive_refresh_token:
# Use MSAL to get token from refresh token
app = msal.PublicClientApplication(settings.onedrive_client_id)
# Use MSAL's ConfidentialClientApplication instead of PublicClientApplication
app = msal.ConfidentialClientApplication(
client_id=settings.onedrive_client_id,
client_credential=settings.onedrive_client_secret,
authority=f"https://login.microsoftonline.com/{tenant}"
)
# Request new token using refresh token
logger.info("Attempting to acquire token using refresh token")
token_response = app.acquire_token_by_refresh_token(
refresh_token=settings.onedrive_refresh_token,
scopes=["https://graph.microsoft.com/Files.ReadWrite"]
scopes=scopes
)
if "access_token" not in token_response:
error = token_response.get("error", "")
error_desc = token_response.get("error_description", "Unknown error")
# Log more details about the error
logger.error(f"Failed to get access token using refresh token")
logger.error(f"Error code: {error}")
logger.error(f"Error description: {error_desc}")
if error == "invalid_grant":
logger.error("The refresh token appears to be expired or revoked")
logger.error("A new authorization flow is required to obtain a fresh token")
raise ValueError(f"Failed to get access token: {error} - {error_desc}")
# Check if we received a new refresh token and update it
if "refresh_token" in token_response:
new_refresh_token = token_response["refresh_token"]
logger.info("Received new refresh token from Microsoft")
# Update the refresh token in memory
settings.onedrive_refresh_token = new_refresh_token
logger.info("Updated refresh token in memory")
return token_response["access_token"]
@@ -50,7 +81,7 @@ def get_onedrive_token():
# Acquire token for application
token_response = app.acquire_token_for_client(
scopes=["https://graph.microsoft.com/.default"]
scopes=scopes
)
if "access_token" not in token_response:
app/utils/config_validator.py
+18 -1
View File
@@ -238,6 +238,21 @@ def get_provider_status():
"folder": getattr(settings, 'webdav_folder', 'Not set')
}
}
# Add FTP configuration to providers
providers["FTP Storage"] = {
"name": "FTP Storage",
"configured": bool(getattr(settings, 'ftp_host', None) and
getattr(settings, 'ftp_username', None) and
getattr(settings, 'ftp_password', None)),
"enabled": True,
"details": {
"host": getattr(settings, 'ftp_host', 'Not set'),
"folder": getattr(settings, 'ftp_folder', 'Not set'),
"tls": getattr(settings, 'ftp_use_tls', True),
"allow_plaintext": getattr(settings, 'ftp_allow_plaintext', True)
}
}
return providers
@@ -363,7 +378,9 @@ def get_settings_for_display(show_values=False):
"ftp_port",
"ftp_username",
"ftp_password",
"ftp_folder"
"ftp_folder",
"ftp_use_tls",
"ftp_allow_plaintext"
],
"S3/AWS": [
"aws_access_key_id",