From 2cafda11dd7cb3fb06bff6a139d54b1aa69566c0 Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Thu, 3 Apr 2025 15:55:34 +0200 Subject: [PATCH] feat: update version to 0.4.3-dev, add file deletion feature, and refactor API structure --- .env.demo | 1 + VERSION | 2 +- app/api.py | 46 +++ app/api/__init__.py | 27 ++ app/api/common.py | 36 ++ app/api/diagnostic.py | 48 +++ app/api/dropbox.py | 308 +++++++++++++++ app/api/files.py | 143 +++++++ app/api/onedrive.py | 364 ++++++++++++++++++ app/api/process.py | 140 +++++++ app/api/user.py | 33 ++ app/config.py | 3 + app/frontend.py | 8 +- app/main.py | 12 +- ...rocess_with_azure_document_intelligence.py | 27 ++ app/utils/config_validator.py | 5 +- frontend/templates/files.html | 132 ++++++- 17 files changed, 1318 insertions(+), 17 deletions(-) create mode 100644 app/api/__init__.py create mode 100644 app/api/common.py create mode 100644 app/api/diagnostic.py create mode 100644 app/api/dropbox.py create mode 100644 app/api/files.py create mode 100644 app/api/onedrive.py create mode 100644 app/api/process.py create mode 100644 app/api/user.py diff --git a/.env.demo b/.env.demo index b79dc7aa..8f423d21 100644 --- a/.env.demo +++ b/.env.demo @@ -4,6 +4,7 @@ DATABASE_URL=sqlite:///./app/database.db REDIS_URL=redis://redis:6379/0 EXTERNAL_HOSTNAME=docunova.example.com GOTENBERG_URL=http://gotenberg:3000 +ALLOW_FILE_DELETE=true # Allow deletion of file records # **Authentication** AUTH_ENABLED=true diff --git a/VERSION b/VERSION index 7532512c..8cf9df5e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.2-dev +0.4.3-dev diff --git a/app/api.py b/app/api.py index 7dc90c7f..20f37645 100644 --- a/app/api.py +++ b/app/api.py @@ -89,6 +89,52 @@ def list_files_api(request: Request, db: Session = Depends(get_db)): }) return result +@router.delete("/files/{file_id}") +@require_login +def delete_file_record(request: Request, file_id: int, db: Session = Depends(get_db)): + """ + Delete a file record from the database. + This only removes the database entry, not the actual file. + """ + # Check if file deletion is allowed + if not settings.allow_file_delete: + raise HTTPException( + status_code=403, + detail="File deletion is disabled in the configuration" + ) + + try: + # Find the file record + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + + if not file_record: + raise HTTPException( + status_code=404, + detail=f"File record with ID {file_id} not found" + ) + + # Log the deletion + logger.info(f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}") + + # Delete the record + db.delete(file_record) + db.commit() + + return { + "status": "success", + "message": f"File record {file_id} deleted successfully" + } + + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.exception(f"Error deleting file record {file_id}: {str(e)}") + raise HTTPException( + status_code=500, + detail=f"Error deleting file record: {str(e)}" + ) + # API endpoints @router.get("/diagnostic/settings") @require_login diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 00000000..a2871ca9 --- /dev/null +++ b/app/api/__init__.py @@ -0,0 +1,27 @@ +""" +API Router module that combines all API endpoints +""" +from fastapi import APIRouter +import logging + +# Import all the individual routers +from app.api.user import router as user_router +from app.api.files import router as files_router +from app.api.process import router as process_router +from app.api.diagnostic import router as diagnostic_router +from app.api.onedrive import router as onedrive_router +from app.api.dropbox import router as dropbox_router + +# Set up logging +logger = logging.getLogger(__name__) + +# Create the main router that includes all the others +router = APIRouter() + +# Include all the routers +router.include_router(user_router) +router.include_router(files_router) +router.include_router(process_router) +router.include_router(diagnostic_router) +router.include_router(onedrive_router) +router.include_router(dropbox_router) diff --git a/app/api/common.py b/app/api/common.py new file mode 100644 index 00000000..fc1e949f --- /dev/null +++ b/app/api/common.py @@ -0,0 +1,36 @@ +""" +Common utilities for API routes +""" +import logging +import os +from sqlalchemy.orm import Session +from fastapi import Depends + +from app.database import SessionLocal +from app.config import settings + +# Set up logging +logger = logging.getLogger(__name__) + +def get_db(): + """Database dependency injection for routes""" + db = SessionLocal() + try: + yield db + finally: + db.close() + +def resolve_file_path(file_path: str, subfolder: str = None) -> str: + """ + Resolves a file path to an absolute path. + If the path is not absolute, it will be joined with the workdir path. + Optionally, can include a subfolder like 'processed'. + + Returns the absolute file path. + """ + if not os.path.isabs(file_path): + if subfolder: + file_path = os.path.join(settings.workdir, subfolder, file_path) + else: + file_path = os.path.join(settings.workdir, file_path) + return file_path diff --git a/app/api/diagnostic.py b/app/api/diagnostic.py new file mode 100644 index 00000000..9691057e --- /dev/null +++ b/app/api/diagnostic.py @@ -0,0 +1,48 @@ +""" +Diagnostic API endpoints +""" +from fastapi import APIRouter, Request, Depends +import logging + +from app.auth import require_login, get_current_user +from app.config import settings + +# Set up logging +logger = logging.getLogger(__name__) + +router = APIRouter() + +@router.get("/diagnostic/settings") +@require_login +async def diagnostic_settings(request: Request, current_user: dict = Depends(get_current_user)): + """ + API endpoint to dump settings to the log and view basic config information + This endpoint doesn't expose sensitive information like passwords or tokens + """ + from app.utils.config_validator import dump_all_settings, get_settings_for_display + # Dump full settings to log for admin to see + dump_all_settings() + + # Return safe subset of settings for API response + safe_settings = { + "workdir": settings.workdir, + "external_hostname": settings.external_hostname, + "configured_services": { + "email": bool(getattr(settings, 'email_host', None)), + "s3": bool(getattr(settings, 's3_bucket_name', None)), + "dropbox": bool(getattr(settings, 'dropbox_refresh_token', None)), + "nextcloud": bool(getattr(settings, 'nextcloud_upload_url', None)), + "sftp": bool(getattr(settings, 'sftp_host', None)), + "paperless": bool(getattr(settings, 'paperless_host', None)), + "google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)), + "uptime_kuma": bool(getattr(settings, 'uptime_kuma_url', None)), + "auth": bool(getattr(settings, 'authentik_config_url', None)), + }, + "imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)), + } + + return { + "status": "success", + "settings": safe_settings, + "message": "Full settings have been dumped to application logs" + } diff --git a/app/api/dropbox.py b/app/api/dropbox.py new file mode 100644 index 00000000..5e3393b9 --- /dev/null +++ b/app/api/dropbox.py @@ -0,0 +1,308 @@ +""" +Dropbox API endpoints +""" +from fastapi import APIRouter, Request, HTTPException, status, Form +import logging +import os +import requests + +from app.auth import require_login +from app.config import settings + +# Set up logging +logger = logging.getLogger(__name__) + +router = APIRouter() + +@router.post("/dropbox/exchange-token") +@require_login +async def exchange_dropbox_token( + request: Request, + client_id: str = Form(...), + client_secret: str = Form(...), + redirect_uri: str = Form(...), + code: str = Form(...), + folder_path: str = Form(None) +): + """ + Exchange an authorization code for a refresh token from Dropbox. + This is done on the server to avoid exposing client secret in the browser. + """ + try: + logger.info("Starting Dropbox token exchange process") + + # Prepare the token request + token_url = "https://api.dropboxapi.com/oauth2/token" + + payload = { + 'client_id': client_id, + 'client_secret': client_secret, + 'code': code, + 'redirect_uri': redirect_uri, + 'grant_type': 'authorization_code' + } + + # Log request details (excluding secret) + safe_payload = payload.copy() + safe_payload['client_secret'] = '[REDACTED]' + safe_payload['code'] = f"{code[:5]}...{code[-5:]}" if len(code) > 10 else '[REDACTED]' + logger.info(f"Token exchange request payload: {safe_payload}") + + # Make the token request + logger.info("Sending POST request to Dropbox for token exchange") + response = requests.post(token_url, data=payload) + + # Check if the request was successful + logger.info(f"Token exchange response status: {response.status_code}") + + if response.status_code != 200: + # Log the error response for debugging + try: + error_json = response.json() + logger.error(f"Token exchange failed with status {response.status_code}: {error_json}") + error_detail = error_json + except Exception as json_err: + logger.error(f"Failed to parse error response as JSON: {str(json_err)}") + logger.error(f"Raw response content: {response.content[:500]}") # Limit log size + error_detail = {"error": "Unknown error", "raw_content_snippet": str(response.content[:100])} + + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Token exchange failed: {error_detail}" + ) + + # Return the token response + token_data = response.json() + + # Validate the token response + if "refresh_token" not in token_data: + logger.error(f"Dropbox returned success but no refresh token found in response: {token_data.keys()}") + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="Dropbox OAuth server returned success but no refresh token was included" + ) + + # Calculate token length for logging + refresh_token_length = len(token_data.get("refresh_token", "")) + access_token_length = len(token_data.get("access_token", "")) + + logger.info(f"Successfully exchanged authorization code for Dropbox tokens. " + f"Refresh token length: {refresh_token_length}, " + f"Access token length: {access_token_length}") + + # Return just what's needed by the frontend + return { + "refresh_token": token_data["refresh_token"], + "access_token": token_data["access_token"], + "expires_in": token_data.get("expires_in", 14400) + } + + except HTTPException: + # Re-raise HTTP exceptions as they already have appropriate status codes + raise + except Exception as e: + logger.exception(f"Unexpected error during Dropbox token exchange: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to exchange token: {str(e)}" + ) + +@router.post("/dropbox/update-settings") +@require_login +async def update_dropbox_settings( + request: Request, + app_key: str = Form(None), + app_secret: str = Form(None), + refresh_token: str = Form(...), + folder_path: str = Form(None) +): + """ + Update Dropbox settings in memory + """ + try: + logger.info("Updating Dropbox settings in memory") + + # Update settings in memory + if refresh_token: + settings.dropbox_refresh_token = refresh_token + logger.info("Updated DROPBOX_REFRESH_TOKEN in memory") + + if app_key: + settings.dropbox_app_key = app_key + logger.info("Updated DROPBOX_APP_KEY in memory") + + if app_secret: + settings.dropbox_app_secret = app_secret + logger.info("Updated DROPBOX_APP_SECRET in memory") + + if folder_path: + settings.dropbox_folder = folder_path + logger.info("Updated DROPBOX_FOLDER in memory") + + # Test token validity would be here, but we'll skip it for now + + return { + "status": "success", + "message": "Dropbox settings have been updated in memory" + } + + except Exception as e: + logger.exception(f"Unexpected error updating Dropbox settings: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to update Dropbox settings: {str(e)}" + ) + +@router.get("/dropbox/test-token") +@require_login +async def test_dropbox_token(request: Request): + """ + Test if the configured Dropbox refresh token is valid. + """ + try: + from app.tasks.upload_to_dropbox import get_dropbox_client + + logger.info("Testing Dropbox token validity") + if not settings.dropbox_refresh_token: + logger.warning("No Dropbox refresh token configured") + return { + "status": "error", + "message": "No Dropbox refresh token is configured" + } + + # Check if app key and app secret are configured + if not settings.dropbox_app_key or not settings.dropbox_app_secret: + logger.warning("Dropbox app key or app secret is missing") + return { + "status": "error", + "message": "Dropbox app key or app secret is missing", + "missing_config": True + } + + # Try to get a client using the configured refresh token + try: + dbx = get_dropbox_client() + # Test connection by getting account info + account = dbx.users_get_current_account() + logger.info(f"Successfully connected to Dropbox as {account.name.display_name}") + + return { + "status": "success", + "message": f"Token is valid! Connected as {account.name.display_name}", + "account": account.name.display_name, + "email": account.email + } + except Exception as e: + error_msg = str(e) + logger.error(f"Dropbox token test failed: {error_msg}") + + # Determine if this is an authentication error + is_auth_error = "auth" in error_msg.lower() or "invalid" in error_msg.lower() + + return { + "status": "error", + "message": f"Token validation failed: {error_msg}", + "is_auth_error": is_auth_error, + "needs_reauth": is_auth_error + } + + except Exception as e: + logger.exception("Unexpected error testing Dropbox token") + return { + "status": "error", + "message": f"Unexpected error: {str(e)}" + } + +@router.post("/dropbox/save-settings") +@require_login +async def save_dropbox_settings( + request: Request, + app_key: str = Form(None), + app_secret: str = Form(None), + refresh_token: str = Form(...), + folder_path: str = Form(None) +): + """ + Save Dropbox settings to the .env file + """ + try: + # Get the path to the .env file + env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env") + + if not os.path.exists(env_path): + logger.error(f".env file not found at {env_path}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Could not find .env file to update" + ) + + logger.info(f"Updating Dropbox settings in {env_path}") + + # Read the current .env file + with open(env_path, "r") as f: + env_lines = f.readlines() + + # Define settings to update + dropbox_settings = { + "DROPBOX_REFRESH_TOKEN": refresh_token, + } + + # Only update these if provided + if app_key: + dropbox_settings["DROPBOX_APP_KEY"] = app_key + if app_secret: + dropbox_settings["DROPBOX_APP_SECRET"] = app_secret + if folder_path: + dropbox_settings["DROPBOX_FOLDER"] = folder_path + + # Process each line and update or add settings + updated = set() + new_env_lines = [] + for line in env_lines: + line = line.rstrip() + is_updated = False + for key, value in dropbox_settings.items(): + if line.startswith(f"{key}=") or line.startswith(f"# {key}="): + if line.startswith("# "): # Uncomment if commented out + line = line[2:] + new_env_lines.append(f"{key}={value}") + updated.add(key) + is_updated = True + break + if not is_updated: + new_env_lines.append(line) + + # Add any settings that weren't updated (they weren't in the file) + for key, value in dropbox_settings.items(): + if key not in updated: + new_env_lines.append(f"{key}={value}") + + # Write the updated .env file + with open(env_path, "w") as f: + f.write("\n".join(new_env_lines) + "\n") + + # Update the settings in memory + if refresh_token: + settings.dropbox_refresh_token = refresh_token + if app_key: + settings.dropbox_app_key = app_key + if app_secret: + settings.dropbox_app_secret = app_secret + if folder_path: + settings.dropbox_folder = folder_path + + logger.info("Successfully updated Dropbox settings") + + return { + "status": "success", + "message": "Dropbox settings have been saved" + } + + except HTTPException: + raise + except Exception as e: + logger.exception(f"Unexpected error saving Dropbox settings: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to save Dropbox settings: {str(e)}" + ) diff --git a/app/api/files.py b/app/api/files.py new file mode 100644 index 00000000..ab17bb46 --- /dev/null +++ b/app/api/files.py @@ -0,0 +1,143 @@ +""" +File-related API endpoints +""" +from fastapi import APIRouter, Request, HTTPException, Depends, UploadFile, File +from sqlalchemy.orm import Session +import logging +import os +import uuid + +from app.auth import require_login +from app.models import FileRecord +from app.config import settings +from app.api.common import get_db +from app.tasks.process_document import process_document + +# Set up logging +logger = logging.getLogger(__name__) + +router = APIRouter() + +@router.get("/files") +@require_login +def list_files_api(request: Request, db: Session = Depends(get_db)): + """ + Returns a JSON list of all FileRecord entries. + Protected by `@require_login`, so only logged-in sessions can access. + + Example response: + [ + { + "id": 123, + "filehash": "abc123...", + "original_filename": "example.pdf", + "local_filename": "/workdir/tmp/.pdf", + "file_size": 1048576, + "mime_type": "application/pdf", + "created_at": "2025-05-01T12:34:56.789000" + }, + ... + ] + """ + files = db.query(FileRecord).order_by(FileRecord.created_at.desc()).all() + # Return a simple list of dicts + result = [] + for f in files: + result.append({ + "id": f.id, + "filehash": f.filehash, + "original_filename": f.original_filename, + "local_filename": f.local_filename, + "file_size": f.file_size, + "mime_type": f.mime_type, + "created_at": f.created_at.isoformat() if f.created_at else None + }) + return result + +@router.delete("/files/{file_id}") +@require_login +def delete_file_record(request: Request, file_id: int, db: Session = Depends(get_db)): + """ + Delete a file record from the database. + This only removes the database entry, not the actual file. + """ + # Check if file deletion is allowed + if not settings.allow_file_delete: + raise HTTPException( + status_code=403, + detail="File deletion is disabled in the configuration" + ) + + try: + # Find the file record + file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() + + if not file_record: + raise HTTPException( + status_code=404, + detail=f"File record with ID {file_id} not found" + ) + + # Log the deletion + logger.info(f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}") + + # Delete the record + db.delete(file_record) + db.commit() + + return { + "status": "success", + "message": f"File record {file_id} deleted successfully" + } + + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.exception(f"Error deleting file record {file_id}: {str(e)}") + raise HTTPException( + status_code=500, + detail=f"Error deleting file record: {str(e)}" + ) + +@router.post("/ui-upload") +@require_login +async def ui_upload(request: Request, file: UploadFile = File(...)): + """Endpoint to accept a user-uploaded file and enqueue it for processing.""" + workdir = settings.workdir + + # Extract just the filename without any path components to prevent path traversal + safe_filename = os.path.basename(file.filename) + + # Generate a unique filename with UUID to prevent overwriting and filename conflicts + unique_id = str(uuid.uuid4()) + # Keep the original extension if present + if "." in safe_filename: + file_extension = safe_filename.rsplit(".", 1)[1] + target_filename = f"{unique_id}.{file_extension}" + else: + target_filename = unique_id + + # Store both the safe original name and the unique name + target_path = os.path.join(workdir, target_filename) + + try: + with open(target_path, "wb") as f: + content = await file.read() + f.write(content) + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"Failed to save file: {e}" + ) + + # Log the mapping between original and safe filename + logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'") + + task = process_document.delay(target_path) + return { + "task_id": task.id, + "status": "queued", + "original_filename": safe_filename, + "stored_filename": target_filename + } diff --git a/app/api/onedrive.py b/app/api/onedrive.py new file mode 100644 index 00000000..1f7bce1e --- /dev/null +++ b/app/api/onedrive.py @@ -0,0 +1,364 @@ +""" +OneDrive API endpoints +""" +from fastapi import APIRouter, Request, HTTPException, status, Form +import logging +import os +import requests + +from app.auth import require_login +from app.config import settings + +# Set up logging +logger = logging.getLogger(__name__) + +router = APIRouter() + +@router.post("/onedrive/exchange-token") +@require_login +async def exchange_onedrive_token( + request: Request, + client_id: str = Form(...), + client_secret: str = Form(...), + redirect_uri: str = Form(...), + code: str = Form(...), + tenant_id: str = Form(...) +): + """ + Exchange an authorization code for a refresh token. + This is done on the server to avoid exposing client secret in the browser. + """ + try: + logger.info(f"Starting OneDrive token exchange process with tenant_id: {tenant_id}") + + # Prepare the token request + token_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" + logger.info(f"Using token URL: {token_url}") + + payload = { + 'client_id': client_id, + 'scope': 'https://graph.microsoft.com/.default offline_access', + 'code': code, + 'redirect_uri': redirect_uri, + 'grant_type': 'authorization_code', + 'client_secret': client_secret + } + + # Log request details (excluding secret) + safe_payload = payload.copy() + safe_payload['client_secret'] = '[REDACTED]' + safe_payload['code'] = f"{code[:5]}...{code[-5:]}" if len(code) > 10 else '[REDACTED]' + logger.info(f"Token exchange request payload: {safe_payload}") + + # Make the token request + logger.info("Sending POST request to Microsoft for token exchange") + response = requests.post(token_url, data=payload) + + # Check if the request was successful + logger.info(f"Token exchange response status: {response.status_code}") + + if response.status_code != 200: + # Log the error response for debugging + try: + error_json = response.json() + logger.error(f"Token exchange failed with status {response.status_code}: {error_json}") + error_detail = error_json + except Exception as json_err: + logger.error(f"Failed to parse error response as JSON: {str(json_err)}") + logger.error(f"Raw response content: {response.content[:500]}") # Limit log size + error_detail = {"error": "Unknown error", "raw_content_snippet": str(response.content[:100])} + + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Token exchange failed: {error_detail}" + ) + + # Return the token response + token_data = response.json() + + # Validate the token response + if "refresh_token" not in token_data: + logger.error(f"Microsoft returned success but no refresh token found in response: {token_data.keys()}") + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="Microsoft OAuth server returned success but no refresh token was included" + ) + + # Calculate token length for logging + refresh_token_length = len(token_data.get("refresh_token", "")) + access_token_length = len(token_data.get("access_token", "")) + + logger.info(f"Successfully exchanged authorization code for OneDrive tokens. " + f"Refresh token length: {refresh_token_length}, " + f"Access token length: {access_token_length}") + + # Return just what's needed by the frontend + return { + "refresh_token": token_data["refresh_token"], + "expires_in": token_data.get("expires_in", 3600) + } + + except HTTPException: + # Re-raise HTTP exceptions as they already have appropriate status codes + raise + except Exception as e: + logger.exception(f"Unexpected error during OneDrive token exchange: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to exchange token: {str(e)}" + ) + +@router.get("/onedrive/test-token") +@require_login +async def test_onedrive_token(request: Request): + """ + Test if the configured OneDrive refresh token is valid. + Provides detailed error information if token is invalid. + """ + try: + from app.tasks.upload_to_onedrive import get_onedrive_token + + logger.info("Testing OneDrive token validity") + if not settings.onedrive_refresh_token: + logger.warning("No OneDrive refresh token configured") + return { + "status": "error", + "message": "No OneDrive refresh token is configured" + } + + # Check if client ID and client secret are configured + if not settings.onedrive_client_id or not settings.onedrive_client_secret: + logger.warning("OneDrive client ID or client secret is missing") + return { + "status": "error", + "message": "OneDrive client ID or client secret is missing", + "missing_config": True + } + + # Try to get an access token using the configured refresh token + try: + access_token = get_onedrive_token() + + # If we got here, token is valid + logger.info("OneDrive token is valid") + return { + "status": "success", + "message": "OneDrive token is valid", + } + except Exception as e: + error_msg = str(e) + logger.error(f"OneDrive token test failed: {error_msg}") + + # Determine if this is an invalid_grant error (expired token) + is_expired = "invalid_grant" in error_msg.lower() + + return { + "status": "error", + "message": f"Token validation failed: {error_msg}", + "is_expired": is_expired, + "needs_reauth": True + } + + except Exception as e: + logger.exception("Unexpected error testing OneDrive token") + return { + "status": "error", + "message": f"Unexpected error: {str(e)}" + } + +@router.post("/onedrive/save-settings") +@require_login +async def save_onedrive_settings( + request: Request, + client_id: str = Form(None), + client_secret: str = Form(None), + refresh_token: str = Form(...), + tenant_id: str = Form("common"), + folder_path: str = Form(None) +): + """ + Save OneDrive settings to the .env file + """ + try: + # Get the path to the .env file + env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env") + + if not os.path.exists(env_path): + logger.error(f".env file not found at {env_path}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Could not find .env file to update" + ) + + logger.info(f"Updating OneDrive settings in {env_path}") + + # Read the current .env file + with open(env_path, "r") as f: + env_lines = f.readlines() + + # Define settings to update + onedrive_settings = { + "ONEDRIVE_REFRESH_TOKEN": refresh_token, + } + + # Only update these if provided + if client_id: + onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id + if client_secret: + onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret + if tenant_id: + onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id + if folder_path: + onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path + + # Process each line and update or add settings + updated = set() + new_env_lines = [] + for line in env_lines: + line = line.rstrip() + is_updated = False + for key, value in onedrive_settings.items(): + if line.startswith(f"{key}=") or line.startswith(f"# {key}="): + if line.startswith("# "): # Uncomment if commented out + line = line[2:] + new_env_lines.append(f"{key}={value}") + updated.add(key) + is_updated = True + break + if not is_updated: + new_env_lines.append(line) + + # Add any settings that weren't updated (they weren't in the file) + for key, value in onedrive_settings.items(): + if key not in updated: + new_env_lines.append(f"{key}={value}") + + # Write the updated .env file + with open(env_path, "w") as f: + f.write("\n".join(new_env_lines) + "\n") + + # Update the settings in memory + if refresh_token: + settings.onedrive_refresh_token = refresh_token + if client_id: + settings.onedrive_client_id = client_id + if client_secret: + settings.onedrive_client_secret = client_secret + if tenant_id: + settings.onedrive_tenant_id = tenant_id + if folder_path: + settings.onedrive_folder_path = folder_path + + logger.info("Successfully updated OneDrive settings") + + return { + "status": "success", + "message": "OneDrive settings have been saved" + } + + except HTTPException: + raise + except Exception as e: + logger.exception(f"Unexpected error saving OneDrive settings: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to save OneDrive settings: {str(e)}" + ) + +@router.post("/onedrive/update-settings") +@require_login +async def update_onedrive_settings( + request: Request, + client_id: str = Form(None), + client_secret: str = Form(None), + refresh_token: str = Form(...), + tenant_id: str = Form("common"), + folder_path: str = Form(None) +): + """ + Update OneDrive settings in memory (without modifying .env file) + """ + try: + logger.info("Updating OneDrive settings in memory") + + # Update settings in memory + if refresh_token: + settings.onedrive_refresh_token = refresh_token + logger.info("Updated ONEDRIVE_REFRESH_TOKEN in memory") + + if client_id: + settings.onedrive_client_id = client_id + logger.info("Updated ONEDRIVE_CLIENT_ID in memory") + + if client_secret: + settings.onedrive_client_secret = client_secret + logger.info("Updated ONEDRIVE_CLIENT_SECRET in memory") + + if tenant_id: + settings.onedrive_tenant_id = tenant_id + logger.info("Updated ONEDRIVE_TENANT_ID in memory") + + if folder_path: + settings.onedrive_folder_path = folder_path + logger.info("Updated ONEDRIVE_FOLDER_PATH in memory") + + # Test the token to make sure it works + try: + from app.tasks.upload_to_onedrive import get_onedrive_token + access_token = get_onedrive_token() + logger.info("Successfully tested OneDrive token") + except Exception as e: + logger.error(f"Token test failed after updating settings: {str(e)}") + return { + "status": "warning", + "message": "Settings updated but token test failed: " + str(e) + } + + return { + "status": "success", + "message": "OneDrive settings have been updated in memory" + } + + except Exception as e: + logger.exception(f"Unexpected error updating OneDrive settings: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to update OneDrive settings: {str(e)}" + ) + +@router.get("/onedrive/get-full-config") +@require_login +async def get_onedrive_full_config(request: Request): + """ + Get the full OneDrive configuration for sharing with worker nodes + """ + try: + # Create a configuration object with all OneDrive settings + config = { + "client_id": settings.onedrive_client_id or "", + "client_secret": settings.onedrive_client_secret or "", + "tenant_id": settings.onedrive_tenant_id or "common", + "refresh_token": settings.onedrive_refresh_token or "", + "folder_path": settings.onedrive_folder_path or "Documents/Uploads" + } + + # Generate environment variable format + env_format = "\n".join([ + f"ONEDRIVE_CLIENT_ID={config['client_id']}", + f"ONEDRIVE_CLIENT_SECRET={config['client_secret']}", + f"ONEDRIVE_TENANT_ID={config['tenant_id']}", + f"ONEDRIVE_REFRESH_TOKEN={config['refresh_token']}", + f"ONEDRIVE_FOLDER_PATH={config['folder_path']}" + ]) + + return { + "status": "success", + "config": config, + "env_format": env_format + } + except Exception as e: + logger.exception("Error getting OneDrive configuration") + return { + "status": "error", + "message": str(e) + } diff --git a/app/api/process.py b/app/api/process.py new file mode 100644 index 00000000..8163a7fe --- /dev/null +++ b/app/api/process.py @@ -0,0 +1,140 @@ +""" +Document processing API endpoints +""" +from fastapi import APIRouter, HTTPException +import logging +import os + +from app.auth import require_login +from app.config import settings +from app.api.common import resolve_file_path +from app.tasks.process_document import process_document +from app.tasks.upload_to_dropbox import upload_to_dropbox +from app.tasks.upload_to_paperless import upload_to_paperless +from app.tasks.upload_to_nextcloud import upload_to_nextcloud +from app.tasks.upload_to_google_drive import upload_to_google_drive +from app.tasks.upload_to_onedrive import upload_to_onedrive +from app.tasks.send_to_all import send_to_all_destinations + +# Set up logging +logger = logging.getLogger(__name__) + +router = APIRouter() + +@router.post("/process/") +@require_login +def process(file_path: str): + """API Endpoint to start document processing.""" + file_path = resolve_file_path(file_path) + + if not os.path.exists(file_path): + raise HTTPException( + status_code=400, detail=f"File {file_path} not found." + ) + + task = process_document.delay(file_path) + return {"task_id": task.id, "status": "queued"} + +@router.post("/send_to_dropbox/") +@require_login +def send_to_dropbox_endpoint(file_path: str): + """Send a document to Dropbox.""" + file_path = resolve_file_path(file_path, 'processed') + if not os.path.exists(file_path): + raise HTTPException( + status_code=400, detail=f"File {file_path} not found." + ) + task = upload_to_dropbox.delay(file_path) + return {"task_id": task.id, "status": "queued"} + +@router.post("/send_to_paperless/") +@require_login +def send_to_paperless_endpoint(file_path: str): + """Send a document to Paperless-ngx.""" + file_path = resolve_file_path(file_path, 'processed') + if not os.path.exists(file_path): + raise HTTPException( + status_code=400, detail=f"File {file_path} not found." + ) + task = upload_to_paperless.delay(file_path) + return {"task_id": task.id, "status": "queued"} + +@router.post("/send_to_nextcloud/") +@require_login +def send_to_nextcloud_endpoint(file_path: str): + """Send a document to NextCloud.""" + file_path = resolve_file_path(file_path, 'processed') + if not os.path.exists(file_path): + raise HTTPException( + status_code=400, detail=f"File {file_path} not found." + ) + task = upload_to_nextcloud.delay(file_path) + return {"task_id": task.id, "status": "queued"} + +@router.post("/send_to_google_drive/") +@require_login +def send_to_google_drive_endpoint(file_path: str): + """Send a document to Google Drive.""" + file_path = resolve_file_path(file_path, 'processed') + if not os.path.exists(file_path): + raise HTTPException( + status_code=400, detail=f"File {file_path} not found." + ) + task = upload_to_google_drive.delay(file_path) + return {"task_id": task.id, "status": "queued"} + +@router.post("/send_to_onedrive/") +@require_login +def send_to_onedrive_endpoint(file_path: str): + """Send a document to OneDrive.""" + file_path = resolve_file_path(file_path, 'processed') + if not os.path.exists(file_path): + raise HTTPException( + status_code=400, detail=f"File {file_path} not found." + ) + task = upload_to_onedrive.delay(file_path) + return {"task_id": task.id, "status": "queued"} + +@router.post("/send_to_all_destinations/") +@require_login +def send_to_all_destinations_endpoint(file_path: str): + """Call the aggregator task that sends this file to all configured destinations.""" + file_path = resolve_file_path(file_path, 'processed') + + if not os.path.exists(file_path): + raise HTTPException( + status_code=400, detail=f"File {file_path} not found." + ) + + task = send_to_all_destinations.delay(file_path) + return {"task_id": task.id, "status": "queued", "file_path": file_path} + +@router.post("/processall") +@require_login +def process_all_pdfs_in_workdir(): + """Finds all .pdf files in and enqueues them for processing.""" + target_dir = settings.workdir + if not os.path.exists(target_dir): + raise HTTPException( + status_code=400, detail=f"Directory {target_dir} does not exist." + ) + + pdf_files = [] + for filename in os.listdir(target_dir): + if filename.lower().endswith(".pdf"): + pdf_files.append(filename) + + if not pdf_files: + return {"message": "No PDF files found in that directory."} + + task_ids = [] + for pdf in pdf_files: + file_path = os.path.join(target_dir, pdf) + task = process_document.delay(file_path) + task_ids.append(task.id) + + return { + "message": f"Enqueued {len(pdf_files)} PDFs to upload_to_s3", + "pdf_files": pdf_files, + "task_ids": task_ids + } diff --git a/app/api/user.py b/app/api/user.py new file mode 100644 index 00000000..c0fdef01 --- /dev/null +++ b/app/api/user.py @@ -0,0 +1,33 @@ +""" +User-related API endpoints +""" +from fastapi import APIRouter, Request, HTTPException +from hashlib import md5 +import logging + +# Set up logging +logger = logging.getLogger(__name__) + +router = APIRouter() + +@router.get("/whoami") +async def whoami(request: Request): + """ + Returns user info if logged in, else 401. + """ + user = request.session.get("user") + if not user: + raise HTTPException(status_code=401, detail="Not logged in") + + email = user.get("email") + if not email: + raise HTTPException(status_code=400, detail="User has no email in session") + + # Generate Gravatar URL from email + email_hash = md5(email.strip().lower().encode()).hexdigest() + gravatar_url = f"https://www.gravatar.com/avatar/{email_hash}?d=identicon" + + return { + "email": email, + "picture": gravatar_url + } diff --git a/app/config.py b/app/config.py index dba2d608..ea30d184 100644 --- a/app/config.py +++ b/app/config.py @@ -118,6 +118,9 @@ class Settings(BaseSettings): uptime_kuma_url: Optional[str] = None uptime_kuma_ping_interval: int = 5 # Default ping interval in minutes + # Feature flags + allow_file_delete: bool = True # Default to allowing file deletion from database + # Get version from file or environment @property def version(self) -> str: diff --git a/app/frontend.py b/app/frontend.py index 35954cb7..dc396ea1 100644 --- a/app/frontend.py +++ b/app/frontend.py @@ -5,6 +5,7 @@ from pathlib import Path from sqlalchemy.orm import Session from datetime import datetime import logging +from fastapi.responses import FileResponse # Add this import from app.auth import require_login from app.database import SessionLocal @@ -50,9 +51,12 @@ async def serve_upload(request: Request): @router.get("/favicon.ico", include_in_schema=False) def favicon(): - # If you have a real favicon in `frontend/static/favicon.ico`: + # Return the favicon file as a FileResponse instead of the path string favicon_path = Path(__file__).parent.parent / "frontend" / "static" / "favicon.ico" - return str(favicon_path) + if not favicon_path.exists(): + # If favicon doesn't exist, return a 404 + raise HTTPException(status_code=404, detail="Favicon not found") + return FileResponse(favicon_path) @router.get("/status") @require_login diff --git a/app/main.py b/app/main.py index 5cfc6185..579267c9 100644 --- a/app/main.py +++ b/app/main.py @@ -15,9 +15,9 @@ from app.database import init_db from app.config import settings from app.utils.config_validator import check_all_configs -# Import both the traditional and new routers - we'll keep both available for compatibility +# Import both the traditional and new routers from app.frontend import router as frontend_router_original -from app.api import router as api_router_original +from app.api import router as api_router # Updated import for the refactored API router from app.auth import router as auth_router # Load configuration from .env for the session key @@ -67,8 +67,7 @@ async def startup_event(): else: logging.info("Application started with valid configuration") - logging.info("Router organization note: Using route handlers from main app directory for now") - logging.info("In the future, we'll transition fully to the frontend/ and api/ submodules") + logging.info("Router organization: Using refactored API routers from app/api/ directory") # Custom 404 - we can still return the Jinja2 template, or the old static file: @app.exception_handler(404) @@ -95,10 +94,9 @@ async def custom_500_handler(request: Request, exc: Exception): def test_500(): raise RuntimeError("Testing forced 500 error!") -# Include the routers - for now we're using the original routers -# Later we can switch to the organized router structure +# Include the routers app.include_router(frontend_router_original) app.include_router(auth_router) -app.include_router(api_router_original, prefix="/api") +app.include_router(api_router, prefix="/api") # Use the refactored API router diff --git a/app/tasks/process_with_azure_document_intelligence.py b/app/tasks/process_with_azure_document_intelligence.py index f0dbf562..80f2c978 100644 --- a/app/tasks/process_with_azure_document_intelligence.py +++ b/app/tasks/process_with_azure_document_intelligence.py @@ -34,6 +34,30 @@ def get_pdf_page_count(file_path): logger.error(f"Error getting PDF page count: {e}") return None +def check_page_rotation(result, filename): + """ + Checks if pages in the document are rotated and logs the rotation information. + + Args: + result: The AnalyzeResult from Azure Document Intelligence API + filename: The name of the file being processed + """ + logger.error(f"Checking rotation for document: {filename}") + + if not hasattr(result, 'pages') or not result.pages: + logger.error(f"No page information available for rotation check: {filename}") + return + + for i, page in enumerate(result.pages): + if hasattr(page, 'angle'): + rotation_angle = page.angle + if rotation_angle != 0: + logger.error(f"Page {i+1} is rotated by {rotation_angle} degrees") + else: + logger.error(f"Page {i+1} has no rotation (0 degrees)") + else: + logger.error(f"Page {i+1} rotation information not available") + @celery.task(base=BaseTaskWithRetry) def process_with_azure_document_intelligence(filename: str): """ @@ -81,6 +105,9 @@ def process_with_azure_document_intelligence(filename: str): result: AnalyzeResult = poller.result() operation_id = poller.details["operation_id"] + # Check and log page rotation information + check_page_rotation(result, filename) + # Retrieve the processed searchable PDF response = document_intelligence_client.get_analyze_result_pdf( model_id=result.model_id, result_id=operation_id diff --git a/app/utils/config_validator.py b/app/utils/config_validator.py index 5d0dfbd3..28e58421 100644 --- a/app/utils/config_validator.py +++ b/app/utils/config_validator.py @@ -246,10 +246,9 @@ def get_provider_status(): # Check NextCloud configuration nextcloud_url = getattr(settings, 'nextcloud_upload_url', 'Not set') # Extract base URL from WebDAV URL (remove the /remote.php part and everything after it) - if nextcloud_url != 'Not set' and '/remote.php' in nextcloud_url: + nextcloud_base_url = nextcloud_url + if nextcloud_url != 'Not set' and nextcloud_url is not None and '/remote.php' in nextcloud_url: nextcloud_base_url = nextcloud_url.split('/remote.php')[0] - else: - nextcloud_base_url = nextcloud_url providers["NextCloud"] = { "name": "NextCloud", diff --git a/frontend/templates/files.html b/frontend/templates/files.html index c975e2dc..43c09983 100644 --- a/frontend/templates/files.html +++ b/frontend/templates/files.html @@ -4,6 +4,43 @@ {% block head_extra %} + + {% endblock %} {% block content %} @@ -11,6 +48,27 @@

File Records

+ + + {% endblock %} @@ -18,13 +76,78 @@