From 94889e9e41ae73194e7c4d9bdb8fbae4cd763dcf Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Thu, 3 Apr 2025 10:35:55 +0200 Subject: [PATCH] feat: implement Dropbox integration with OAuth setup and error handling --- app/api.py | 293 +++++++++++++ app/frontend.py | 57 +++ app/tasks/process_document.py | 2 +- app/tasks/refine_text_with_gpt.py | 2 +- app/tasks/upload_to_dropbox.py | 51 ++- app/utils/config_validator.py | 9 +- docs/setup-guides/dropbox-setup.md | 0 frontend/templates/base.html | 1 + frontend/templates/dropbox.html | 406 ++++++++++++++++++ frontend/templates/dropbox_callback.html | 227 ++++++++++ .../templates/dropbox_callback_error.html | 40 ++ requirements.txt | 2 +- 12 files changed, 1079 insertions(+), 11 deletions(-) delete mode 100644 docs/setup-guides/dropbox-setup.md create mode 100644 frontend/templates/dropbox.html create mode 100644 frontend/templates/dropbox_callback.html create mode 100644 frontend/templates/dropbox_callback_error.html diff --git a/app/api.py b/app/api.py index 17c45f98..7dc90c7f 100644 --- a/app/api.py +++ b/app/api.py @@ -644,4 +644,297 @@ async def get_onedrive_full_config(request: Request): "message": str(e) } +@router.post("/dropbox/exchange-token") +@require_login +async def exchange_dropbox_token( + request: Request, + client_id: str = Form(...), + client_secret: str = Form(...), + redirect_uri: str = Form(...), + code: str = Form(...), + folder_path: str = Form(None) +): + """ + Exchange an authorization code for a refresh token from Dropbox. + This is done on the server to avoid exposing client secret in the browser. + """ + try: + logger.info("Starting Dropbox token exchange process") + + # Prepare the token request + token_url = "https://api.dropboxapi.com/oauth2/token" + + payload = { + 'client_id': client_id, + 'client_secret': client_secret, + 'code': code, + 'redirect_uri': redirect_uri, + 'grant_type': 'authorization_code' + } + + # Log request details (excluding secret) + safe_payload = payload.copy() + safe_payload['client_secret'] = '[REDACTED]' + safe_payload['code'] = f"{code[:5]}...{code[-5:]}" if len(code) > 10 else '[REDACTED]' + logger.info(f"Token exchange request payload: {safe_payload}") + + # Make the token request + logger.info("Sending POST request to Dropbox for token exchange") + response = requests.post(token_url, data=payload) + + # Check if the request was successful + logger.info(f"Token exchange response status: {response.status_code}") + + if response.status_code != 200: + # Log the error response for debugging + try: + error_json = response.json() + logger.error(f"Token exchange failed with status {response.status_code}: {error_json}") + error_detail = error_json + except Exception as json_err: + logger.error(f"Failed to parse error response as JSON: {str(json_err)}") + logger.error(f"Raw response content: {response.content[:500]}") # Limit log size + error_detail = {"error": "Unknown error", "raw_content_snippet": str(response.content[:100])} + + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Token exchange failed: {error_detail}" + ) + + # Return the token response + token_data = response.json() + + # Validate the token response + if "refresh_token" not in token_data: + logger.error(f"Dropbox returned success but no refresh token found in response: {token_data.keys()}") + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="Dropbox OAuth server returned success but no refresh token was included" + ) + + # Calculate token length for logging + refresh_token_length = len(token_data.get("refresh_token", "")) + access_token_length = len(token_data.get("access_token", "")) + + logger.info(f"Successfully exchanged authorization code for Dropbox tokens. " + f"Refresh token length: {refresh_token_length}, " + f"Access token length: {access_token_length}") + + # Return just what's needed by the frontend + return { + "refresh_token": token_data["refresh_token"], + "access_token": token_data["access_token"], + "expires_in": token_data.get("expires_in", 14400) + } + + except HTTPException: + # Re-raise HTTP exceptions as they already have appropriate status codes + raise + except Exception as e: + logger.exception(f"Unexpected error during Dropbox token exchange: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to exchange token: {str(e)}" + ) + +@router.post("/dropbox/update-settings") +@require_login +async def update_dropbox_settings( + request: Request, + app_key: str = Form(None), + app_secret: str = Form(None), + refresh_token: str = Form(...), + folder_path: str = Form(None) +): + """ + Update Dropbox settings in memory + """ + try: + logger.info("Updating Dropbox settings in memory") + + # Update settings in memory + if refresh_token: + settings.dropbox_refresh_token = refresh_token + logger.info("Updated DROPBOX_REFRESH_TOKEN in memory") + + if app_key: + settings.dropbox_app_key = app_key + logger.info("Updated DROPBOX_APP_KEY in memory") + + if app_secret: + settings.dropbox_app_secret = app_secret + logger.info("Updated DROPBOX_APP_SECRET in memory") + + if folder_path: + settings.dropbox_folder = folder_path + logger.info("Updated DROPBOX_FOLDER in memory") + + # Test token validity would be here, but we'll skip it for now + + return { + "status": "success", + "message": "Dropbox settings have been updated in memory" + } + + except Exception as e: + logger.exception(f"Unexpected error updating Dropbox settings: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to update Dropbox settings: {str(e)}" + ) + +@router.get("/dropbox/test-token") +@require_login +async def test_dropbox_token(request: Request): + """ + Test if the configured Dropbox refresh token is valid. + """ + try: + from app.tasks.upload_to_dropbox import get_dropbox_client + + logger.info("Testing Dropbox token validity") + if not settings.dropbox_refresh_token: + logger.warning("No Dropbox refresh token configured") + return { + "status": "error", + "message": "No Dropbox refresh token is configured" + } + + # Check if app key and app secret are configured + if not settings.dropbox_app_key or not settings.dropbox_app_secret: + logger.warning("Dropbox app key or app secret is missing") + return { + "status": "error", + "message": "Dropbox app key or app secret is missing", + "missing_config": True + } + + # Try to get a client using the configured refresh token + try: + dbx = get_dropbox_client() + # Test connection by getting account info + account = dbx.users_get_current_account() + logger.info(f"Successfully connected to Dropbox as {account.name.display_name}") + + return { + "status": "success", + "message": f"Token is valid! Connected as {account.name.display_name}", + "account": account.name.display_name, + "email": account.email + } + except Exception as e: + error_msg = str(e) + logger.error(f"Dropbox token test failed: {error_msg}") + + # Determine if this is an authentication error + is_auth_error = "auth" in error_msg.lower() or "invalid" in error_msg.lower() + + return { + "status": "error", + "message": f"Token validation failed: {error_msg}", + "is_auth_error": is_auth_error, + "needs_reauth": is_auth_error + } + + except Exception as e: + logger.exception("Unexpected error testing Dropbox token") + return { + "status": "error", + "message": f"Unexpected error: {str(e)}" + } + +@router.post("/dropbox/save-settings") +@require_login +async def save_dropbox_settings( + request: Request, + app_key: str = Form(None), + app_secret: str = Form(None), + refresh_token: str = Form(...), + folder_path: str = Form(None) +): + """ + Save Dropbox settings to the .env file + """ + try: + # Get the path to the .env file + env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env") + + if not os.path.exists(env_path): + logger.error(f".env file not found at {env_path}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Could not find .env file to update" + ) + + logger.info(f"Updating Dropbox settings in {env_path}") + + # Read the current .env file + with open(env_path, "r") as f: + env_lines = f.readlines() + + # Define settings to update + dropbox_settings = { + "DROPBOX_REFRESH_TOKEN": refresh_token, + } + + # Only update these if provided + if app_key: + dropbox_settings["DROPBOX_APP_KEY"] = app_key + if app_secret: + dropbox_settings["DROPBOX_APP_SECRET"] = app_secret + if folder_path: + dropbox_settings["DROPBOX_FOLDER"] = folder_path + + # Process each line and update or add settings + updated = set() + new_env_lines = [] + for line in env_lines: + line = line.rstrip() + is_updated = False + for key, value in dropbox_settings.items(): + if line.startswith(f"{key}=") or line.startswith(f"# {key}="): + if line.startswith("# "): # Uncomment if commented out + line = line[2:] + new_env_lines.append(f"{key}={value}") + updated.add(key) + is_updated = True + break + if not is_updated: + new_env_lines.append(line) + + # Add any settings that weren't updated (they weren't in the file) + for key, value in dropbox_settings.items(): + if key not in updated: + new_env_lines.append(f"{key}={value}") + + # Write the updated .env file + with open(env_path, "w") as f: + f.write("\n".join(new_env_lines) + "\n") + + # Update the settings in memory + if refresh_token: + settings.dropbox_refresh_token = refresh_token + if app_key: + settings.dropbox_app_key = app_key + if app_secret: + settings.dropbox_app_secret = app_secret + if folder_path: + settings.dropbox_folder = folder_path + + logger.info("Successfully updated Dropbox settings") + + return { + "status": "success", + "message": "Dropbox settings have been saved" + } + + except HTTPException: + raise + except Exception as e: + logger.exception(f"Unexpected error saving Dropbox settings: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to save Dropbox settings: {str(e)}" + ) + diff --git a/app/frontend.py b/app/frontend.py index 30ebf69e..b38befc1 100644 --- a/app/frontend.py +++ b/app/frontend.py @@ -159,3 +159,60 @@ async def onedrive_callback(request: Request, code: str = None, error: str = Non } ) +@router.get("/dropbox-setup") +@require_login +async def dropbox_setup_page(request: Request): + """ + Setup page for the Dropbox integration. + Shows configuration status and setup instructions. + """ + # Check Dropbox configuration + is_configured = bool(settings.dropbox_app_key and + settings.dropbox_app_secret and + settings.dropbox_refresh_token) + + return templates.TemplateResponse( + "dropbox.html", + { + "request": request, + "is_configured": is_configured, + "app_key_value": settings.dropbox_app_key or "", + "app_secret_value": settings.dropbox_app_secret if settings.dropbox_app_secret else "", + "refresh_token_value": settings.dropbox_refresh_token if settings.dropbox_refresh_token else "", + "folder_path": settings.dropbox_folder or "/Documents/Uploads" # Default folder path + } + ) + +@router.get("/dropbox-callback") +@require_login +async def dropbox_callback(request: Request, code: str = None, error: str = None): + """ + Callback endpoint for Dropbox OAuth flow. + Automatically exchanges the code for a token and saves it to the configuration. + """ + if error: + return templates.TemplateResponse( + "dropbox_callback_error.html", + {"request": request, "error": error} + ) + + if not code: + return templates.TemplateResponse( + "dropbox_callback_error.html", + {"request": request, "error": "No authorization code received from Dropbox"} + ) + + # Display the processing page with automatic token exchange + # Note: We provide empty strings for app_key_value and app_secret_value + # to prevent overriding what's in sessionStorage + return templates.TemplateResponse( + "dropbox_callback.html", + { + "request": request, + "code": code, + "app_key_value": "", # The callback will prioritize sessionStorage values + "app_secret_value": "", # The callback will prioritize sessionStorage values + "folder_path": "" # The callback will prioritize sessionStorage values + } + ) + diff --git a/app/tasks/process_document.py b/app/tasks/process_document.py index 95658aa5..880264d2 100644 --- a/app/tasks/process_document.py +++ b/app/tasks/process_document.py @@ -26,7 +26,7 @@ def process_document(original_local_file: str): 2. If not found, insert a new DB row and continue with the pipeline: - Copy file to /workdir/tmp - Check for embedded text. If present, run local GPT extraction - - Otherwise, queue Textract-based OCR + - Otherwise, queue Azure Document Intelligence processing """ if not os.path.exists(original_local_file): diff --git a/app/tasks/refine_text_with_gpt.py b/app/tasks/refine_text_with_gpt.py index b01cce3a..599c568a 100644 --- a/app/tasks/refine_text_with_gpt.py +++ b/app/tasks/refine_text_with_gpt.py @@ -30,5 +30,5 @@ def refine_text_with_gpt(filename: str, raw_text: str): from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt extract_metadata_with_gpt.delay(filename, cleaned_text) - return {"s3_file": filename, "cleaned_text": cleaned_text} + return {"filename": filename, "cleaned_text": cleaned_text} diff --git a/app/tasks/upload_to_dropbox.py b/app/tasks/upload_to_dropbox.py index 24497a95..03415953 100644 --- a/app/tasks/upload_to_dropbox.py +++ b/app/tasks/upload_to_dropbox.py @@ -56,6 +56,49 @@ def get_dropbox_access_token(): logger.error(error_msg) raise Exception(error_msg) +def get_dropbox_client(): + """ + Create and return an authenticated Dropbox client using the configured refresh token. + + Returns: + dropbox.Dropbox: Authenticated Dropbox client instance + + Raises: + ValueError: If required Dropbox configuration is missing + AuthError: If authentication with Dropbox fails + """ + app_key = settings.dropbox_app_key + app_secret = settings.dropbox_app_secret + refresh_token = settings.dropbox_refresh_token + + # Validate configuration + if not app_key or not app_secret: + raise ValueError("Dropbox app key or app secret is not configured") + + if not refresh_token: + raise ValueError("Dropbox refresh token is not configured") + + # Create a Dropbox client with refresh token + try: + dbx = dropbox.Dropbox( + app_key=app_key, + app_secret=app_secret, + oauth2_refresh_token=refresh_token + ) + + # Test the connection + dbx.users_get_current_account() + logger.info("Successfully authenticated with Dropbox") + return dbx + + except AuthError as auth_error: + logger.error(f"Dropbox authentication failed: {str(auth_error)}") + raise + + except Exception as e: + logger.error(f"Error creating Dropbox client: {str(e)}") + raise + @celery.task(base=BaseTaskWithRetry) def upload_to_dropbox(file_path: str): """ @@ -76,12 +119,8 @@ def upload_to_dropbox(file_path: str): filename = os.path.basename(file_path) try: - # Get access token from refresh token - access_token = get_dropbox_access_token() - if not access_token: - return {"status": "Failed", "reason": "Could not obtain access token"} - - dbx = dropbox.Dropbox(access_token) + # Get the Dropbox client + dbx = get_dropbox_client() # Calculate remote path based on local file structure remote_base = settings.dropbox_folder or "" diff --git a/app/utils/config_validator.py b/app/utils/config_validator.py index 6eb453d2..6bc87ea3 100644 --- a/app/utils/config_validator.py +++ b/app/utils/config_validator.py @@ -144,10 +144,15 @@ def get_provider_status(): # Check Dropbox configuration providers["Dropbox"] = { "name": "Dropbox", - "configured": bool(getattr(settings, 'dropbox_refresh_token', None)), + "configured": bool(getattr(settings, 'dropbox_app_key', None) and + getattr(settings, 'dropbox_app_secret', None) and + getattr(settings, 'dropbox_refresh_token', None)), "enabled": True, "details": { - "folder": getattr(settings, 'dropbox_folder', 'Not set') + "folder": getattr(settings, 'dropbox_folder', 'Not set'), + "app_key": getattr(settings, 'dropbox_app_key', 'Not set'), + "app_secret": getattr(settings, 'dropbox_app_secret', None) and "Configured" or "Not set", + "refresh_token": getattr(settings, 'dropbox_refresh_token', None) and "Configured" or "Not set" } } diff --git a/docs/setup-guides/dropbox-setup.md b/docs/setup-guides/dropbox-setup.md deleted file mode 100644 index e69de29b..00000000 diff --git a/frontend/templates/base.html b/frontend/templates/base.html index 78918304..0bfa04f0 100644 --- a/frontend/templates/base.html +++ b/frontend/templates/base.html @@ -59,6 +59,7 @@ >
OneDrive Setup + Dropbox Setup
diff --git a/frontend/templates/dropbox.html b/frontend/templates/dropbox.html new file mode 100644 index 00000000..5f82a80e --- /dev/null +++ b/frontend/templates/dropbox.html @@ -0,0 +1,406 @@ +{% extends "base.html" %} +{% block title %}Dropbox Setup{% endblock %} + +{% block content %} +
+
+

Dropbox Integration Setup

+

+ Configure the Dropbox integration for DocuNova using our setup wizard. +

+ + + +
+
+
+ + + +
+
+

+ Note: You need to create a Dropbox app and obtain API credentials. The setup wizard will guide you through this process. +

+
+
+
+
+ +
+

Quick Setup Guide

+ +
+

Step 1: Create a Dropbox App

+
    +
  1. Go to the Dropbox Developer Apps Console
    2. +
  2. Click "Create app"
    3. +
  3. Select "Scoped access" for API
    4. +
  4. Choose "Full Dropbox" access (or "App folder" for more restricted access)
    5. +
  5. Give your app a name (e.g., "DocuNova")
    6. +
  6. Click "Create app"
    7. +
+
+ +
+

Step 2: Configure App Permissions

+
    +
  1. In your app's settings page, go to the "Permissions" tab
    2. +
  2. Enable the following permissions: +
      +
    • files.content.write (to upload files)
      • +
    • files.content.read (if you need to read file content)
      • +
    +
    3. +
  3. Click "Submit" to save changes
    4. +
+
+ +
+

Step 3: Set OAuth 2 Redirect URI

+
    +
  1. In your app's settings page, go to the "OAuth 2" section
    2. +
  2. Add a redirect URI: {{ request.url.scheme }}://{{ request.url.netloc }}/dropbox-callback
    3. +
  3. Click "Add" to save the redirect URI
    4. +
+
+
+ +
+

Complete Setup with Wizard

+ +
+
+ + +
+ +
+ + +
+ +
+ + +

Enter the folder path where files should be uploaded (e.g., /Documents/Uploads)

+
+ +
+ +
+ + +
+
+
+
+ {% if is_configured %} + + {% else %} + + {% endif %} +
+
+

+ {% if is_configured %} + Dropbox integration is properly configured! Your refresh token is valid. + {% else %} + Dropbox integration is not completely configured or token may be invalid. + {% endif %} +

+
+
+
+ +
+ + +
+ + +
+

Configuration for Worker Nodes

+

+ Copy these environment variables to configure all worker nodes: +

+ +
+ 
DROPBOX_APP_KEY={{ app_key_value }}
+DROPBOX_APP_SECRET={{ app_secret_value|default('YOUR_APP_SECRET', true) }}
+DROPBOX_REFRESH_TOKEN={{ refresh_token_value|default('YOUR_REFRESH_TOKEN', true) }}
+DROPBOX_FOLDER={{ folder_path|default('/Documents/Uploads', true) }}
+ + +
+ +

+ Add these variables to your .env file or environment configuration. +

+
+
+
+
+ +
+
+
+ + + +
+
+

Need More Information?

+

+ For detailed instructions and troubleshooting, refer to the + Dropbox Setup Documentation. +

+
+
+
+ +
+ + Back to Status + +
+ + + +
+{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/frontend/templates/dropbox_callback.html b/frontend/templates/dropbox_callback.html new file mode 100644 index 00000000..5787120e --- /dev/null +++ b/frontend/templates/dropbox_callback.html @@ -0,0 +1,227 @@ +{% extends "base.html" %} +{% block title %}Dropbox Authorization Processing{% endblock %} + +{% block content %} +
+
+
+ + + +

Processing Authorization

+

Please wait while we complete the Dropbox authorization process...

+
+ +
+
+
+ +
+

Exchanging authorization code for refresh token...

+
+ + + + +
+
+{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/frontend/templates/dropbox_callback_error.html b/frontend/templates/dropbox_callback_error.html new file mode 100644 index 00000000..442321d3 --- /dev/null +++ b/frontend/templates/dropbox_callback_error.html @@ -0,0 +1,40 @@ +{% extends "base.html" %} +{% block title %}Dropbox Authorization Error{% endblock %} + +{% block content %} +
+
+
+ + + +

Authorization Failed

+

Sorry, we couldn't complete the Dropbox authorization.

+
+ +
+
+
+
+ +
+
+

Error details:

+
+

{{ error }}

+
+
+
+
+
+ +
+ + Return to Setup + +
+
+
+{% endblock %} diff --git a/requirements.txt b/requirements.txt index 9eb9a356..9394c34a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ openai # GPT integration for metadata extraction pymupdf # PDF processing, text extraction, and detection (imported as 'fitz') PyPDF2 # PDF processing for page counting requests # HTTP client -dropbox # Dropbox integration +dropbox>=11.36.0 # Dropbox integration azure-ai-documentintelligence # Azure OCR service authlib # Authentication python-dotenv # Environment variables