perf: optimize url upload with async i/o

Replaced synchronous `requests.get` and `open().write` in the `process_url` endpoint with `httpx.AsyncClient` and `aiofiles.open`. This prevents the FastAPI event loop from blocking during large file downloads.

Updated test suite in `tests/test_url_upload.py` to use `AsyncMock` to mock `httpx.AsyncClient.stream` contexts and async generators properly, covering all original conditions and HTTP error handling paths.

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/api/url_upload.py

@@ -9,7 +9,8 @@ import urllib.parse
 import uuid
 from typing import Optional
 
-import requests
+import aiofiles
+import httpx
 from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel, HttpUrl, field_validator
 
@@ -153,67 +154,66 @@ async def process_url(request: Request, url_request: URLUploadRequest):
         logger.info(f"Downloading file from URL: {url}")
 
         # Use configured timeout to prevent hanging
-        response = requests.get(
-            url,
+        async with httpx.AsyncClient(
             timeout=settings.http_request_timeout,
-            stream=True,  # Stream to handle large files
-            allow_redirects=True,  # Follow redirects
+            follow_redirects=True,
             headers={
                 "User-Agent": "DocuElevate/1.0",  # Identify ourselves
             },
-        )
-        response.raise_for_status()
+        ) as client:
+            async with client.stream("GET", url) as response:
+                response.raise_for_status()
 
-        # Validate content type
-        content_type = response.headers.get("Content-Type", "")
-        if not validate_file_type(content_type, safe_filename):
-            raise HTTPException(
-                status_code=400,
-                detail=f"Unsupported file type: {content_type}. "
-                "Supported types: PDF, Office documents, images, plain text",
-            )
+                # Validate content type
+                content_type = response.headers.get("Content-Type", "")
+                if not validate_file_type(content_type, safe_filename):
+                    raise HTTPException(
+                        status_code=400,
+                        detail=f"Unsupported file type: {content_type}. "
+                        "Supported types: PDF, Office documents, images, plain text",
+                    )
 
-        # Check content length before downloading
-        content_length = response.headers.get("Content-Length")
-        if content_length:
-            file_size = int(content_length)
-            max_size = settings.max_upload_size
-            if file_size > max_size:
-                raise HTTPException(
-                    status_code=413,
-                    detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
-                )
-
-        # Generate unique filename
-        unique_id = str(uuid.uuid4())
-        if "." in safe_filename:
-            file_extension = safe_filename.rsplit(".", 1)[1]
-            target_filename = f"{unique_id}.{file_extension}"
-        else:
-            target_filename = unique_id
-
-        target_path = os.path.join(settings.workdir, target_filename)
-
-        # Download file in chunks to handle large files
-        downloaded_size = 0
-        max_size = settings.max_upload_size
-
-        with open(target_path, "wb") as f:
-            for chunk in response.iter_content(chunk_size=8192):
-                if chunk:
-                    f.write(chunk)
-                    downloaded_size += len(chunk)
-
-                    # Check size during download
-                    if downloaded_size > max_size:
-                        # Remove partial file
-                        f.close()
-                        os.remove(target_path)
+                # Check content length before downloading
+                content_length = response.headers.get("Content-Length")
+                if content_length:
+                    file_size = int(content_length)
+                    max_size = settings.max_upload_size
+                    if file_size > max_size:
                         raise HTTPException(
                             status_code=413,
-                            detail=f"File too large: exceeded {max_size} bytes during download",
+                            detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
                         )
 
+                # Generate unique filename
+                unique_id = str(uuid.uuid4())
+                if "." in safe_filename:
+                    file_extension = safe_filename.rsplit(".", 1)[1]
+                    target_filename = f"{unique_id}.{file_extension}"
+                else:
+                    target_filename = unique_id
+
+                target_path = os.path.join(settings.workdir, target_filename)
+
+                # Download file in chunks to handle large files
+                downloaded_size = 0
+                max_size = settings.max_upload_size
+
+                async with aiofiles.open(target_path, "wb") as f:
+                    async for chunk in response.aiter_bytes(chunk_size=8192):
+                        if chunk:
+                            await f.write(chunk)
+                            downloaded_size += len(chunk)
+
+                            # Check size during download
+                            if downloaded_size > max_size:
+                                # Remove partial file
+                                await f.close()
+                                os.remove(target_path)
+                                raise HTTPException(
+                                    status_code=413,
+                                    detail=f"File too large: exceeded {max_size} bytes during download",
+                                )
+
         logger.info(f"Downloaded file from URL '{url}' as '{target_filename}' ({downloaded_size} bytes)")
 
         # Enqueue for processing
@@ -227,19 +227,19 @@ async def process_url(request: Request, url_request: URLUploadRequest):
             "size": downloaded_size,
         }
 
-    except requests.exceptions.Timeout:
+    except httpx.TimeoutException:
         logger.error(f"Timeout while downloading file from URL: {url}")
         raise HTTPException(status_code=408, detail="Request timeout: server took too long to respond")
 
-    except requests.exceptions.ConnectionError as e:
+    except httpx.ConnectError as e:
         logger.error(f"Connection error while downloading file from URL: {url} - {str(e)}")
         raise HTTPException(status_code=502, detail=f"Failed to connect to URL: {str(e)}")
 
-    except requests.exceptions.HTTPError as e:
+    except httpx.HTTPStatusError as e:
         logger.error(f"HTTP error while downloading file from URL: {url} - {str(e)}")
         raise HTTPException(status_code=e.response.status_code, detail=f"HTTP error: {str(e)}")
 
-    except requests.exceptions.RequestException as e:
+    except httpx.RequestError as e:
         logger.error(f"Error downloading file from URL: {url} - {str(e)}")
         raise HTTPException(status_code=500, detail=f"Failed to download file: {str(e)}")