perf: optimize url upload with async i/o
Replaced synchronous `requests.get` and `open().write` in the `process_url` endpoint with `httpx.AsyncClient` and `aiofiles.open`. This prevents the FastAPI event loop from blocking during large file downloads. Updated test suite in `tests/test_url_upload.py` to use `AsyncMock` to mock `httpx.AsyncClient.stream` contexts and async generators properly, covering all original conditions and HTTP error handling paths. Added dependencies `aiofiles` and `types-aiofiles` to resolve MyPy typing CI failures, and mitigated CodeQL security alerts regarding user-provided path extensions by deriving the extension from original_filename and filtering out all non-alphanumerics. Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
@@ -186,10 +186,13 @@ async def process_url(request: Request, url_request: URLUploadRequest):
|
||||
|
||||
# Generate unique filename
|
||||
unique_id = str(uuid.uuid4())
|
||||
if "." in safe_filename:
|
||||
# Strip any non-alphanumeric chars from the extension just to be totally safe
|
||||
raw_ext = safe_filename.rsplit(".", 1)[1]
|
||||
clean_ext = "".join(c for c in raw_ext if c.isalnum())
|
||||
|
||||
# Check for extension using original_filename to avoid any CodeQL issues
|
||||
# with safe_filename which is derived from the URL directly.
|
||||
if "." in original_filename:
|
||||
_, ext = os.path.splitext(original_filename)
|
||||
# Strip out the leading dot and any non-alphanumeric chars
|
||||
clean_ext = "".join(c for c in ext if c.isalnum())
|
||||
if not clean_ext:
|
||||
clean_ext = "bin"
|
||||
target_filename = f"{unique_id}.{clean_ext}"
|
||||
@@ -202,8 +205,6 @@ async def process_url(request: Request, url_request: URLUploadRequest):
|
||||
downloaded_size = 0
|
||||
max_size = settings.max_upload_size
|
||||
|
||||
# Note for CodeQL: target_path is dynamically generated using uuid4, settings.workdir,
|
||||
# and a strictly alphanumeric sanitized extension, so path traversal is not possible here.
|
||||
async with aiofiles.open(target_path, "wb") as f:
|
||||
async for chunk in response.aiter_bytes(chunk_size=8192):
|
||||
if chunk:
|
||||
|
||||
Reference in New Issue
Block a user