fix: merge main, address code review feedback for security fix PR #816
- Merge origin/main into branch (resolve conflict in integrations_dashboard.html) - Add defensive JSON parsing with try/except for integration.config - Wrap tester() call in try/except to prevent 500 errors from bad config - Add i18n key integrations.connection_test_failed_fallback in en.json - Reference i18n key in template JS fallback message - Update SECURITY_AUDIT.md: add fix date (2026-03-23), update doc date - Remove accidental revert.sh file - Fix missing MagicMock/patch imports in test file - Add tests for invalid JSON config and tester exception error paths Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> Agent-Logs-Url: https://github.com/christianlouis/DocuElevate/sessions/daebb70e-059a-4601-8864-88eef49f99cf
This commit is contained in:
copilot-swe-agent[bot]
+70
-58
@@ -9,12 +9,14 @@ import urllib.parse
|
||||
import uuid
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
import aiofiles
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel, HttpUrl, field_validator
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.middleware.upload_rate_limit import require_upload_rate_limit
|
||||
from app.tasks.process_document import process_document
|
||||
from app.utils.allowed_types import ALLOWED_MIME_TYPES
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
@@ -106,7 +108,11 @@ def validate_file_type(content_type: str, filename: str) -> bool:
|
||||
|
||||
@router.post("/process-url")
|
||||
@require_login
|
||||
async def process_url(request: Request, url_request: URLUploadRequest):
|
||||
async def process_url(
|
||||
request: Request,
|
||||
url_request: URLUploadRequest,
|
||||
_rate_ok: None = Depends(require_upload_rate_limit),
|
||||
):
|
||||
"""
|
||||
Download a file from a URL and enqueue it for processing.
|
||||
|
||||
@@ -153,67 +159,73 @@ async def process_url(request: Request, url_request: URLUploadRequest):
|
||||
logger.info(f"Downloading file from URL: {url}")
|
||||
|
||||
# Use configured timeout to prevent hanging
|
||||
response = requests.get(
|
||||
url,
|
||||
async with httpx.AsyncClient(
|
||||
timeout=settings.http_request_timeout,
|
||||
stream=True, # Stream to handle large files
|
||||
allow_redirects=True, # Follow redirects
|
||||
follow_redirects=True,
|
||||
headers={
|
||||
"User-Agent": "DocuElevate/1.0", # Identify ourselves
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
) as client:
|
||||
async with client.stream("GET", url) as response:
|
||||
response.raise_for_status()
|
||||
|
||||
# Validate content type
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if not validate_file_type(content_type, safe_filename):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unsupported file type: {content_type}. "
|
||||
"Supported types: PDF, Office documents, images, plain text",
|
||||
)
|
||||
# Validate content type
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if not validate_file_type(content_type, safe_filename):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unsupported file type: {content_type}. "
|
||||
"Supported types: PDF, Office documents, images, plain text",
|
||||
)
|
||||
|
||||
# Check content length before downloading
|
||||
content_length = response.headers.get("Content-Length")
|
||||
if content_length:
|
||||
file_size = int(content_length)
|
||||
max_size = settings.max_upload_size
|
||||
if file_size > max_size:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
|
||||
)
|
||||
|
||||
# Generate unique filename
|
||||
unique_id = str(uuid.uuid4())
|
||||
if "." in safe_filename:
|
||||
file_extension = safe_filename.rsplit(".", 1)[1]
|
||||
target_filename = f"{unique_id}.{file_extension}"
|
||||
else:
|
||||
target_filename = unique_id
|
||||
|
||||
target_path = os.path.join(settings.workdir, target_filename)
|
||||
|
||||
# Download file in chunks to handle large files
|
||||
downloaded_size = 0
|
||||
max_size = settings.max_upload_size
|
||||
|
||||
with open(target_path, "wb") as f:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
downloaded_size += len(chunk)
|
||||
|
||||
# Check size during download
|
||||
if downloaded_size > max_size:
|
||||
# Remove partial file
|
||||
f.close()
|
||||
os.remove(target_path)
|
||||
# Check content length before downloading
|
||||
content_length = response.headers.get("Content-Length")
|
||||
if content_length:
|
||||
file_size = int(content_length)
|
||||
max_size = settings.max_upload_size
|
||||
if file_size > max_size:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: exceeded {max_size} bytes during download",
|
||||
detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
|
||||
)
|
||||
|
||||
# Generate unique filename
|
||||
unique_id = str(uuid.uuid4())
|
||||
|
||||
# Check for extension using original_filename to avoid any CodeQL issues
|
||||
# with safe_filename which is derived from the URL directly.
|
||||
if "." in original_filename:
|
||||
_, ext = os.path.splitext(original_filename)
|
||||
# Strip out the leading dot and any non-alphanumeric chars
|
||||
clean_ext = "".join(c for c in ext if c.isalnum())
|
||||
if not clean_ext:
|
||||
clean_ext = "bin"
|
||||
target_filename = f"{unique_id}.{clean_ext}"
|
||||
else:
|
||||
target_filename = unique_id
|
||||
|
||||
target_path = os.path.join(settings.workdir, target_filename)
|
||||
|
||||
# Download file in chunks to handle large files
|
||||
downloaded_size = 0
|
||||
max_size = settings.max_upload_size
|
||||
|
||||
async with aiofiles.open(target_path, "wb") as f:
|
||||
async for chunk in response.aiter_bytes(chunk_size=8192):
|
||||
if chunk:
|
||||
await f.write(chunk)
|
||||
downloaded_size += len(chunk)
|
||||
|
||||
# Check size during download
|
||||
if downloaded_size > max_size:
|
||||
# Remove partial file
|
||||
await f.close()
|
||||
os.remove(target_path)
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: exceeded {max_size} bytes during download",
|
||||
)
|
||||
|
||||
logger.info(f"Downloaded file from URL '{url}' as '{target_filename}' ({downloaded_size} bytes)")
|
||||
|
||||
# Enqueue for processing
|
||||
@@ -227,19 +239,19 @@ async def process_url(request: Request, url_request: URLUploadRequest):
|
||||
"size": downloaded_size,
|
||||
}
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
except httpx.TimeoutException:
|
||||
logger.error(f"Timeout while downloading file from URL: {url}")
|
||||
raise HTTPException(status_code=408, detail="Request timeout: server took too long to respond")
|
||||
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
except httpx.ConnectError as e:
|
||||
logger.error(f"Connection error while downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=502, detail=f"Failed to connect to URL: {str(e)}")
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.error(f"HTTP error while downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=e.response.status_code, detail=f"HTTP error: {str(e)}")
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
except httpx.RequestError as e:
|
||||
logger.error(f"Error downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to download file: {str(e)}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user