feat(security): add request size limits to API endpoints
- Add RequestSizeLimitMiddleware that checks Content-Length header before request body is read: non-multipart requests capped at MAX_REQUEST_BODY_SIZE (default 1 MB), multipart uploads capped at MAX_UPLOAD_SIZE (default 1 GB). Returns HTTP 413 on violation. - Register middleware in app/main.py - Add max_request_body_size setting to app/config.py - Fix ui_upload in files.py to check Content-Length early and read in 64 KB chunks (bounded memory usage), removing the post-write os.path.getsize check - Document MAX_REQUEST_BODY_SIZE in .env.demo and ConfigurationGuide.md - Mark SECURITY_AUDIT.md item #4 as resolved - Add 9 tests in test_request_size_limit.py - Update test_upload_file_too_large to use patch.object instead of the now-unused os.path.getsize mock Closes #173 Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
+313
-95
@@ -45,7 +45,8 @@ def list_files_api(
|
||||
page: int = Query(1, ge=1, description="Page number"),
|
||||
per_page: int = Query(50, ge=1, le=200, description="Items per page"),
|
||||
sort_by: str = Query(
|
||||
"created_at", description="Sort field: id, original_filename, file_size, mime_type, created_at, status"
|
||||
"created_at",
|
||||
description="Sort field: id, original_filename, file_size, mime_type, created_at, status",
|
||||
),
|
||||
sort_order: str = Query("desc", description="Sort order: asc or desc"),
|
||||
search: Optional[str] = Query(None, description="Search in filename"),
|
||||
@@ -128,7 +129,13 @@ def list_files_api(
|
||||
"mime_type": f.mime_type,
|
||||
"created_at": f.created_at.isoformat() if f.created_at else None,
|
||||
"processing_status": statuses.get(
|
||||
f.id, {"status": "pending", "last_step": None, "has_errors": False, "total_steps": 0}
|
||||
f.id,
|
||||
{
|
||||
"status": "pending",
|
||||
"last_step": None,
|
||||
"has_errors": False,
|
||||
"total_steps": 0,
|
||||
},
|
||||
),
|
||||
}
|
||||
)
|
||||
@@ -138,7 +145,12 @@ def list_files_api(
|
||||
|
||||
return {
|
||||
"files": result,
|
||||
"pagination": {"page": page, "per_page": per_page, "total_items": total_items, "total_pages": total_pages},
|
||||
"pagination": {
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"total_items": total_items,
|
||||
"total_pages": total_pages,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -162,11 +174,16 @@ def get_file_details(request: Request, file_id: int, db: DbSession):
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File record with ID {file_id} not found")
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"File record with ID {file_id} not found"
|
||||
)
|
||||
|
||||
# Get processing logs
|
||||
logs = (
|
||||
db.query(ProcessingLog).filter(ProcessingLog.file_id == file_id).order_by(ProcessingLog.timestamp.desc()).all()
|
||||
db.query(ProcessingLog)
|
||||
.filter(ProcessingLog.file_id == file_id)
|
||||
.order_by(ProcessingLog.timestamp.desc())
|
||||
.all()
|
||||
)
|
||||
|
||||
# Build log list
|
||||
@@ -187,7 +204,13 @@ def get_file_details(request: Request, file_id: int, db: DbSession):
|
||||
processing_status = _get_file_processing_status(db, file_id)
|
||||
|
||||
# Check if files exist on disk
|
||||
files_on_disk = {"original": os.path.exists(file_record.local_filename) if file_record.local_filename else False}
|
||||
files_on_disk = {
|
||||
"original": (
|
||||
os.path.exists(file_record.local_filename)
|
||||
if file_record.local_filename
|
||||
else False
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
"file": {
|
||||
@@ -197,7 +220,9 @@ def get_file_details(request: Request, file_id: int, db: DbSession):
|
||||
"local_filename": file_record.local_filename,
|
||||
"file_size": file_record.file_size,
|
||||
"mime_type": file_record.mime_type,
|
||||
"created_at": file_record.created_at.isoformat() if file_record.created_at else None,
|
||||
"created_at": (
|
||||
file_record.created_at.isoformat() if file_record.created_at else None
|
||||
),
|
||||
},
|
||||
"processing_status": processing_status,
|
||||
"logs": log_list,
|
||||
@@ -214,30 +239,41 @@ def delete_file_record(request: Request, file_id: int, db: DbSession):
|
||||
"""
|
||||
# Check if file deletion is allowed
|
||||
if not settings.allow_file_delete:
|
||||
raise HTTPException(status_code=403, detail="File deletion is disabled in the configuration")
|
||||
raise HTTPException(
|
||||
status_code=403, detail="File deletion is disabled in the configuration"
|
||||
)
|
||||
|
||||
try:
|
||||
# Find the file record
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File record with ID {file_id} not found")
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"File record with ID {file_id} not found"
|
||||
)
|
||||
|
||||
# Log the deletion
|
||||
logger.info(f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}")
|
||||
logger.info(
|
||||
f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}"
|
||||
)
|
||||
|
||||
# Delete the record
|
||||
db.delete(file_record)
|
||||
db.commit()
|
||||
|
||||
return {"status": "success", "message": f"File record {file_id} deleted successfully"}
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"File record {file_id} deleted successfully",
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.exception(f"Error deleting file record {file_id}: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Error deleting file record: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Error deleting file record: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/files/bulk-delete")
|
||||
@@ -249,14 +285,18 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: DbSession):
|
||||
"""
|
||||
# Check if file deletion is allowed
|
||||
if not settings.allow_file_delete:
|
||||
raise HTTPException(status_code=403, detail="File deletion is disabled in the configuration")
|
||||
raise HTTPException(
|
||||
status_code=403, detail="File deletion is disabled in the configuration"
|
||||
)
|
||||
|
||||
try:
|
||||
# Find all file records
|
||||
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
|
||||
|
||||
if not file_records:
|
||||
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
|
||||
raise HTTPException(
|
||||
status_code=404, detail="No files found with the provided IDs"
|
||||
)
|
||||
|
||||
deleted_count = len(file_records)
|
||||
deleted_ids = [f.id for f in file_records]
|
||||
@@ -281,7 +321,9 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: DbSession):
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.exception(f"Error bulk deleting file records: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Error bulk deleting file records: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Error bulk deleting file records: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/files/bulk-reprocess")
|
||||
@@ -295,7 +337,9 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
|
||||
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
|
||||
|
||||
if not file_records:
|
||||
raise HTTPException(status_code=404, detail="No files found with the provided IDs")
|
||||
raise HTTPException(
|
||||
status_code=404, detail="No files found with the provided IDs"
|
||||
)
|
||||
|
||||
task_ids = []
|
||||
processed_files = []
|
||||
@@ -304,7 +348,9 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
|
||||
for file_record in file_records:
|
||||
try:
|
||||
# Check if local file exists
|
||||
if not file_record.local_filename or not os.path.exists(file_record.local_filename):
|
||||
if not file_record.local_filename or not os.path.exists(
|
||||
file_record.local_filename
|
||||
):
|
||||
errors.append(
|
||||
{
|
||||
"file_id": file_record.id,
|
||||
@@ -315,10 +361,16 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
|
||||
continue
|
||||
|
||||
# Queue the file for processing, passing file_id to skip duplicate check
|
||||
task = process_document.delay(file_record.local_filename, file_id=file_record.id)
|
||||
task = process_document.delay(
|
||||
file_record.local_filename, file_id=file_record.id
|
||||
)
|
||||
task_ids.append(task.id)
|
||||
processed_files.append(
|
||||
{"file_id": file_record.id, "filename": file_record.original_filename, "task_id": task.id}
|
||||
{
|
||||
"file_id": file_record.id,
|
||||
"filename": file_record.original_filename,
|
||||
"task_id": task.id,
|
||||
}
|
||||
)
|
||||
|
||||
logger.info(
|
||||
@@ -328,7 +380,13 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Error reprocessing file {file_record.id}: {str(e)}")
|
||||
errors.append({"file_id": file_record.id, "filename": file_record.original_filename, "error": str(e)})
|
||||
errors.append(
|
||||
{
|
||||
"file_id": file_record.id,
|
||||
"filename": file_record.original_filename,
|
||||
"error": str(e),
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success" if processed_files else "error",
|
||||
@@ -342,7 +400,9 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Error bulk reprocessing files: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Error bulk reprocessing files: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/files/{file_id}/reprocess")
|
||||
@@ -362,15 +422,24 @@ def reprocess_single_file(request: Request, file_id: int, db: DbSession):
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"File with ID {file_id} not found"
|
||||
)
|
||||
|
||||
# Check if local file exists
|
||||
if not file_record.local_filename or not os.path.exists(file_record.local_filename):
|
||||
raise HTTPException(status_code=400, detail="Local file not found on disk. Cannot reprocess.")
|
||||
if not file_record.local_filename or not os.path.exists(
|
||||
file_record.local_filename
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Local file not found on disk. Cannot reprocess.",
|
||||
)
|
||||
|
||||
# Queue the file for processing, passing file_id to skip duplicate check
|
||||
task = process_document.delay(
|
||||
file_record.local_filename, original_filename=file_record.original_filename, file_id=file_record.id
|
||||
file_record.local_filename,
|
||||
original_filename=file_record.original_filename,
|
||||
file_id=file_record.id,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
@@ -389,7 +458,9 @@ def reprocess_single_file(request: Request, file_id: int, db: DbSession):
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Error reprocessing file {file_id}: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Error reprocessing file: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Error reprocessing file: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/files/{file_id}/reprocess-with-cloud-ocr")
|
||||
@@ -413,24 +484,34 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession):
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"File with ID {file_id} not found"
|
||||
)
|
||||
|
||||
# Prefer using the original_file_path if available, otherwise fall back to local_filename
|
||||
source_file = None
|
||||
if file_record.original_file_path and os.path.exists(file_record.original_file_path):
|
||||
if file_record.original_file_path and os.path.exists(
|
||||
file_record.original_file_path
|
||||
):
|
||||
source_file = file_record.original_file_path
|
||||
logger.info(f"Using original file for Cloud OCR reprocessing: {source_file}")
|
||||
logger.info(
|
||||
f"Using original file for Cloud OCR reprocessing: {source_file}"
|
||||
)
|
||||
elif file_record.local_filename and os.path.exists(file_record.local_filename):
|
||||
source_file = file_record.local_filename
|
||||
logger.info(f"Using local file for Cloud OCR reprocessing: {source_file}")
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Neither original nor local file found on disk. Cannot reprocess."
|
||||
status_code=400,
|
||||
detail="Neither original nor local file found on disk. Cannot reprocess.",
|
||||
)
|
||||
|
||||
# Queue the file for processing with force_cloud_ocr=True
|
||||
task = process_document.delay(
|
||||
source_file, original_filename=file_record.original_filename, file_id=file_record.id, force_cloud_ocr=True
|
||||
source_file,
|
||||
original_filename=file_record.original_filename,
|
||||
file_id=file_record.id,
|
||||
force_cloud_ocr=True,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
@@ -451,7 +532,9 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession):
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Error reprocessing file {file_id} with Cloud OCR: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Error reprocessing file with Cloud OCR: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Error reprocessing file with Cloud OCR: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
def _extract_text_from_pdf(file_path: str) -> str:
|
||||
@@ -496,24 +579,40 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
|
||||
|
||||
if step_name == "process_document":
|
||||
# Full reprocessing with duplicate check bypass
|
||||
logger.info(f"Retrying process_document for file {file_id}: local_filename={file_record.local_filename!r}")
|
||||
logger.info(
|
||||
f"Retrying process_document for file {file_id}: local_filename={file_record.local_filename!r}"
|
||||
)
|
||||
if not file_record.local_filename:
|
||||
logger.error(f"process_document retry failed for file {file_id}: local_filename is None")
|
||||
raise HTTPException(status_code=400, detail="Local file path is None. Cannot retry.")
|
||||
logger.error(
|
||||
f"process_document retry failed for file {file_id}: local_filename is None"
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Local file path is None. Cannot retry."
|
||||
)
|
||||
|
||||
exists = os.path.exists(file_record.local_filename)
|
||||
logger.info(f"Checking local_filename: {file_record.local_filename!r}, exists={exists}")
|
||||
logger.info(
|
||||
f"Checking local_filename: {file_record.local_filename!r}, exists={exists}"
|
||||
)
|
||||
if not exists:
|
||||
error_message = f"Local file not found on disk. Cannot retry. Path checked: local_filename={file_record.local_filename!r} (exists=False)"
|
||||
logger.error(f"process_document retry failed for file {file_id}: {error_message}")
|
||||
logger.error(
|
||||
f"process_document retry failed for file {file_id}: {error_message}"
|
||||
)
|
||||
raise HTTPException(status_code=400, detail=error_message)
|
||||
|
||||
logger.info(f"Found file for process_document retry at: {file_record.local_filename!r}")
|
||||
logger.info(
|
||||
f"Found file for process_document retry at: {file_record.local_filename!r}"
|
||||
)
|
||||
task = process_document.delay(
|
||||
file_record.local_filename, original_filename=file_record.original_filename, file_id=file_id
|
||||
file_record.local_filename,
|
||||
original_filename=file_record.original_filename,
|
||||
file_id=file_id,
|
||||
)
|
||||
elif step_name == "process_with_azure_document_intelligence":
|
||||
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence
|
||||
from app.tasks.process_with_azure_document_intelligence import (
|
||||
process_with_azure_document_intelligence,
|
||||
)
|
||||
|
||||
# OCR needs the file in workdir/tmp
|
||||
logger.info(
|
||||
@@ -522,10 +621,14 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
|
||||
)
|
||||
if not file_record.local_filename:
|
||||
logger.error(f"OCR retry failed for file {file_id}: local_filename is None")
|
||||
raise HTTPException(status_code=400, detail="Local file path is None. Cannot retry OCR.")
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Local file path is None. Cannot retry OCR."
|
||||
)
|
||||
|
||||
exists = os.path.exists(file_record.local_filename)
|
||||
logger.info(f"Checking local_filename: {file_record.local_filename!r}, exists={exists}")
|
||||
logger.info(
|
||||
f"Checking local_filename: {file_record.local_filename!r}, exists={exists}"
|
||||
)
|
||||
if not exists:
|
||||
error_message = f"Local file not found on disk. Cannot retry OCR. Path checked: local_filename={file_record.local_filename!r} (exists=False)"
|
||||
logger.error(f"OCR retry failed for file {file_id}: {error_message}")
|
||||
@@ -541,22 +644,35 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
|
||||
f"Retrying extract_metadata_with_gpt for file {file_id}: local_filename={file_record.local_filename!r}"
|
||||
)
|
||||
if not file_record.local_filename:
|
||||
logger.error(f"Metadata extraction retry failed for file {file_id}: local_filename is None")
|
||||
raise HTTPException(status_code=400, detail="Local file path is None. Cannot retry metadata extraction.")
|
||||
logger.error(
|
||||
f"Metadata extraction retry failed for file {file_id}: local_filename is None"
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Local file path is None. Cannot retry metadata extraction.",
|
||||
)
|
||||
|
||||
exists = os.path.exists(file_record.local_filename)
|
||||
logger.info(f"Checking local_filename: {file_record.local_filename!r}, exists={exists}")
|
||||
logger.info(
|
||||
f"Checking local_filename: {file_record.local_filename!r}, exists={exists}"
|
||||
)
|
||||
if not exists:
|
||||
error_message = f"Local file not found on disk. Cannot retry metadata extraction. Path checked: local_filename={file_record.local_filename!r} (exists=False)"
|
||||
logger.error(f"Metadata extraction retry failed for file {file_id}: {error_message}")
|
||||
logger.error(
|
||||
f"Metadata extraction retry failed for file {file_id}: {error_message}"
|
||||
)
|
||||
raise HTTPException(status_code=400, detail=error_message)
|
||||
|
||||
logger.info(f"Found file for metadata extraction retry at: {file_record.local_filename!r}")
|
||||
logger.info(
|
||||
f"Found file for metadata extraction retry at: {file_record.local_filename!r}"
|
||||
)
|
||||
extracted_text = _extract_text_from_pdf(file_record.local_filename)
|
||||
filename = os.path.basename(file_record.local_filename)
|
||||
task = extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
|
||||
elif step_name == "embed_metadata_into_pdf":
|
||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt as extract_metadata_task
|
||||
from app.tasks.extract_metadata_with_gpt import (
|
||||
extract_metadata_with_gpt as extract_metadata_task,
|
||||
)
|
||||
|
||||
# Retrying embed requires re-running metadata extraction first, because
|
||||
# embed_metadata_into_pdf needs the actual metadata dict (not empty).
|
||||
@@ -581,24 +697,36 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
|
||||
# Check 1: local_filename (original tmp location)
|
||||
if file_record.local_filename:
|
||||
exists = os.path.exists(file_record.local_filename)
|
||||
checked_paths.append(f"local_filename={file_record.local_filename!r} (exists={exists})")
|
||||
logger.info(f"Checking local_filename: {file_record.local_filename!r}, exists={exists}")
|
||||
checked_paths.append(
|
||||
f"local_filename={file_record.local_filename!r} (exists={exists})"
|
||||
)
|
||||
logger.info(
|
||||
f"Checking local_filename: {file_record.local_filename!r}, exists={exists}"
|
||||
)
|
||||
if exists:
|
||||
file_path = file_record.local_filename
|
||||
|
||||
# Check 2: processed_file_path
|
||||
if not file_path and file_record.processed_file_path:
|
||||
exists = os.path.exists(file_record.processed_file_path)
|
||||
checked_paths.append(f"processed_file_path={file_record.processed_file_path!r} (exists={exists})")
|
||||
logger.info(f"Checking processed_file_path: {file_record.processed_file_path!r}, exists={exists}")
|
||||
checked_paths.append(
|
||||
f"processed_file_path={file_record.processed_file_path!r} (exists={exists})"
|
||||
)
|
||||
logger.info(
|
||||
f"Checking processed_file_path: {file_record.processed_file_path!r}, exists={exists}"
|
||||
)
|
||||
if exists:
|
||||
file_path = file_record.processed_file_path
|
||||
|
||||
# Check 3: original_file_path (immutable copy in workdir/original/)
|
||||
if not file_path and file_record.original_file_path:
|
||||
exists = os.path.exists(file_record.original_file_path)
|
||||
checked_paths.append(f"original_file_path={file_record.original_file_path!r} (exists={exists})")
|
||||
logger.info(f"Checking original_file_path: {file_record.original_file_path!r}, exists={exists}")
|
||||
checked_paths.append(
|
||||
f"original_file_path={file_record.original_file_path!r} (exists={exists})"
|
||||
)
|
||||
logger.info(
|
||||
f"Checking original_file_path: {file_record.original_file_path!r}, exists={exists}"
|
||||
)
|
||||
if exists:
|
||||
file_path = file_record.original_file_path
|
||||
|
||||
@@ -606,17 +734,25 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
|
||||
if not file_path and file_record.local_filename:
|
||||
workdir = settings.workdir
|
||||
tmp_dir = os.path.join(workdir, "tmp")
|
||||
fallback_path = os.path.join(tmp_dir, os.path.basename(file_record.local_filename))
|
||||
fallback_path = os.path.join(
|
||||
tmp_dir, os.path.basename(file_record.local_filename)
|
||||
)
|
||||
exists = os.path.exists(fallback_path)
|
||||
checked_paths.append(f"workdir_tmp_fallback={fallback_path!r} (exists={exists})")
|
||||
logger.info(f"Checking workdir/tmp fallback: {fallback_path!r}, exists={exists}")
|
||||
checked_paths.append(
|
||||
f"workdir_tmp_fallback={fallback_path!r} (exists={exists})"
|
||||
)
|
||||
logger.info(
|
||||
f"Checking workdir/tmp fallback: {fallback_path!r}, exists={exists}"
|
||||
)
|
||||
if exists:
|
||||
file_path = fallback_path
|
||||
|
||||
if not file_path:
|
||||
paths_detail = "; ".join(checked_paths)
|
||||
error_message = f"File not found on disk. Cannot retry metadata embedding. Paths checked: {paths_detail}"
|
||||
logger.error(f"embed_metadata_into_pdf retry failed for file {file_id}: {error_message}")
|
||||
logger.error(
|
||||
f"embed_metadata_into_pdf retry failed for file {file_id}: {error_message}"
|
||||
)
|
||||
raise HTTPException(status_code=400, detail=error_message)
|
||||
|
||||
logger.info(f"Found file for embed_metadata_into_pdf retry at: {file_path!r}")
|
||||
@@ -624,9 +760,13 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
|
||||
# Pass the full path to the task so it can locate the file
|
||||
task = extract_metadata_task.delay(file_path, extracted_text, file_id)
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail=f"Unsupported pipeline step: {step_name}")
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Unsupported pipeline step: {step_name}"
|
||||
)
|
||||
|
||||
logger.info(f"Retrying pipeline step: FileID={file_record.id}, Step={step_name}, TaskID={task.id}")
|
||||
logger.info(
|
||||
f"Retrying pipeline step: FileID={file_record.id}, Step={step_name}, TaskID={task.id}"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
@@ -644,7 +784,8 @@ def retry_subtask(
|
||||
file_id: int,
|
||||
db: DbSession,
|
||||
subtask_name: str = Query(
|
||||
..., description="Name of the subtask to retry (e.g., 'upload_to_dropbox', 'extract_metadata_with_gpt')"
|
||||
...,
|
||||
description="Name of the subtask to retry (e.g., 'upload_to_dropbox', 'extract_metadata_with_gpt')",
|
||||
),
|
||||
):
|
||||
"""
|
||||
@@ -666,7 +807,9 @@ def retry_subtask(
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"File with ID {file_id} not found"
|
||||
)
|
||||
|
||||
# Pipeline processing steps that can be retried from the failed step
|
||||
pipeline_step_names = {
|
||||
@@ -749,7 +892,9 @@ def retry_subtask(
|
||||
upload_task = task_map[subtask_name]
|
||||
task = upload_task.delay(file_path, file_id)
|
||||
|
||||
logger.info(f"Retrying upload subtask: FileID={file_record.id}, Subtask={subtask_name}, TaskID={task.id}")
|
||||
logger.info(
|
||||
f"Retrying upload subtask: FileID={file_record.id}, Subtask={subtask_name}, TaskID={task.id}"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
@@ -762,7 +907,9 @@ def retry_subtask(
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Error retrying subtask {subtask_name} for file {file_id}: {str(e)}")
|
||||
logger.exception(
|
||||
f"Error retrying subtask {subtask_name} for file {file_id}: {str(e)}"
|
||||
)
|
||||
raise HTTPException(status_code=500, detail=f"Error retrying subtask: {str(e)}")
|
||||
|
||||
|
||||
@@ -791,12 +938,18 @@ def get_file_preview(
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"File with ID {file_id} not found"
|
||||
)
|
||||
|
||||
if version == "original":
|
||||
# Return the original file from tmp
|
||||
if not file_record.local_filename or not os.path.exists(file_record.local_filename):
|
||||
raise HTTPException(status_code=404, detail="Original file not found on disk")
|
||||
if not file_record.local_filename or not os.path.exists(
|
||||
file_record.local_filename
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=404, detail="Original file not found on disk"
|
||||
)
|
||||
|
||||
file_path = file_record.local_filename
|
||||
|
||||
@@ -822,20 +975,27 @@ def get_file_preview(
|
||||
if not file_path:
|
||||
raise HTTPException(status_code=404, detail="Processed file not found")
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="Invalid version parameter. Use 'original' or 'processed'")
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Invalid version parameter. Use 'original' or 'processed'",
|
||||
)
|
||||
|
||||
# Return the file
|
||||
return FileResponse(
|
||||
path=file_path,
|
||||
media_type=file_record.mime_type or "application/pdf",
|
||||
headers={"Content-Disposition": f'inline; filename="{file_record.original_filename}"'},
|
||||
headers={
|
||||
"Content-Disposition": f'inline; filename="{file_record.original_filename}"'
|
||||
},
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Error retrieving file preview: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Error retrieving file preview: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Error retrieving file preview: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/files/{file_id}/download")
|
||||
@@ -863,12 +1023,18 @@ def download_file(
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"File with ID {file_id} not found"
|
||||
)
|
||||
|
||||
if version == "original":
|
||||
# Return the original file from tmp
|
||||
if not file_record.local_filename or not os.path.exists(file_record.local_filename):
|
||||
raise HTTPException(status_code=404, detail="Original file not found on disk")
|
||||
if not file_record.local_filename or not os.path.exists(
|
||||
file_record.local_filename
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=404, detail="Original file not found on disk"
|
||||
)
|
||||
|
||||
file_path = file_record.local_filename
|
||||
|
||||
@@ -894,13 +1060,18 @@ def download_file(
|
||||
if not file_path:
|
||||
raise HTTPException(status_code=404, detail="Processed file not found")
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="Invalid version parameter. Use 'original' or 'processed'")
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Invalid version parameter. Use 'original' or 'processed'",
|
||||
)
|
||||
|
||||
# Return the file with attachment disposition to trigger download
|
||||
return FileResponse(
|
||||
path=file_path,
|
||||
media_type=file_record.mime_type or "application/pdf",
|
||||
headers={"Content-Disposition": f'attachment; filename="{file_record.original_filename}"'},
|
||||
headers={
|
||||
"Content-Disposition": f'attachment; filename="{file_record.original_filename}"'
|
||||
},
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
@@ -916,6 +1087,21 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
|
||||
"""Endpoint to accept a user-uploaded file and enqueue it for processing."""
|
||||
workdir = settings.workdir
|
||||
|
||||
# Early size check: reject before reading the body if Content-Length is known
|
||||
max_size = settings.max_upload_size
|
||||
content_length_header = request.headers.get("content-length")
|
||||
if content_length_header is not None:
|
||||
try:
|
||||
declared_size = int(content_length_header)
|
||||
if declared_size > max_size:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: declared size {declared_size} bytes exceeds maximum "
|
||||
f"{max_size} bytes. See SECURITY_AUDIT.md for configuration details.",
|
||||
)
|
||||
except ValueError:
|
||||
pass # Malformed header; proceed and check actual size after reading
|
||||
|
||||
# Extract just the filename without any path components to prevent path traversal
|
||||
# First, use basename to remove any directory components
|
||||
base_filename = os.path.basename(file.filename)
|
||||
@@ -934,27 +1120,37 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
|
||||
# Store both the safe original name and the unique name
|
||||
target_path = os.path.join(workdir, target_filename)
|
||||
|
||||
# Read file in chunks to avoid loading the entire body into memory at once,
|
||||
# enforcing the size limit during the read so memory usage stays bounded.
|
||||
try:
|
||||
written_size = 0
|
||||
with open(target_path, "wb") as f:
|
||||
content = await file.read()
|
||||
f.write(content)
|
||||
chunk_size = 65536 # 64 KB chunks
|
||||
while True:
|
||||
chunk = await file.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
written_size += len(chunk)
|
||||
if written_size > max_size:
|
||||
# Exceeded limit mid-stream; clean up and reject
|
||||
f.close()
|
||||
os.remove(target_path)
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: exceeded {max_size} bytes during upload. "
|
||||
f"See SECURITY_AUDIT.md for configuration details.",
|
||||
)
|
||||
f.write(chunk)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
if os.path.exists(target_path):
|
||||
os.remove(target_path)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to save file: {e}")
|
||||
|
||||
# Log the mapping between original and safe filename
|
||||
logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")
|
||||
|
||||
# Check file size against configured maximum
|
||||
file_size = os.path.getsize(target_path)
|
||||
max_size = settings.max_upload_size
|
||||
if file_size > max_size:
|
||||
# Remove the file if it's too large
|
||||
os.remove(target_path)
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: {file_size} bytes (max {max_size} bytes). "
|
||||
f"See SECURITY_AUDIT.md for configuration details.",
|
||||
)
|
||||
file_size = written_size
|
||||
|
||||
# Same set of allowed file types as in the IMAP task
|
||||
ALLOWED_MIME_TYPES = {
|
||||
@@ -993,7 +1189,9 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
|
||||
# Check if file splitting is needed (only for PDFs)
|
||||
from app.utils.file_splitting import should_split_file
|
||||
|
||||
should_split = is_pdf and should_split_file(target_path, settings.max_single_file_size)
|
||||
should_split = is_pdf and should_split_file(
|
||||
target_path, settings.max_single_file_size
|
||||
)
|
||||
|
||||
if should_split:
|
||||
# File needs to be split before processing
|
||||
@@ -1011,7 +1209,9 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
|
||||
task_ids = []
|
||||
for split_file in split_files:
|
||||
split_filename = os.path.basename(split_file)
|
||||
task = process_document.delay(split_file, original_filename=split_filename)
|
||||
task = process_document.delay(
|
||||
split_file, original_filename=split_filename
|
||||
)
|
||||
task_ids.append(task.id)
|
||||
logger.info(f"Enqueued split PDF part for processing: {split_file}")
|
||||
|
||||
@@ -1029,7 +1229,9 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
|
||||
except Exception as e:
|
||||
logger.exception(f"Failed to split file {target_path}: {str(e)}")
|
||||
# Fall back to processing the whole file
|
||||
logger.warning(f"Falling back to processing whole file due to split error: {str(e)}")
|
||||
logger.warning(
|
||||
f"Falling back to processing whole file due to split error: {str(e)}"
|
||||
)
|
||||
should_split = False
|
||||
|
||||
if is_pdf and not should_split:
|
||||
@@ -1037,21 +1239,37 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
|
||||
task = process_document.delay(target_path, original_filename=safe_filename)
|
||||
logger.info(f"Enqueued PDF for processing: {target_path}")
|
||||
elif mime_type in IMAGE_MIME_TYPES or any(
|
||||
file_ext.endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"]
|
||||
file_ext.endswith(ext)
|
||||
for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"]
|
||||
):
|
||||
# If it's an image, convert to PDF first
|
||||
task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
|
||||
logger.info(f"Enqueued image for PDF conversion: {target_path}")
|
||||
elif mime_type in ALLOWED_MIME_TYPES or any(
|
||||
file_ext.endswith(ext)
|
||||
for ext in [".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".odt", ".ods", ".odp", ".rtf", ".txt", ".csv"]
|
||||
for ext in [
|
||||
".doc",
|
||||
".docx",
|
||||
".xls",
|
||||
".xlsx",
|
||||
".ppt",
|
||||
".pptx",
|
||||
".odt",
|
||||
".ods",
|
||||
".odp",
|
||||
".rtf",
|
||||
".txt",
|
||||
".csv",
|
||||
]
|
||||
):
|
||||
# If it's an office document, convert to PDF first
|
||||
task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
|
||||
logger.info(f"Enqueued office document for PDF conversion: {target_path}")
|
||||
else:
|
||||
# For any other file type, attempt conversion but log a warning
|
||||
logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
|
||||
logger.warning(
|
||||
f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion"
|
||||
)
|
||||
task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
|
||||
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user