gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: apply ruff auto-fix

Browse Source 
- Auto-formatted code with ruff format
- Applied ruff linting fixes with --fix

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
github-actions[bot]
2026-02-22 15:25:48 +00:00
parent ca27a0b687
commit 77e777418c
5 changed files with 112 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/files.py
+63 -193
View File
@@ -174,16 +174,11 @@ def get_file_details(request: Request, file_id: int, db: DbSession):
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record: if not file_record:
raise HTTPException( raise HTTPException(status_code=404, detail=f"File record with ID {file_id} not found")
status_code=404, detail=f"File record with ID {file_id} not found"
)
# Get processing logs # Get processing logs
logs = ( logs = (
db.query(ProcessingLog) db.query(ProcessingLog).filter(ProcessingLog.file_id == file_id).order_by(ProcessingLog.timestamp.desc()).all()
.filter(ProcessingLog.file_id == file_id)
.order_by(ProcessingLog.timestamp.desc())
.all()
) )
# Build log list # Build log list
@@ -204,13 +199,7 @@ def get_file_details(request: Request, file_id: int, db: DbSession):
processing_status = _get_file_processing_status(db, file_id) processing_status = _get_file_processing_status(db, file_id)
# Check if files exist on disk # Check if files exist on disk
files_on_disk = { files_on_disk = {"original": (os.path.exists(file_record.local_filename) if file_record.local_filename else False)}
"original": (
os.path.exists(file_record.local_filename)
if file_record.local_filename
else False
)
}
return { return {
"file": { "file": {
@@ -220,9 +209,7 @@ def get_file_details(request: Request, file_id: int, db: DbSession):
"local_filename": file_record.local_filename, "local_filename": file_record.local_filename,
"file_size": file_record.file_size, "file_size": file_record.file_size,
"mime_type": file_record.mime_type, "mime_type": file_record.mime_type,
"created_at": ( "created_at": (file_record.created_at.isoformat() if file_record.created_at else None),
file_record.created_at.isoformat() if file_record.created_at else None
),
}, },
"processing_status": processing_status, "processing_status": processing_status,
"logs": log_list, "logs": log_list,
@@ -239,23 +226,17 @@ def delete_file_record(request: Request, file_id: int, db: DbSession):
""" """
# Check if file deletion is allowed # Check if file deletion is allowed
if not settings.allow_file_delete: if not settings.allow_file_delete:
raise HTTPException( raise HTTPException(status_code=403, detail="File deletion is disabled in the configuration")
status_code=403, detail="File deletion is disabled in the configuration"
)
try: try:
# Find the file record # Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record: if not file_record:
raise HTTPException( raise HTTPException(status_code=404, detail=f"File record with ID {file_id} not found")
status_code=404, detail=f"File record with ID {file_id} not found"
)
# Log the deletion # Log the deletion
logger.info( logger.info(f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}")
f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}"
)
# Delete the record # Delete the record
db.delete(file_record) db.delete(file_record)
@@ -271,9 +252,7 @@ def delete_file_record(request: Request, file_id: int, db: DbSession):
except Exception as e: except Exception as e:
db.rollback() db.rollback()
logger.exception(f"Error deleting file record {file_id}: {str(e)}") logger.exception(f"Error deleting file record {file_id}: {str(e)}")
raise HTTPException( raise HTTPException(status_code=500, detail=f"Error deleting file record: {str(e)}")
status_code=500, detail=f"Error deleting file record: {str(e)}"
)
@router.post("/files/bulk-delete") @router.post("/files/bulk-delete")
@@ -285,18 +264,14 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: DbSession):
""" """
# Check if file deletion is allowed # Check if file deletion is allowed
if not settings.allow_file_delete: if not settings.allow_file_delete:
raise HTTPException( raise HTTPException(status_code=403, detail="File deletion is disabled in the configuration")
status_code=403, detail="File deletion is disabled in the configuration"
)
try: try:
# Find all file records # Find all file records
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
if not file_records: if not file_records:
raise HTTPException( raise HTTPException(status_code=404, detail="No files found with the provided IDs")
status_code=404, detail="No files found with the provided IDs"
)
deleted_count = len(file_records) deleted_count = len(file_records)
deleted_ids = [f.id for f in file_records] deleted_ids = [f.id for f in file_records]
@@ -321,9 +296,7 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: DbSession):
except Exception as e: except Exception as e:
db.rollback() db.rollback()
logger.exception(f"Error bulk deleting file records: {str(e)}") logger.exception(f"Error bulk deleting file records: {str(e)}")
raise HTTPException( raise HTTPException(status_code=500, detail=f"Error bulk deleting file records: {str(e)}")
status_code=500, detail=f"Error bulk deleting file records: {str(e)}"
)
@router.post("/files/bulk-reprocess") @router.post("/files/bulk-reprocess")
@@ -337,9 +310,7 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
if not file_records: if not file_records:
raise HTTPException( raise HTTPException(status_code=404, detail="No files found with the provided IDs")
status_code=404, detail="No files found with the provided IDs"
)
task_ids = [] task_ids = []
processed_files = [] processed_files = []
@@ -348,9 +319,7 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
for file_record in file_records: for file_record in file_records:
try: try:
# Check if local file exists # Check if local file exists
if not file_record.local_filename or not os.path.exists( if not file_record.local_filename or not os.path.exists(file_record.local_filename):
file_record.local_filename
):
errors.append( errors.append(
{ {
"file_id": file_record.id, "file_id": file_record.id,
@@ -361,9 +330,7 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
continue continue
# Queue the file for processing, passing file_id to skip duplicate check # Queue the file for processing, passing file_id to skip duplicate check
task = process_document.delay( task = process_document.delay(file_record.local_filename, file_id=file_record.id)
file_record.local_filename, file_id=file_record.id
)
task_ids.append(task.id) task_ids.append(task.id)
processed_files.append( processed_files.append(
{ {
@@ -400,9 +367,7 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: DbSession):
raise raise
except Exception as e: except Exception as e:
logger.exception(f"Error bulk reprocessing files: {str(e)}") logger.exception(f"Error bulk reprocessing files: {str(e)}")
raise HTTPException( raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files: {str(e)}")
status_code=500, detail=f"Error bulk reprocessing files: {str(e)}"
)
@router.post("/files/{file_id}/reprocess") @router.post("/files/{file_id}/reprocess")
@@ -422,14 +387,10 @@ def reprocess_single_file(request: Request, file_id: int, db: DbSession):
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record: if not file_record:
raise HTTPException( raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
status_code=404, detail=f"File with ID {file_id} not found"
)
# Check if local file exists # Check if local file exists
if not file_record.local_filename or not os.path.exists( if not file_record.local_filename or not os.path.exists(file_record.local_filename):
file_record.local_filename
):
raise HTTPException( raise HTTPException(
status_code=400, status_code=400,
detail="Local file not found on disk. Cannot reprocess.", detail="Local file not found on disk. Cannot reprocess.",
@@ -458,9 +419,7 @@ def reprocess_single_file(request: Request, file_id: int, db: DbSession):
raise raise
except Exception as e: except Exception as e:
logger.exception(f"Error reprocessing file {file_id}: {str(e)}") logger.exception(f"Error reprocessing file {file_id}: {str(e)}")
raise HTTPException( raise HTTPException(status_code=500, detail=f"Error reprocessing file: {str(e)}")
status_code=500, detail=f"Error reprocessing file: {str(e)}"
)
@router.post("/files/{file_id}/reprocess-with-cloud-ocr") @router.post("/files/{file_id}/reprocess-with-cloud-ocr")
@@ -484,19 +443,13 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession):
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record: if not file_record:
raise HTTPException( raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
status_code=404, detail=f"File with ID {file_id} not found"
)
# Prefer using the original_file_path if available, otherwise fall back to local_filename # Prefer using the original_file_path if available, otherwise fall back to local_filename
source_file = None source_file = None
if file_record.original_file_path and os.path.exists( if file_record.original_file_path and os.path.exists(file_record.original_file_path):
file_record.original_file_path
):
source_file = file_record.original_file_path source_file = file_record.original_file_path
logger.info( logger.info(f"Using original file for Cloud OCR reprocessing: {source_file}")
f"Using original file for Cloud OCR reprocessing: {source_file}"
)
elif file_record.local_filename and os.path.exists(file_record.local_filename): elif file_record.local_filename and os.path.exists(file_record.local_filename):
source_file = file_record.local_filename source_file = file_record.local_filename
logger.info(f"Using local file for Cloud OCR reprocessing: {source_file}") logger.info(f"Using local file for Cloud OCR reprocessing: {source_file}")
@@ -532,9 +485,7 @@ def reprocess_with_cloud_ocr(request: Request, file_id: int, db: DbSession):
raise raise
except Exception as e: except Exception as e:
logger.exception(f"Error reprocessing file {file_id} with Cloud OCR: {str(e)}") logger.exception(f"Error reprocessing file {file_id} with Cloud OCR: {str(e)}")
raise HTTPException( raise HTTPException(status_code=500, detail=f"Error reprocessing file with Cloud OCR: {str(e)}")
status_code=500, detail=f"Error reprocessing file with Cloud OCR: {str(e)}"
)
def _extract_text_from_pdf(file_path: str) -> str: def _extract_text_from_pdf(file_path: str) -> str:
@@ -579,31 +530,19 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
if step_name == "process_document": if step_name == "process_document":
# Full reprocessing with duplicate check bypass # Full reprocessing with duplicate check bypass
logger.info( logger.info(f"Retrying process_document for file {file_id}: local_filename={file_record.local_filename!r}")
f"Retrying process_document for file {file_id}: local_filename={file_record.local_filename!r}"
)
if not file_record.local_filename: if not file_record.local_filename:
logger.error( logger.error(f"process_document retry failed for file {file_id}: local_filename is None")
f"process_document retry failed for file {file_id}: local_filename is None" raise HTTPException(status_code=400, detail="Local file path is None. Cannot retry.")
)
raise HTTPException(
status_code=400, detail="Local file path is None. Cannot retry."
)
exists = os.path.exists(file_record.local_filename) exists = os.path.exists(file_record.local_filename)
logger.info( logger.info(f"Checking local_filename: {file_record.local_filename!r}, exists={exists}")
f"Checking local_filename: {file_record.local_filename!r}, exists={exists}"
)
if not exists: if not exists:
error_message = f"Local file not found on disk. Cannot retry. Path checked: local_filename={file_record.local_filename!r} (exists=False)" error_message = f"Local file not found on disk. Cannot retry. Path checked: local_filename={file_record.local_filename!r} (exists=False)"
logger.error( logger.error(f"process_document retry failed for file {file_id}: {error_message}")
f"process_document retry failed for file {file_id}: {error_message}"
)
raise HTTPException(status_code=400, detail=error_message) raise HTTPException(status_code=400, detail=error_message)
logger.info( logger.info(f"Found file for process_document retry at: {file_record.local_filename!r}")
f"Found file for process_document retry at: {file_record.local_filename!r}"
)
task = process_document.delay( task = process_document.delay(
file_record.local_filename, file_record.local_filename,
original_filename=file_record.original_filename, original_filename=file_record.original_filename,
@@ -621,14 +560,10 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
) )
if not file_record.local_filename: if not file_record.local_filename:
logger.error(f"OCR retry failed for file {file_id}: local_filename is None") logger.error(f"OCR retry failed for file {file_id}: local_filename is None")
raise HTTPException( raise HTTPException(status_code=400, detail="Local file path is None. Cannot retry OCR.")
status_code=400, detail="Local file path is None. Cannot retry OCR."
)
exists = os.path.exists(file_record.local_filename) exists = os.path.exists(file_record.local_filename)
logger.info( logger.info(f"Checking local_filename: {file_record.local_filename!r}, exists={exists}")
f"Checking local_filename: {file_record.local_filename!r}, exists={exists}"
)
if not exists: if not exists:
error_message = f"Local file not found on disk. Cannot retry OCR. Path checked: local_filename={file_record.local_filename!r} (exists=False)" error_message = f"Local file not found on disk. Cannot retry OCR. Path checked: local_filename={file_record.local_filename!r} (exists=False)"
logger.error(f"OCR retry failed for file {file_id}: {error_message}") logger.error(f"OCR retry failed for file {file_id}: {error_message}")
@@ -644,28 +579,20 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
f"Retrying extract_metadata_with_gpt for file {file_id}: local_filename={file_record.local_filename!r}" f"Retrying extract_metadata_with_gpt for file {file_id}: local_filename={file_record.local_filename!r}"
) )
if not file_record.local_filename: if not file_record.local_filename:
logger.error( logger.error(f"Metadata extraction retry failed for file {file_id}: local_filename is None")
f"Metadata extraction retry failed for file {file_id}: local_filename is None"
)
raise HTTPException( raise HTTPException(
status_code=400, status_code=400,
detail="Local file path is None. Cannot retry metadata extraction.", detail="Local file path is None. Cannot retry metadata extraction.",
) )
exists = os.path.exists(file_record.local_filename) exists = os.path.exists(file_record.local_filename)
logger.info( logger.info(f"Checking local_filename: {file_record.local_filename!r}, exists={exists}")
f"Checking local_filename: {file_record.local_filename!r}, exists={exists}"
)
if not exists: if not exists:
error_message = f"Local file not found on disk. Cannot retry metadata extraction. Path checked: local_filename={file_record.local_filename!r} (exists=False)" error_message = f"Local file not found on disk. Cannot retry metadata extraction. Path checked: local_filename={file_record.local_filename!r} (exists=False)"
logger.error( logger.error(f"Metadata extraction retry failed for file {file_id}: {error_message}")
f"Metadata extraction retry failed for file {file_id}: {error_message}"
)
raise HTTPException(status_code=400, detail=error_message) raise HTTPException(status_code=400, detail=error_message)
logger.info( logger.info(f"Found file for metadata extraction retry at: {file_record.local_filename!r}")
f"Found file for metadata extraction retry at: {file_record.local_filename!r}"
)
extracted_text = _extract_text_from_pdf(file_record.local_filename) extracted_text = _extract_text_from_pdf(file_record.local_filename)
filename = os.path.basename(file_record.local_filename) filename = os.path.basename(file_record.local_filename)
task = extract_metadata_with_gpt.delay(filename, extracted_text, file_id) task = extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
@@ -697,36 +624,24 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
# Check 1: local_filename (original tmp location) # Check 1: local_filename (original tmp location)
if file_record.local_filename: if file_record.local_filename:
exists = os.path.exists(file_record.local_filename) exists = os.path.exists(file_record.local_filename)
checked_paths.append( checked_paths.append(f"local_filename={file_record.local_filename!r} (exists={exists})")
f"local_filename={file_record.local_filename!r} (exists={exists})" logger.info(f"Checking local_filename: {file_record.local_filename!r}, exists={exists}")
)
logger.info(
f"Checking local_filename: {file_record.local_filename!r}, exists={exists}"
)
if exists: if exists:
file_path = file_record.local_filename file_path = file_record.local_filename
# Check 2: processed_file_path # Check 2: processed_file_path
if not file_path and file_record.processed_file_path: if not file_path and file_record.processed_file_path:
exists = os.path.exists(file_record.processed_file_path) exists = os.path.exists(file_record.processed_file_path)
checked_paths.append( checked_paths.append(f"processed_file_path={file_record.processed_file_path!r} (exists={exists})")
f"processed_file_path={file_record.processed_file_path!r} (exists={exists})" logger.info(f"Checking processed_file_path: {file_record.processed_file_path!r}, exists={exists}")
)
logger.info(
f"Checking processed_file_path: {file_record.processed_file_path!r}, exists={exists}"
)
if exists: if exists:
file_path = file_record.processed_file_path file_path = file_record.processed_file_path
# Check 3: original_file_path (immutable copy in workdir/original/) # Check 3: original_file_path (immutable copy in workdir/original/)
if not file_path and file_record.original_file_path: if not file_path and file_record.original_file_path:
exists = os.path.exists(file_record.original_file_path) exists = os.path.exists(file_record.original_file_path)
checked_paths.append( checked_paths.append(f"original_file_path={file_record.original_file_path!r} (exists={exists})")
f"original_file_path={file_record.original_file_path!r} (exists={exists})" logger.info(f"Checking original_file_path: {file_record.original_file_path!r}, exists={exists}")
)
logger.info(
f"Checking original_file_path: {file_record.original_file_path!r}, exists={exists}"
)
if exists: if exists:
file_path = file_record.original_file_path file_path = file_record.original_file_path
@@ -734,25 +649,17 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
if not file_path and file_record.local_filename: if not file_path and file_record.local_filename:
workdir = settings.workdir workdir = settings.workdir
tmp_dir = os.path.join(workdir, "tmp") tmp_dir = os.path.join(workdir, "tmp")
fallback_path = os.path.join( fallback_path = os.path.join(tmp_dir, os.path.basename(file_record.local_filename))
tmp_dir, os.path.basename(file_record.local_filename)
)
exists = os.path.exists(fallback_path) exists = os.path.exists(fallback_path)
checked_paths.append( checked_paths.append(f"workdir_tmp_fallback={fallback_path!r} (exists={exists})")
f"workdir_tmp_fallback={fallback_path!r} (exists={exists})" logger.info(f"Checking workdir/tmp fallback: {fallback_path!r}, exists={exists}")
)
logger.info(
f"Checking workdir/tmp fallback: {fallback_path!r}, exists={exists}"
)
if exists: if exists:
file_path = fallback_path file_path = fallback_path
if not file_path: if not file_path:
paths_detail = "; ".join(checked_paths) paths_detail = "; ".join(checked_paths)
error_message = f"File not found on disk. Cannot retry metadata embedding. Paths checked: {paths_detail}" error_message = f"File not found on disk. Cannot retry metadata embedding. Paths checked: {paths_detail}"
logger.error( logger.error(f"embed_metadata_into_pdf retry failed for file {file_id}: {error_message}")
f"embed_metadata_into_pdf retry failed for file {file_id}: {error_message}"
)
raise HTTPException(status_code=400, detail=error_message) raise HTTPException(status_code=400, detail=error_message)
logger.info(f"Found file for embed_metadata_into_pdf retry at: {file_path!r}") logger.info(f"Found file for embed_metadata_into_pdf retry at: {file_path!r}")
@@ -760,13 +667,9 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
# Pass the full path to the task so it can locate the file # Pass the full path to the task so it can locate the file
task = extract_metadata_task.delay(file_path, extracted_text, file_id) task = extract_metadata_task.delay(file_path, extracted_text, file_id)
else: else:
raise HTTPException( raise HTTPException(status_code=400, detail=f"Unsupported pipeline step: {step_name}")
status_code=400, detail=f"Unsupported pipeline step: {step_name}"
)
logger.info( logger.info(f"Retrying pipeline step: FileID={file_record.id}, Step={step_name}, TaskID={task.id}")
f"Retrying pipeline step: FileID={file_record.id}, Step={step_name}, TaskID={task.id}"
)
return { return {
"status": "success", "status": "success",
@@ -807,9 +710,7 @@ def retry_subtask(
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record: if not file_record:
raise HTTPException( raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
status_code=404, detail=f"File with ID {file_id} not found"
)
# Pipeline processing steps that can be retried from the failed step # Pipeline processing steps that can be retried from the failed step
pipeline_step_names = { pipeline_step_names = {
@@ -892,9 +793,7 @@ def retry_subtask(
upload_task = task_map[subtask_name] upload_task = task_map[subtask_name]
task = upload_task.delay(file_path, file_id) task = upload_task.delay(file_path, file_id)
logger.info( logger.info(f"Retrying upload subtask: FileID={file_record.id}, Subtask={subtask_name}, TaskID={task.id}")
f"Retrying upload subtask: FileID={file_record.id}, Subtask={subtask_name}, TaskID={task.id}"
)
return { return {
"status": "success", "status": "success",
@@ -907,9 +806,7 @@ def retry_subtask(
except HTTPException: except HTTPException:
raise raise
except Exception as e: except Exception as e:
logger.exception( logger.exception(f"Error retrying subtask {subtask_name} for file {file_id}: {str(e)}")
f"Error retrying subtask {subtask_name} for file {file_id}: {str(e)}"
)
raise HTTPException(status_code=500, detail=f"Error retrying subtask: {str(e)}") raise HTTPException(status_code=500, detail=f"Error retrying subtask: {str(e)}")
@@ -938,18 +835,12 @@ def get_file_preview(
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record: if not file_record:
raise HTTPException( raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
status_code=404, detail=f"File with ID {file_id} not found"
)
if version == "original": if version == "original":
# Return the original file from tmp # Return the original file from tmp
if not file_record.local_filename or not os.path.exists( if not file_record.local_filename or not os.path.exists(file_record.local_filename):
file_record.local_filename raise HTTPException(status_code=404, detail="Original file not found on disk")
):
raise HTTPException(
status_code=404, detail="Original file not found on disk"
)
file_path = file_record.local_filename file_path = file_record.local_filename
@@ -984,18 +875,14 @@ def get_file_preview(
return FileResponse( return FileResponse(
path=file_path, path=file_path,
media_type=file_record.mime_type or "application/pdf", media_type=file_record.mime_type or "application/pdf",
headers={ headers={"Content-Disposition": f'inline; filename="{file_record.original_filename}"'},
"Content-Disposition": f'inline; filename="{file_record.original_filename}"'
},
) )
except HTTPException: except HTTPException:
raise raise
except Exception as e: except Exception as e:
logger.exception(f"Error retrieving file preview: {str(e)}") logger.exception(f"Error retrieving file preview: {str(e)}")
raise HTTPException( raise HTTPException(status_code=500, detail=f"Error retrieving file preview: {str(e)}")
status_code=500, detail=f"Error retrieving file preview: {str(e)}"
)
@router.get("/files/{file_id}/download") @router.get("/files/{file_id}/download")
@@ -1023,18 +910,12 @@ def download_file(
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record: if not file_record:
raise HTTPException( raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
status_code=404, detail=f"File with ID {file_id} not found"
)
if version == "original": if version == "original":
# Return the original file from tmp # Return the original file from tmp
if not file_record.local_filename or not os.path.exists( if not file_record.local_filename or not os.path.exists(file_record.local_filename):
file_record.local_filename raise HTTPException(status_code=404, detail="Original file not found on disk")
):
raise HTTPException(
status_code=404, detail="Original file not found on disk"
)
file_path = file_record.local_filename file_path = file_record.local_filename
@@ -1069,9 +950,7 @@ def download_file(
return FileResponse( return FileResponse(
path=file_path, path=file_path,
media_type=file_record.mime_type or "application/pdf", media_type=file_record.mime_type or "application/pdf",
headers={ headers={"Content-Disposition": f'attachment; filename="{file_record.original_filename}"'},
"Content-Disposition": f'attachment; filename="{file_record.original_filename}"'
},
) )
except HTTPException: except HTTPException:
@@ -1189,9 +1068,7 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
# Check if file splitting is needed (only for PDFs) # Check if file splitting is needed (only for PDFs)
from app.utils.file_splitting import should_split_file from app.utils.file_splitting import should_split_file
should_split = is_pdf and should_split_file( should_split = is_pdf and should_split_file(target_path, settings.max_single_file_size)
target_path, settings.max_single_file_size
)
if should_split: if should_split:
# File needs to be split before processing # File needs to be split before processing
@@ -1209,9 +1086,7 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
task_ids = [] task_ids = []
for split_file in split_files: for split_file in split_files:
split_filename = os.path.basename(split_file) split_filename = os.path.basename(split_file)
task = process_document.delay( task = process_document.delay(split_file, original_filename=split_filename)
split_file, original_filename=split_filename
)
task_ids.append(task.id) task_ids.append(task.id)
logger.info(f"Enqueued split PDF part for processing: {split_file}") logger.info(f"Enqueued split PDF part for processing: {split_file}")
@@ -1229,9 +1104,7 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
except Exception as e: except Exception as e:
logger.exception(f"Failed to split file {target_path}: {str(e)}") logger.exception(f"Failed to split file {target_path}: {str(e)}")
# Fall back to processing the whole file # Fall back to processing the whole file
logger.warning( logger.warning(f"Falling back to processing whole file due to split error: {str(e)}")
f"Falling back to processing whole file due to split error: {str(e)}"
)
should_split = False should_split = False
if is_pdf and not should_split: if is_pdf and not should_split:
@@ -1239,8 +1112,7 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
task = process_document.delay(target_path, original_filename=safe_filename) task = process_document.delay(target_path, original_filename=safe_filename)
logger.info(f"Enqueued PDF for processing: {target_path}") logger.info(f"Enqueued PDF for processing: {target_path}")
elif mime_type in IMAGE_MIME_TYPES or any( elif mime_type in IMAGE_MIME_TYPES or any(
file_ext.endswith(ext) file_ext.endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"]
for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"]
): ):
# If it's an image, convert to PDF first # If it's an image, convert to PDF first
task = convert_to_pdf.delay(target_path, original_filename=safe_filename) task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
@@ -1267,9 +1139,7 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
logger.info(f"Enqueued office document for PDF conversion: {target_path}") logger.info(f"Enqueued office document for PDF conversion: {target_path}")
else: else:
# For any other file type, attempt conversion but log a warning # For any other file type, attempt conversion but log a warning
logger.warning( logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion"
)
task = convert_to_pdf.delay(target_path, original_filename=safe_filename) task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
return { return {
app/config.py
+15 -45
View File
@@ -33,9 +33,7 @@ class Settings(BaseSettings):
# Making Paperless optional # Making Paperless optional
paperless_ngx_api_token: Optional[str] = None paperless_ngx_api_token: Optional[str] = None
paperless_host: Optional[str] = None paperless_host: Optional[str] = None
paperless_custom_field_absender: Optional[str] = ( paperless_custom_field_absender: Optional[str] = None # Name of the "absender" custom field in Paperless
None # Name of the "absender" custom field in Paperless
)
# JSON mapping of metadata field names to Paperless custom field names # JSON mapping of metadata field names to Paperless custom field names
# Example: {"absender": "Sender", "empfaenger": "Recipient", # Example: {"absender": "Sender", "empfaenger": "Recipient",
# "language": "Language", "correspondent": "Correspondent"} # "language": "Language", "correspondent": "Correspondent"}
@@ -115,9 +113,7 @@ class Settings(BaseSettings):
sftp_private_key_passphrase: Optional[str] = None sftp_private_key_passphrase: Optional[str] = None
# Security: Host key verification is enabled by default for security # Security: Host key verification is enabled by default for security
# In development/testing, set to True to disable verification (not recommended) # In development/testing, set to True to disable verification (not recommended)
sftp_disable_host_key_verification: bool = ( sftp_disable_host_key_verification: bool = False # Default enforces host key verification
False # Default enforces host key verification
)
# Email settings # Email settings
email_host: Optional[str] = None email_host: Optional[str] = None
@@ -125,17 +121,13 @@ class Settings(BaseSettings):
email_username: Optional[str] = None email_username: Optional[str] = None
email_password: Optional[str] = None email_password: Optional[str] = None
email_use_tls: bool = True email_use_tls: bool = True
email_sender: Optional[str] = ( email_sender: Optional[str] = None # From address, defaults to email_username if not set
None # From address, defaults to email_username if not set
)
email_default_recipient: Optional[str] = None email_default_recipient: Optional[str] = None
# OneDrive settings # OneDrive settings
onedrive_client_id: Optional[str] = None onedrive_client_id: Optional[str] = None
onedrive_client_secret: Optional[str] = None onedrive_client_secret: Optional[str] = None
onedrive_tenant_id: Optional[str] = ( onedrive_tenant_id: Optional[str] = "common" # Default to "common" for personal accounts
"common" # Default to "common" for personal accounts
)
onedrive_refresh_token: Optional[str] = None # Required for personal accounts onedrive_refresh_token: Optional[str] = None # Required for personal accounts
onedrive_folder_path: Optional[str] = None onedrive_folder_path: Optional[str] = None
@@ -153,9 +145,7 @@ class Settings(BaseSettings):
uptime_kuma_ping_interval: int = 5 # Default ping interval in minutes uptime_kuma_ping_interval: int = 5 # Default ping interval in minutes
# HTTP request settings # HTTP request settings
http_request_timeout: int = ( http_request_timeout: int = 120 # Default timeout for HTTP requests in seconds (handles large file operations)
120 # Default timeout for HTTP requests in seconds (handles large file operations)
)
# Feature flags # Feature flags
allow_file_delete: bool = True # Default to allowing file deletion from database allow_file_delete: bool = True # Default to allowing file deletion from database
@@ -175,18 +165,12 @@ class Settings(BaseSettings):
default_factory=list, default_factory=list,
description="List of Apprise notification URLs (e.g., discord://, telegram://, etc.)", description="List of Apprise notification URLs (e.g., discord://, telegram://, etc.)",
) )
notify_on_task_failure: bool = Field( notify_on_task_failure: bool = Field(default=True, description="Send notifications when Celery tasks fail")
default=True, description="Send notifications when Celery tasks fail"
)
notify_on_credential_failure: bool = Field( notify_on_credential_failure: bool = Field(
default=True, description="Send notifications when credential checks fail" default=True, description="Send notifications when credential checks fail"
) )
notify_on_startup: bool = Field( notify_on_startup: bool = Field(default=True, description="Send notifications when application starts")
default=True, description="Send notifications when application starts" notify_on_shutdown: bool = Field(default=False, description="Send notifications when application shuts down")
)
notify_on_shutdown: bool = Field(
default=False, description="Send notifications when application shuts down"
)
notify_on_file_processed: bool = Field( notify_on_file_processed: bool = Field(
default=True, default=True,
description="Send notifications when files are successfully processed", description="Send notifications when files are successfully processed",
@@ -256,9 +240,7 @@ class Settings(BaseSettings):
) )
# Content-Security-Policy (CSP) - Controls resource loading # Content-Security-Policy (CSP) - Controls resource loading
security_header_csp_enabled: bool = Field( security_header_csp_enabled: bool = Field(default=True, description="Enable CSP header.")
default=True, description="Enable CSP header."
)
security_header_csp_value: str = Field( security_header_csp_value: str = Field(
default=( default=(
"default-src 'self'; script-src 'self' 'unsafe-inline';" "default-src 'self'; script-src 'self' 'unsafe-inline';"
@@ -268,9 +250,7 @@ class Settings(BaseSettings):
) )
# X-Frame-Options - Prevents clickjacking # X-Frame-Options - Prevents clickjacking
security_header_x_frame_options_enabled: bool = Field( security_header_x_frame_options_enabled: bool = Field(default=True, description="Enable X-Frame-Options header.")
default=True, description="Enable X-Frame-Options header."
)
security_header_x_frame_options_value: str = Field( security_header_x_frame_options_value: str = Field(
default="DENY", default="DENY",
description="X-Frame-Options header value. Options: DENY, SAMEORIGIN, or ALLOW-FROM uri", description="X-Frame-Options header value. Options: DENY, SAMEORIGIN, or ALLOW-FROM uri",
@@ -335,9 +315,7 @@ class Settings(BaseSettings):
if isinstance(data, dict): if isinstance(data, dict):
for key, value in data.items(): for key, value in data.items():
if isinstance(value, str) and len(value) >= 2: if isinstance(value, str) and len(value) >= 2:
if (value[0] == '"' and value[-1] == '"') or ( if (value[0] == '"' and value[-1] == '"') or (value[0] == "'" and value[-1] == "'"):
value[0] == "'" and value[-1] == "'"
):
data[key] = value[1:-1] data[key] = value[1:-1]
return data return data
@@ -372,9 +350,7 @@ class Settings(BaseSettings):
return env_build_date return env_build_date
# Then try to get build date from BUILD_DATE file # Then try to get build date from BUILD_DATE file
build_date_file = os.path.join( build_date_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "BUILD_DATE")
os.path.dirname(os.path.dirname(__file__)), "BUILD_DATE"
)
if os.path.exists(build_date_file): if os.path.exists(build_date_file):
with open(build_date_file, "r") as f: with open(build_date_file, "r") as f:
return f.read().strip() return f.read().strip()
@@ -391,9 +367,7 @@ class Settings(BaseSettings):
return env_version return env_version
# Then try to get version from VERSION file # Then try to get version from VERSION file
version_file = os.path.join( version_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "VERSION")
os.path.dirname(os.path.dirname(__file__)), "VERSION"
)
if os.path.exists(version_file): if os.path.exists(version_file):
with open(version_file, "r") as f: with open(version_file, "r") as f:
return f.read().strip() return f.read().strip()
@@ -410,9 +384,7 @@ class Settings(BaseSettings):
return env_sha return env_sha
# Then try to get from GIT_SHA file # Then try to get from GIT_SHA file
git_sha_file = os.path.join( git_sha_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "GIT_SHA")
os.path.dirname(os.path.dirname(__file__)), "GIT_SHA"
)
if os.path.exists(git_sha_file): if os.path.exists(git_sha_file):
with open(git_sha_file, "r") as f: with open(git_sha_file, "r") as f:
return f.read().strip() return f.read().strip()
@@ -423,9 +395,7 @@ class Settings(BaseSettings):
@property @property
def runtime_info(self) -> str: def runtime_info(self) -> str:
"""Get runtime information from file.""" """Get runtime information from file."""
runtime_info_file = os.path.join( runtime_info_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "RUNTIME_INFO")
os.path.dirname(os.path.dirname(__file__)), "RUNTIME_INFO"
)
if os.path.exists(runtime_info_file): if os.path.exists(runtime_info_file):
with open(runtime_info_file, "r") as f: with open(runtime_info_file, "r") as f:
return f.read().strip() return f.read().strip()
app/main.py
+6 -17
View File
@@ -41,8 +41,7 @@ if settings.auth_enabled and not settings.session_secret:
"Generate one with: python -c 'import secrets; print(secrets.token_hex(32))'" "Generate one with: python -c 'import secrets; print(secrets.token_hex(32))'"
) )
SESSION_SECRET = ( SESSION_SECRET = (
settings.session_secret settings.session_secret or "INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS"
or "INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS"
) )
@@ -81,15 +80,11 @@ async def lifespan(app: FastAPI):
len(issues) > 0 for provider, issues in config_issues["storage"].items() len(issues) > 0 for provider, issues in config_issues["storage"].items()
) )
if has_issues: if has_issues:
logging.warning( logging.warning("Application started with configuration issues - some features may be unavailable")
"Application started with configuration issues - some features may be unavailable"
)
else: else:
logging.info("Application started with valid configuration") logging.info("Application started with valid configuration")
logging.info( logging.info("Router organization: Using refactored API routers from app/api/ directory")
"Router organization: Using refactored API routers from app/api/ directory"
)
# Initialize notification system # Initialize notification system
init_apprise() init_apprise()
@@ -110,9 +105,7 @@ async def lifespan(app: FastAPI):
app = FastAPI(title="DocuElevate", lifespan=lifespan) app = FastAPI(title="DocuElevate", lifespan=lifespan)
# Initialize rate limiter and attach to app state # Initialize rate limiter and attach to app state
limiter = create_limiter( limiter = create_limiter(redis_url=settings.redis_url, enabled=settings.rate_limiting_enabled)
redis_url=settings.redis_url, enabled=settings.rate_limiting_enabled
)
app.state.limiter = limiter app.state.limiter = limiter
app.add_exception_handler(RateLimitExceeded, get_rate_limit_exceeded_handler()) app.add_exception_handler(RateLimitExceeded, get_rate_limit_exceeded_handler())
@@ -152,9 +145,7 @@ static_dir = pathlib.Path(__file__).parents[1] / "frontend" / "static"
if os.path.exists(static_dir): if os.path.exists(static_dir):
app.mount("/static", StaticFiles(directory=str(static_dir)), name="static") app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
else: else:
print( print(f"WARNING: Static directory not found at {static_dir}. Static files will not be served.")
f"WARNING: Static directory not found at {static_dir}. Static files will not be served."
)
# Custom exception handlers that return JSON for API routes and HTML for frontend routes # Custom exception handlers that return JSON for API routes and HTML for frontend routes
@@ -173,9 +164,7 @@ async def http_exception_handler(request: Request, exc: HTTPException):
# Handle 404 errors with a custom template # Handle 404 errors with a custom template
if exc.status_code == 404: if exc.status_code == 404:
return templates.TemplateResponse( return templates.TemplateResponse("404.html", {"request": request}, status_code=status.HTTP_404_NOT_FOUND)
"404.html", {"request": request}, status_code=status.HTTP_404_NOT_FOUND
)
# For other HTTP errors, we could create specific templates or use a generic one # For other HTTP errors, we could create specific templates or use a generic one
# For now, return a simple error page # For now, return a simple error page
tests/test_file_upload.py
+24 -73
View File
@@ -42,14 +42,10 @@ def mock_celery_tasks():
class TestValidFileUploads: class TestValidFileUploads:
"""Tests for successful file uploads with various valid file types.""" """Tests for successful file uploads with various valid file types."""
def test_upload_valid_pdf( def test_upload_valid_pdf(self, client: TestClient, sample_pdf_path: str, mock_celery_tasks):
self, client: TestClient, sample_pdf_path: str, mock_celery_tasks
):
"""Test uploading a valid PDF file.""" """Test uploading a valid PDF file."""
with open(sample_pdf_path, "rb") as f: with open(sample_pdf_path, "rb") as f:
response = client.post( response = client.post("/api/ui-upload", files={"file": ("document.pdf", f, "application/pdf")})
"/api/ui-upload", files={"file": ("document.pdf", f, "application/pdf")}
)
assert response.status_code == 200 assert response.status_code == 200
data = response.json() data = response.json()
@@ -182,9 +178,7 @@ class TestInvalidFileUploads:
with patch.object(settings, "max_upload_size", small_limit): with patch.object(settings, "max_upload_size", small_limit):
response = client.post( response = client.post(
"/api/ui-upload", "/api/ui-upload",
files={ files={"file": ("huge.pdf", io.BytesIO(small_content), "application/pdf")},
"file": ("huge.pdf", io.BytesIO(small_content), "application/pdf")
},
) )
assert response.status_code == 413 # Request Entity Too Large assert response.status_code == 413 # Request Entity Too Large
@@ -236,9 +230,7 @@ class TestInvalidFileUploads:
class TestUploadSecurity: class TestUploadSecurity:
"""Tests for security aspects of file uploads.""" """Tests for security aspects of file uploads."""
def test_path_traversal_prevention_dotdot( def test_path_traversal_prevention_dotdot(self, client: TestClient, mock_celery_tasks):
self, client: TestClient, mock_celery_tasks
):
"""Test that path traversal attempts are prevented.""" """Test that path traversal attempts are prevented."""
# Try to upload a file with path traversal in filename # Try to upload a file with path traversal in filename
malicious_filename = "../../etc/passwd.pdf" malicious_filename = "../../etc/passwd.pdf"
@@ -246,9 +238,7 @@ class TestUploadSecurity:
response = client.post( response = client.post(
"/api/ui-upload", "/api/ui-upload",
files={ files={"file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf")},
"file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf")
},
) )
assert response.status_code == 200 assert response.status_code == 200
@@ -259,18 +249,14 @@ class TestUploadSecurity:
assert ".." not in data["stored_filename"] assert ".." not in data["stored_filename"]
assert "/" not in data["stored_filename"] assert "/" not in data["stored_filename"]
def test_path_traversal_prevention_absolute( def test_path_traversal_prevention_absolute(self, client: TestClient, mock_celery_tasks):
self, client: TestClient, mock_celery_tasks
):
"""Test that absolute path attempts are prevented.""" """Test that absolute path attempts are prevented."""
malicious_filename = "/etc/shadow.pdf" malicious_filename = "/etc/shadow.pdf"
pdf_content = b"%PDF-1.4\n%EOF" pdf_content = b"%PDF-1.4\n%EOF"
response = client.post( response = client.post(
"/api/ui-upload", "/api/ui-upload",
files={ files={"file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf")},
"file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf")
},
) )
assert response.status_code == 200 assert response.status_code == 200
@@ -280,18 +266,14 @@ class TestUploadSecurity:
assert data["original_filename"] == "shadow.pdf" assert data["original_filename"] == "shadow.pdf"
assert not data["stored_filename"].startswith("/") assert not data["stored_filename"].startswith("/")
def test_filename_with_special_characters( def test_filename_with_special_characters(self, client: TestClient, mock_celery_tasks):
self, client: TestClient, mock_celery_tasks
):
"""Test handling of filenames with special characters.""" """Test handling of filenames with special characters."""
special_filename = "file name with spaces & special!@#chars.pdf" special_filename = "file name with spaces & special!@#chars.pdf"
pdf_content = b"%PDF-1.4\n%EOF" pdf_content = b"%PDF-1.4\n%EOF"
response = client.post( response = client.post(
"/api/ui-upload", "/api/ui-upload",
files={ files={"file": (special_filename, io.BytesIO(pdf_content), "application/pdf")},
"file": (special_filename, io.BytesIO(pdf_content), "application/pdf")
},
) )
assert response.status_code == 200 assert response.status_code == 200
@@ -307,9 +289,7 @@ class TestUploadSecurity:
# Stored filename should have UUID and extension # Stored filename should have UUID and extension
assert data["stored_filename"].endswith(".pdf") assert data["stored_filename"].endswith(".pdf")
def test_path_traversal_prevention_windows_style( def test_path_traversal_prevention_windows_style(self, client: TestClient, mock_celery_tasks):
self, client: TestClient, mock_celery_tasks
):
"""Test that Windows-style path traversal attempts are prevented.""" """Test that Windows-style path traversal attempts are prevented."""
# Try Windows-style path with backslashes # Try Windows-style path with backslashes
malicious_filename = "..\\..\\..\\windows\\system32\\config.pdf" malicious_filename = "..\\..\\..\\windows\\system32\\config.pdf"
@@ -317,9 +297,7 @@ class TestUploadSecurity:
response = client.post( response = client.post(
"/api/ui-upload", "/api/ui-upload",
files={ files={"file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf")},
"file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf")
},
) )
assert response.status_code == 200 assert response.status_code == 200
@@ -338,18 +316,14 @@ class TestUploadSecurity:
assert ".." not in data["original_filename"] assert ".." not in data["original_filename"]
assert "/" not in data["original_filename"] assert "/" not in data["original_filename"]
def test_path_traversal_prevention_mixed_separators( def test_path_traversal_prevention_mixed_separators(self, client: TestClient, mock_celery_tasks):
self, client: TestClient, mock_celery_tasks
):
"""Test handling of filenames with mixed path separators.""" """Test handling of filenames with mixed path separators."""
malicious_filename = "../path\\to/file.pdf" malicious_filename = "../path\\to/file.pdf"
pdf_content = b"%PDF-1.4\n%EOF" pdf_content = b"%PDF-1.4\n%EOF"
response = client.post( response = client.post(
"/api/ui-upload", "/api/ui-upload",
files={ files={"file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf")},
"file": (malicious_filename, io.BytesIO(pdf_content), "application/pdf")
},
) )
assert response.status_code == 200 assert response.status_code == 200
@@ -376,9 +350,7 @@ class TestUploadErrorHandling:
response = client.post( response = client.post(
"/api/ui-upload", "/api/ui-upload",
files={ files={"file": ("test.pdf", io.BytesIO(pdf_content), "application/pdf")},
"file": ("test.pdf", io.BytesIO(pdf_content), "application/pdf")
},
) )
assert response.status_code == 500 assert response.status_code == 500
@@ -386,9 +358,7 @@ class TestUploadErrorHandling:
def test_upload_celery_task_failure(self, client: TestClient, mock_celery_tasks): def test_upload_celery_task_failure(self, client: TestClient, mock_celery_tasks):
"""Test handling when Celery task queueing fails.""" """Test handling when Celery task queueing fails."""
mock_celery_tasks["process_document"].side_effect = Exception( mock_celery_tasks["process_document"].side_effect = Exception("Celery connection failed")
"Celery connection failed"
)
pdf_content = b"%PDF-1.4\n%EOF" pdf_content = b"%PDF-1.4\n%EOF"
@@ -397,9 +367,7 @@ class TestUploadErrorHandling:
with pytest.raises(Exception): with pytest.raises(Exception):
client.post( client.post(
"/api/ui-upload", "/api/ui-upload",
files={ files={"file": ("test.pdf", io.BytesIO(pdf_content), "application/pdf")},
"file": ("test.pdf", io.BytesIO(pdf_content), "application/pdf")
},
) )
@@ -440,19 +408,14 @@ class TestUploadFilenameHandling:
response = client.post( response = client.post(
"/api/ui-upload", "/api/ui-upload",
files={ files={"file": ("NOEXTENSION", io.BytesIO(content), "application/octet-stream")},
"file": ("NOEXTENSION", io.BytesIO(content), "application/octet-stream")
},
) )
assert response.status_code == 200 assert response.status_code == 200
data = response.json() data = response.json()
assert data["original_filename"] == "NOEXTENSION" assert data["original_filename"] == "NOEXTENSION"
# Stored filename should be just the UUID without extension # Stored filename should be just the UUID without extension
assert ( assert "." not in data["stored_filename"] or data["stored_filename"].count(".") == 0
"." not in data["stored_filename"]
or data["stored_filename"].count(".") == 0
)
@pytest.mark.integration @pytest.mark.integration
@@ -465,9 +428,7 @@ class TestUploadMimeTypeDetection:
response = client.post( response = client.post(
"/api/ui-upload", "/api/ui-upload",
files={ files={"file": ("doc.pdf", io.BytesIO(pdf_content), "application/octet-stream")},
"file": ("doc.pdf", io.BytesIO(pdf_content), "application/octet-stream")
},
) )
assert response.status_code == 200 assert response.status_code == 200
@@ -499,9 +460,7 @@ class TestUploadMimeTypeDetection:
class TestFileSplitting: class TestFileSplitting:
"""Tests for file splitting functionality when MAX_SINGLE_FILE_SIZE is configured.""" """Tests for file splitting functionality when MAX_SINGLE_FILE_SIZE is configured."""
def test_pdf_splitting_when_configured( def test_pdf_splitting_when_configured(self, client: TestClient, sample_pdf_path: str, mock_celery_tasks):
self, client: TestClient, sample_pdf_path: str, mock_celery_tasks
):
"""Test that PDFs are split when they exceed MAX_SINGLE_FILE_SIZE.""" """Test that PDFs are split when they exceed MAX_SINGLE_FILE_SIZE."""
from app.config import settings from app.config import settings
@@ -516,9 +475,7 @@ class TestFileSplitting:
] ]
# Mock should_split_file to return True # Mock should_split_file to return True
with patch( with patch("app.utils.file_splitting.should_split_file", return_value=True):
"app.utils.file_splitting.should_split_file", return_value=True
):
with open(sample_pdf_path, "rb") as f: with open(sample_pdf_path, "rb") as f:
response = client.post( response = client.post(
"/api/ui-upload", "/api/ui-upload",
@@ -539,9 +496,7 @@ class TestFileSplitting:
# Verify each split file was queued for processing # Verify each split file was queued for processing
assert mock_celery_tasks["process_document"].call_count == 3 assert mock_celery_tasks["process_document"].call_count == 3
def test_no_splitting_when_not_configured( def test_no_splitting_when_not_configured(self, client: TestClient, sample_pdf_path: str, mock_celery_tasks):
self, client: TestClient, sample_pdf_path: str, mock_celery_tasks
):
"""Test that PDFs are not split when MAX_SINGLE_FILE_SIZE is None.""" """Test that PDFs are not split when MAX_SINGLE_FILE_SIZE is None."""
from app.config import settings from app.config import settings
@@ -564,9 +519,7 @@ class TestFileSplitting:
# Verify file was processed directly without splitting # Verify file was processed directly without splitting
mock_celery_tasks["process_document"].assert_called_once() mock_celery_tasks["process_document"].assert_called_once()
def test_no_splitting_for_small_files( def test_no_splitting_for_small_files(self, client: TestClient, sample_pdf_path: str, mock_celery_tasks):
self, client: TestClient, sample_pdf_path: str, mock_celery_tasks
):
"""Test that small PDFs are not split even when MAX_SINGLE_FILE_SIZE is configured.""" """Test that small PDFs are not split even when MAX_SINGLE_FILE_SIZE is configured."""
from app.config import settings from app.config import settings
@@ -588,9 +541,7 @@ class TestFileSplitting:
# Verify file was processed directly # Verify file was processed directly
mock_celery_tasks["process_document"].assert_called_once() mock_celery_tasks["process_document"].assert_called_once()
def test_splitting_fallback_on_error( def test_splitting_fallback_on_error(self, client: TestClient, sample_pdf_path: str, mock_celery_tasks):
self, client: TestClient, sample_pdf_path: str, mock_celery_tasks
):
"""Test that if splitting fails, the file is processed as a whole.""" """Test that if splitting fails, the file is processed as a whole."""
from app.config import settings from app.config import settings
tests/test_request_size_limit.py
+4 -12
View File
@@ -93,9 +93,7 @@ class TestRequestSizeLimitMiddleware:
# May get 200/401/403 but not 413 # May get 200/401/403 but not 413
assert response.status_code != 413 assert response.status_code != 413
def test_middleware_error_message_contains_limit_and_config_hint( def test_middleware_error_message_contains_limit_and_config_hint(self, client: TestClient):
self, client: TestClient
):
"""413 response body contains limit details and config variable name.""" """413 response body contains limit details and config variable name."""
from app.config import settings from app.config import settings
@@ -132,9 +130,7 @@ class TestFileUploadSizeLimitStreaming:
mock_conv.delay.return_value = task mock_conv.delay.return_value = task
yield yield
def test_upload_rejected_when_content_length_declared_too_large( def test_upload_rejected_when_content_length_declared_too_large(self, client: TestClient):
self, client: TestClient
):
"""Upload is rejected early via Content-Length check before reading data.""" """Upload is rejected early via Content-Length check before reading data."""
from app.config import settings from app.config import settings
@@ -150,9 +146,7 @@ class TestFileUploadSizeLimitStreaming:
) )
assert response.status_code == 413 assert response.status_code == 413
def test_upload_rejected_mid_stream_when_data_exceeds_limit( def test_upload_rejected_mid_stream_when_data_exceeds_limit(self, client: TestClient):
self, client: TestClient
):
"""Upload is rejected mid-stream when actual data exceeds max_upload_size.""" """Upload is rejected mid-stream when actual data exceeds max_upload_size."""
from app.config import settings from app.config import settings
@@ -162,9 +156,7 @@ class TestFileUploadSizeLimitStreaming:
large_content = b"x" * (small_limit + 1) large_content = b"x" * (small_limit + 1)
response = client.post( response = client.post(
"/api/ui-upload", "/api/ui-upload",
files={ files={"file": ("big.pdf", io.BytesIO(large_content), "application/pdf")},
"file": ("big.pdf", io.BytesIO(large_content), "application/pdf")
},
) )
assert response.status_code == 413 assert response.status_code == 413
assert "too large" in response.json()["detail"].lower() assert "too large" in response.json()["detail"].lower()