gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply code formatting and fix linting issues

Browse Source 
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-07 17:37:50 +00:00
parent 2db58afedf
commit 7d24b7c668
4 changed files with 288 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/files.py
+163 -192
View File
@@ -1,28 +1,31 @@
""" """
File-related API endpoints File-related API endpoints
""" """
from fastapi import APIRouter, Request, HTTPException, Depends, UploadFile, File, Query
from sqlalchemy.orm import Session
from sqlalchemy import desc, asc, or_, func
from typing import Optional, List
import logging import logging
import mimetypes
import os import os
import uuid import uuid
import mimetypes from typing import List, Optional
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile
from sqlalchemy import asc, desc, or_
from sqlalchemy.orm import Session
from app.auth import require_login
from app.models import FileRecord, ProcessingLog
from app.config import settings
from app.api.common import get_db from app.api.common import get_db
from app.tasks.process_document import process_document from app.auth import require_login
from app.config import settings
from app.models import FileRecord, ProcessingLog
from app.tasks.convert_to_pdf import convert_to_pdf from app.tasks.convert_to_pdf import convert_to_pdf
from app.utils.file_status import get_file_processing_status, get_files_processing_status from app.tasks.process_document import process_document
from app.utils.file_status import get_files_processing_status
# Set up logging # Set up logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
router = APIRouter() router = APIRouter()
@router.get("/files") @router.get("/files")
@require_login @require_login
def list_files_api( def list_files_api(
@@ -30,16 +33,18 @@ def list_files_api(
db: Session = Depends(get_db), db: Session = Depends(get_db),
page: int = Query(1, ge=1, description="Page number"), page: int = Query(1, ge=1, description="Page number"),
per_page: int = Query(50, ge=1, le=200, description="Items per page"), per_page: int = Query(50, ge=1, le=200, description="Items per page"),
sort_by: str = Query("created_at", description="Sort field: id, original_filename, file_size, mime_type, created_at, status"), sort_by: str = Query(
"created_at", description="Sort field: id, original_filename, file_size, mime_type, created_at, status"
),
sort_order: str = Query("desc", description="Sort order: asc or desc"), sort_order: str = Query("desc", description="Sort order: asc or desc"),
search: Optional[str] = Query(None, description="Search in filename"), search: Optional[str] = Query(None, description="Search in filename"),
mime_type: Optional[str] = Query(None, description="Filter by MIME type"), mime_type: Optional[str] = Query(None, description="Filter by MIME type"),
status: Optional[str] = Query(None, description="Filter by processing status") status: Optional[str] = Query(None, description="Filter by processing status"),
): ):
""" """
Returns a paginated JSON list of FileRecord entries with processing status. Returns a paginated JSON list of FileRecord entries with processing status.
Supports server-side sorting, filtering, and searching. Supports server-side sorting, filtering, and searching.
Query Parameters: Query Parameters:
- page: Page number (default: 1) - page: Page number (default: 1)
- per_page: Items per page (default: 50, max: 200) - per_page: Items per page (default: 50, max: 200)
@@ -48,7 +53,7 @@ def list_files_api(
- search: Search in filename - search: Search in filename
- mime_type: Filter by MIME type - mime_type: Filter by MIME type
- status: Filter by processing status (pending, processing, completed, failed) - status: Filter by processing status (pending, processing, completed, failed)
Example response: Example response:
{ {
"files": [...], "files": [...],
@@ -62,15 +67,15 @@ def list_files_api(
""" """
# Start with base query # Start with base query
query = db.query(FileRecord) query = db.query(FileRecord)
# Apply search filter # Apply search filter
if search: if search:
query = query.filter(FileRecord.original_filename.ilike(f"%{search}%")) query = query.filter(FileRecord.original_filename.ilike(f"%{search}%"))
# Apply MIME type filter # Apply MIME type filter
if mime_type: if mime_type:
query = query.filter(FileRecord.mime_type == mime_type) query = query.filter(FileRecord.mime_type == mime_type)
# Apply status filter (before pagination for correct counts) # Apply status filter (before pagination for correct counts)
if status: if status:
# Subquery to get file IDs matching the status # Subquery to get file IDs matching the status
@@ -80,87 +85,78 @@ def list_files_api(
query = query.filter(~FileRecord.id.in_(subq)) query = query.filter(~FileRecord.id.in_(subq))
elif status == "processing": elif status == "processing":
# Files with in_progress logs # Files with in_progress logs
subq = db.query(ProcessingLog.file_id).filter( subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "in_progress").distinct()
ProcessingLog.status == "in_progress"
).distinct()
query = query.filter(FileRecord.id.in_(subq)) query = query.filter(FileRecord.id.in_(subq))
elif status == "failed": elif status == "failed":
# Files with failure logs # Files with failure logs
subq = db.query(ProcessingLog.file_id).filter( subq = db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "failure").distinct()
ProcessingLog.status == "failure"
).distinct()
query = query.filter(FileRecord.id.in_(subq)) query = query.filter(FileRecord.id.in_(subq))
elif status == "completed": elif status == "completed":
# Files with success logs but no failures or in_progress # Files with success logs but no failures or in_progress
success_files = db.query(ProcessingLog.file_id).filter( success_files = (
ProcessingLog.status == "success" db.query(ProcessingLog.file_id).filter(ProcessingLog.status == "success").distinct().subquery()
).distinct().subquery() )
failed_files = db.query(ProcessingLog.file_id).filter( failed_files = (
or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress") db.query(ProcessingLog.file_id)
).distinct().subquery() .filter(or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress"))
.distinct()
query = query.filter( .subquery()
FileRecord.id.in_(db.query(success_files.c.file_id)) )
).filter(
query = query.filter(FileRecord.id.in_(db.query(success_files.c.file_id))).filter(
~FileRecord.id.in_(db.query(failed_files.c.file_id)) ~FileRecord.id.in_(db.query(failed_files.c.file_id))
) )
# Get total count before pagination (after all filters) # Get total count before pagination (after all filters)
total_items = query.count() total_items = query.count()
# Apply sorting # Apply sorting
sort_column = { sort_column = {
"id": FileRecord.id, "id": FileRecord.id,
"original_filename": FileRecord.original_filename, "original_filename": FileRecord.original_filename,
"file_size": FileRecord.file_size, "file_size": FileRecord.file_size,
"mime_type": FileRecord.mime_type, "mime_type": FileRecord.mime_type,
"created_at": FileRecord.created_at "created_at": FileRecord.created_at,
}.get(sort_by, FileRecord.created_at) }.get(sort_by, FileRecord.created_at)
if sort_order == "asc": if sort_order == "asc":
query = query.order_by(asc(sort_column)) query = query.order_by(asc(sort_column))
else: else:
query = query.order_by(desc(sort_column)) query = query.order_by(desc(sort_column))
# Apply pagination # Apply pagination
offset = (page - 1) * per_page offset = (page - 1) * per_page
files = query.offset(offset).limit(per_page).all() files = query.offset(offset).limit(per_page).all()
# Get processing status for all files efficiently # Get processing status for all files efficiently
file_ids = [f.id for f in files] file_ids = [f.id for f in files]
statuses = get_files_processing_status(db, file_ids) statuses = get_files_processing_status(db, file_ids)
# Build result with processing status # Build result with processing status
result = [] result = []
for f in files: for f in files:
result.append({ result.append(
"id": f.id, {
"filehash": f.filehash, "id": f.id,
"original_filename": f.original_filename, "filehash": f.filehash,
"local_filename": f.local_filename, "original_filename": f.original_filename,
"file_size": f.file_size, "local_filename": f.local_filename,
"mime_type": f.mime_type, "file_size": f.file_size,
"created_at": f.created_at.isoformat() if f.created_at else None, "mime_type": f.mime_type,
"processing_status": statuses.get(f.id, { "created_at": f.created_at.isoformat() if f.created_at else None,
"status": "pending", "processing_status": statuses.get(
"last_step": None, f.id, {"status": "pending", "last_step": None, "has_errors": False, "total_steps": 0}
"has_errors": False, ),
"total_steps": 0 }
}) )
})
# Calculate pagination info # Calculate pagination info
total_pages = (total_items + per_page - 1) // per_page total_pages = (total_items + per_page - 1) // per_page
return { return {
"files": result, "files": result,
"pagination": { "pagination": {"page": page, "per_page": per_page, "total_items": total_items, "total_pages": total_pages},
"page": page,
"per_page": per_page,
"total_items": total_items,
"total_pages": total_pages
}
} }
@@ -170,8 +166,8 @@ def _get_file_processing_status(db: Session, file_id: int) -> dict:
Kept for backward compatibility with file detail endpoint. Kept for backward compatibility with file detail endpoint.
""" """
from app.utils.file_status import get_file_processing_status from app.utils.file_status import get_file_processing_status
return get_file_processing_status(db, file_id)
return get_file_processing_status(db, file_id)
@router.get("/files/{file_id}") @router.get("/files/{file_id}")
@@ -182,38 +178,35 @@ def get_file_details(request: Request, file_id: int, db: Session = Depends(get_d
""" """
# Find the file record # Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record: if not file_record:
raise HTTPException( raise HTTPException(status_code=404, detail=f"File record with ID {file_id} not found")
status_code=404,
detail=f"File record with ID {file_id} not found"
)
# Get processing logs # Get processing logs
logs = db.query(ProcessingLog).filter( logs = (
ProcessingLog.file_id == file_id db.query(ProcessingLog).filter(ProcessingLog.file_id == file_id).order_by(ProcessingLog.timestamp.desc()).all()
).order_by(ProcessingLog.timestamp.desc()).all() )
# Build log list # Build log list
log_list = [] log_list = []
for log in logs: for log in logs:
log_list.append({ log_list.append(
"id": log.id, {
"task_id": log.task_id, "id": log.id,
"step_name": log.step_name, "task_id": log.task_id,
"status": log.status, "step_name": log.step_name,
"message": log.message, "status": log.status,
"timestamp": log.timestamp.isoformat() if log.timestamp else None "message": log.message,
}) "timestamp": log.timestamp.isoformat() if log.timestamp else None,
}
)
# Get processing status # Get processing status
processing_status = _get_file_processing_status(db, file_id) processing_status = _get_file_processing_status(db, file_id)
# Check if files exist on disk # Check if files exist on disk
files_on_disk = { files_on_disk = {"original": os.path.exists(file_record.local_filename) if file_record.local_filename else False}
"original": os.path.exists(file_record.local_filename) if file_record.local_filename else False
}
return { return {
"file": { "file": {
"id": file_record.id, "id": file_record.id,
@@ -222,13 +215,14 @@ def get_file_details(request: Request, file_id: int, db: Session = Depends(get_d
"local_filename": file_record.local_filename, "local_filename": file_record.local_filename,
"file_size": file_record.file_size, "file_size": file_record.file_size,
"mime_type": file_record.mime_type, "mime_type": file_record.mime_type,
"created_at": file_record.created_at.isoformat() if file_record.created_at else None "created_at": file_record.created_at.isoformat() if file_record.created_at else None,
}, },
"processing_status": processing_status, "processing_status": processing_status,
"logs": log_list, "logs": log_list,
"files_on_disk": files_on_disk "files_on_disk": files_on_disk,
} }
@router.delete("/files/{file_id}") @router.delete("/files/{file_id}")
@require_login @require_login
def delete_file_record(request: Request, file_id: int, db: Session = Depends(get_db)): def delete_file_record(request: Request, file_id: int, db: Session = Depends(get_db)):
@@ -238,42 +232,31 @@ def delete_file_record(request: Request, file_id: int, db: Session = Depends(get
""" """
# Check if file deletion is allowed # Check if file deletion is allowed
if not settings.allow_file_delete: if not settings.allow_file_delete:
raise HTTPException( raise HTTPException(status_code=403, detail="File deletion is disabled in the configuration")
status_code=403,
detail="File deletion is disabled in the configuration"
)
try: try:
# Find the file record # Find the file record
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
if not file_record: if not file_record:
raise HTTPException( raise HTTPException(status_code=404, detail=f"File record with ID {file_id} not found")
status_code=404,
detail=f"File record with ID {file_id} not found"
)
# Log the deletion # Log the deletion
logger.info(f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}") logger.info(f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}")
# Delete the record # Delete the record
db.delete(file_record) db.delete(file_record)
db.commit() db.commit()
return { return {"status": "success", "message": f"File record {file_id} deleted successfully"}
"status": "success",
"message": f"File record {file_id} deleted successfully"
}
except HTTPException: except HTTPException:
raise raise
except Exception as e: except Exception as e:
db.rollback() db.rollback()
logger.exception(f"Error deleting file record {file_id}: {str(e)}") logger.exception(f"Error deleting file record {file_id}: {str(e)}")
raise HTTPException( raise HTTPException(status_code=500, detail=f"Error deleting file record: {str(e)}")
status_code=500,
detail=f"Error deleting file record: {str(e)}"
)
@router.post("/files/bulk-delete") @router.post("/files/bulk-delete")
@require_login @require_login
@@ -284,48 +267,39 @@ def bulk_delete_files(request: Request, file_ids: List[int], db: Session = Depen
""" """
# Check if file deletion is allowed # Check if file deletion is allowed
if not settings.allow_file_delete: if not settings.allow_file_delete:
raise HTTPException( raise HTTPException(status_code=403, detail="File deletion is disabled in the configuration")
status_code=403,
detail="File deletion is disabled in the configuration"
)
try: try:
# Find all file records # Find all file records
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
if not file_records: if not file_records:
raise HTTPException( raise HTTPException(status_code=404, detail="No files found with the provided IDs")
status_code=404,
detail="No files found with the provided IDs"
)
deleted_count = len(file_records) deleted_count = len(file_records)
deleted_ids = [f.id for f in file_records] deleted_ids = [f.id for f in file_records]
# Log the deletion # Log the deletion
logger.info(f"Bulk deleting {deleted_count} file records: IDs={deleted_ids}") logger.info(f"Bulk deleting {deleted_count} file records: IDs={deleted_ids}")
# Delete all records # Delete all records
for file_record in file_records: for file_record in file_records:
db.delete(file_record) db.delete(file_record)
db.commit() db.commit()
return { return {
"status": "success", "status": "success",
"message": f"Successfully deleted {deleted_count} file records", "message": f"Successfully deleted {deleted_count} file records",
"deleted_ids": deleted_ids "deleted_ids": deleted_ids,
} }
except HTTPException: except HTTPException:
raise raise
except Exception as e: except Exception as e:
db.rollback() db.rollback()
logger.exception(f"Error bulk deleting file records: {str(e)}") logger.exception(f"Error bulk deleting file records: {str(e)}")
raise HTTPException( raise HTTPException(status_code=500, detail=f"Error bulk deleting file records: {str(e)}")
status_code=500,
detail=f"Error bulk deleting file records: {str(e)}"
)
@router.post("/files/bulk-reprocess") @router.post("/files/bulk-reprocess")
@@ -337,63 +311,56 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: Session = De
try: try:
# Find all file records # Find all file records
file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all()
if not file_records: if not file_records:
raise HTTPException( raise HTTPException(status_code=404, detail="No files found with the provided IDs")
status_code=404,
detail="No files found with the provided IDs"
)
task_ids = [] task_ids = []
processed_files = [] processed_files = []
errors = [] errors = []
for file_record in file_records: for file_record in file_records:
try: try:
# Check if local file exists # Check if local file exists
if not file_record.local_filename or not os.path.exists(file_record.local_filename): if not file_record.local_filename or not os.path.exists(file_record.local_filename):
errors.append({ errors.append(
"file_id": file_record.id, {
"filename": file_record.original_filename, "file_id": file_record.id,
"error": "Local file not found" "filename": file_record.original_filename,
}) "error": "Local file not found",
}
)
continue continue
# Queue the file for processing # Queue the file for processing
task = process_document.delay(file_record.local_filename) task = process_document.delay(file_record.local_filename)
task_ids.append(task.id) task_ids.append(task.id)
processed_files.append({ processed_files.append(
"file_id": file_record.id, {"file_id": file_record.id, "filename": file_record.original_filename, "task_id": task.id}
"filename": file_record.original_filename, )
"task_id": task.id
}) logger.info(
f"Reprocessing file: ID={file_record.id}, "
logger.info(f"Reprocessing file: ID={file_record.id}, Filename={file_record.original_filename}, TaskID={task.id}") f"Filename={file_record.original_filename}, TaskID={task.id}"
)
except Exception as e: except Exception as e:
logger.exception(f"Error reprocessing file {file_record.id}: {str(e)}") logger.exception(f"Error reprocessing file {file_record.id}: {str(e)}")
errors.append({ errors.append({"file_id": file_record.id, "filename": file_record.original_filename, "error": str(e)})
"file_id": file_record.id,
"filename": file_record.original_filename,
"error": str(e)
})
return { return {
"status": "success" if processed_files else "error", "status": "success" if processed_files else "error",
"message": f"Successfully queued {len(processed_files)} files for reprocessing", "message": f"Successfully queued {len(processed_files)} files for reprocessing",
"processed_files": processed_files, "processed_files": processed_files,
"errors": errors if errors else None, "errors": errors if errors else None,
"task_ids": task_ids "task_ids": task_ids,
} }
except HTTPException: except HTTPException:
raise raise
except Exception as e: except Exception as e:
logger.exception(f"Error bulk reprocessing files: {str(e)}") logger.exception(f"Error bulk reprocessing files: {str(e)}")
raise HTTPException( raise HTTPException(status_code=500, detail=f"Error bulk reprocessing files: {str(e)}")
status_code=500,
detail=f"Error bulk reprocessing files: {str(e)}"
)
@router.post("/ui-upload") @router.post("/ui-upload")
@@ -401,10 +368,10 @@ def bulk_reprocess_files(request: Request, file_ids: List[int], db: Session = De
async def ui_upload(request: Request, file: UploadFile = File(...)): async def ui_upload(request: Request, file: UploadFile = File(...)):
"""Endpoint to accept a user-uploaded file and enqueue it for processing.""" """Endpoint to accept a user-uploaded file and enqueue it for processing."""
workdir = settings.workdir workdir = settings.workdir
# Extract just the filename without any path components to prevent path traversal # Extract just the filename without any path components to prevent path traversal
safe_filename = os.path.basename(file.filename) safe_filename = os.path.basename(file.filename)
# Generate a unique filename with UUID to prevent overwriting and filename conflicts # Generate a unique filename with UUID to prevent overwriting and filename conflicts
unique_id = str(uuid.uuid4()) unique_id = str(uuid.uuid4())
# Keep the original extension if present # Keep the original extension if present
@@ -413,34 +380,28 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
target_filename = f"{unique_id}.{file_extension}" target_filename = f"{unique_id}.{file_extension}"
else: else:
target_filename = unique_id target_filename = unique_id
# Store both the safe original name and the unique name # Store both the safe original name and the unique name
target_path = os.path.join(workdir, target_filename) target_path = os.path.join(workdir, target_filename)
try: try:
with open(target_path, "wb") as f: with open(target_path, "wb") as f:
content = await file.read() content = await file.read()
f.write(content) f.write(content)
except Exception as e: except Exception as e:
raise HTTPException( raise HTTPException(status_code=500, detail=f"Failed to save file: {e}")
status_code=500,
detail=f"Failed to save file: {e}"
)
# Log the mapping between original and safe filename # Log the mapping between original and safe filename
logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'") logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'")
# Check file size # Check file size
file_size = os.path.getsize(target_path) file_size = os.path.getsize(target_path)
max_size = 500 * 1024 * 1024 # 500MB max_size = 500 * 1024 * 1024 # 500MB
if file_size > max_size: if file_size > max_size:
# Remove the file if it's too large # Remove the file if it's too large
os.remove(target_path) os.remove(target_path)
raise HTTPException( raise HTTPException(status_code=413, detail=f"File too large: {file_size} bytes (max {max_size} bytes)")
status_code=413,
detail=f"File too large: {file_size} bytes (max {max_size} bytes)"
)
# Same set of allowed file types as in the IMAP task # Same set of allowed file types as in the IMAP task
ALLOWED_MIME_TYPES = { ALLOWED_MIME_TYPES = {
"application/pdf", "application/pdf",
@@ -455,30 +416,40 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
"application/rtf", "application/rtf",
"text/rtf", "text/rtf",
} }
# Image MIME types that need conversion # Image MIME types that need conversion
IMAGE_MIME_TYPES = { IMAGE_MIME_TYPES = {
'image/jpeg', 'image/jpg', 'image/png', "image/jpeg",
'image/gif', 'image/bmp', 'image/tiff', "image/jpg",
'image/webp', 'image/svg+xml' "image/png",
"image/gif",
"image/bmp",
"image/tiff",
"image/webp",
"image/svg+xml",
} }
# Determine if the file is a PDF or needs conversion # Determine if the file is a PDF or needs conversion
mime_type, _ = mimetypes.guess_type(target_path) mime_type, _ = mimetypes.guess_type(target_path)
file_ext = os.path.splitext(target_path)[1].lower() file_ext = os.path.splitext(target_path)[1].lower()
# Check if it's a PDF by extension or MIME type # Check if it's a PDF by extension or MIME type
is_pdf = file_ext == ".pdf" or mime_type == "application/pdf" is_pdf = file_ext == ".pdf" or mime_type == "application/pdf"
if is_pdf: if is_pdf:
# If it's a PDF, process directly # If it's a PDF, process directly
task = process_document.delay(target_path, original_filename=safe_filename) task = process_document.delay(target_path, original_filename=safe_filename)
logger.info(f"Enqueued PDF for processing: {target_path}") logger.info(f"Enqueued PDF for processing: {target_path}")
elif mime_type in IMAGE_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg']): elif mime_type in IMAGE_MIME_TYPES or any(
file_ext.endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".svg"]
):
# If it's an image, convert to PDF first # If it's an image, convert to PDF first
task = convert_to_pdf.delay(target_path, original_filename=safe_filename) task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
logger.info(f"Enqueued image for PDF conversion: {target_path}") logger.info(f"Enqueued image for PDF conversion: {target_path}")
elif mime_type in ALLOWED_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.odt', '.ods', '.odp', '.rtf', '.txt', '.csv']): elif mime_type in ALLOWED_MIME_TYPES or any(
file_ext.endswith(ext)
for ext in [".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".odt", ".ods", ".odp", ".rtf", ".txt", ".csv"]
):
# If it's an office document, convert to PDF first # If it's an office document, convert to PDF first
task = convert_to_pdf.delay(target_path, original_filename=safe_filename) task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
logger.info(f"Enqueued office document for PDF conversion: {target_path}") logger.info(f"Enqueued office document for PDF conversion: {target_path}")
@@ -486,10 +457,10 @@ async def ui_upload(request: Request, file: UploadFile = File(...)):
# For any other file type, attempt conversion but log a warning # For any other file type, attempt conversion but log a warning
logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion") logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion")
task = convert_to_pdf.delay(target_path, original_filename=safe_filename) task = convert_to_pdf.delay(target_path, original_filename=safe_filename)
return { return {
"task_id": task.id, "task_id": task.id,
"status": "queued", "status": "queued",
"original_filename": safe_filename, "original_filename": safe_filename,
"stored_filename": target_filename "stored_filename": target_filename,
} }
app/tasks/convert_to_pdf.py
+94 -69
View File
@@ -1,23 +1,25 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import os
import requests
import logging import logging
import mimetypes import mimetypes
import json import os
import requests
from celery import shared_task from celery import shared_task
from app.config import settings from app.config import settings
from app.tasks.process_document import process_document from app.tasks.process_document import process_document
from app.utils import log_task_progress from app.utils import log_task_progress
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@shared_task(bind=True) @shared_task(bind=True)
def convert_to_pdf(self, file_path, original_filename=None): def convert_to_pdf(self, file_path, original_filename=None):
""" """
Converts a file to PDF using Gotenberg's API. Converts a file to PDF using Gotenberg's API.
Determines the appropriate Gotenberg endpoint based on the file's MIME type. Determines the appropriate Gotenberg endpoint based on the file's MIME type.
On success, saves the PDF locally and enqueues it for processing. On success, saves the PDF locally and enqueues it for processing.
Args: Args:
file_path: Path to the file to convert file_path: Path to the file to convert
original_filename: Optional original filename (if different from path basename) original_filename: Optional original filename (if different from path basename)
@@ -25,7 +27,7 @@ def convert_to_pdf(self, file_path, original_filename=None):
task_id = self.request.id task_id = self.request.id
logger.info(f"[{task_id}] Starting PDF conversion: {file_path}") logger.info(f"[{task_id}] Starting PDF conversion: {file_path}")
log_task_progress(task_id, "convert_to_pdf", "in_progress", f"Converting file: {os.path.basename(file_path)}") log_task_progress(task_id, "convert_to_pdf", "in_progress", f"Converting file: {os.path.basename(file_path)}")
gotenberg_url = getattr(settings, "gotenberg_url", None) gotenberg_url = getattr(settings, "gotenberg_url", None)
if not gotenberg_url: if not gotenberg_url:
logger.error(f"[{task_id}] Gotenberg URL is not configured in settings.") logger.error(f"[{task_id}] Gotenberg URL is not configured in settings.")
@@ -42,68 +44,89 @@ def convert_to_pdf(self, file_path, original_filename=None):
endpoint = None endpoint = None
form_data = {} form_data = {}
files = {} files = {}
# Dictionary mapping file extensions to their handlers # Dictionary mapping file extensions to their handlers
OFFICE_EXTENSIONS = { OFFICE_EXTENSIONS = {
'.doc', '.docx', '.docm', '.dot', '.dotx', '.dotm', # Word ".doc",
'.xls', '.xlsx', '.xlsm', '.xlsb', '.xlt', '.xltx', '.xlw', # Excel ".docx",
'.ppt', '.pptx', '.pptm', '.pps', '.ppsx', '.pot', '.potx', # PowerPoint ".docm",
'.odt', '.ods', '.odp', '.odg', '.odf', # OpenOffice/LibreOffice ".dot",
'.rtf', '.txt', '.csv', # Text formats ".dotx",
'.pdf', # PDF (already in PDF format but can be processed) ".dotm", # Word
".xls",
".xlsx",
".xlsm",
".xlsb",
".xlt",
".xltx",
".xlw", # Excel
".ppt",
".pptx",
".pptm",
".pps",
".ppsx",
".pot",
".potx", # PowerPoint
".odt",
".ods",
".odp",
".odg",
".odf", # OpenOffice/LibreOffice
".rtf",
".txt",
".csv", # Text formats
".pdf", # PDF (already in PDF format but can be processed)
} }
IMAGE_EXTENSIONS = { IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".webp", ".svg"}
'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif', '.webp', '.svg'
} HTML_EXTENSIONS = {".html", ".htm"}
HTML_EXTENSIONS = {
'.html', '.htm'
}
# Use LibreOffice endpoint for office documents and images # Use LibreOffice endpoint for office documents and images
if (mime_type and 'office' in mime_type) or \ if (
(mime_type and 'opendocument' in mime_type) or \ (mime_type and "office" in mime_type)
(mime_type and mime_type.startswith('image/')) or \ or (mime_type and "opendocument" in mime_type)
file_ext in OFFICE_EXTENSIONS or \ or (mime_type and mime_type.startswith("image/"))
file_ext in IMAGE_EXTENSIONS: or file_ext in OFFICE_EXTENSIONS
or file_ext in IMAGE_EXTENSIONS
):
endpoint = f"{gotenberg_url}/forms/libreoffice/convert" endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
files = {'files': (os.path.basename(file_path), open(file_path, 'rb'))} files = {"files": (os.path.basename(file_path), open(file_path, "rb"))}
# Add some quality settings for better PDF output # Add some quality settings for better PDF output
form_data = { form_data = {
'landscape': 'false', "landscape": "false",
'exportBookmarks': 'true', "exportBookmarks": "true",
'exportNotes': 'false', "exportNotes": "false",
'losslessImageCompression': 'true', # Use lossless compression for images "losslessImageCompression": "true", # Use lossless compression for images
'pdfa': 'PDF/A-2b', # Produce PDF/A-2b compatible output "pdfa": "PDF/A-2b", # Produce PDF/A-2b compatible output
} }
# Use Chromium endpoint for HTML documents # Use Chromium endpoint for HTML documents
elif (mime_type and mime_type == 'text/html') or file_ext in HTML_EXTENSIONS: elif (mime_type and mime_type == "text/html") or file_ext in HTML_EXTENSIONS:
endpoint = f"{gotenberg_url}/forms/chromium/convert/html" endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
# Gotenberg requires the form field to be exactly 'index.html' # Gotenberg requires the form field to be exactly 'index.html'
# The content filename doesn't matter, just the form field key # The content filename doesn't matter, just the form field key
files = {'index.html': ('index.html', open(file_path, 'rb'))} files = {"index.html": ("index.html", open(file_path, "rb"))}
# Add options for better HTML to PDF conversion # Add options for better HTML to PDF conversion
form_data = { form_data = {
'paperWidth': '8.27', # A4 width in inches "paperWidth": "8.27", # A4 width in inches
'paperHeight': '11.7', # A4 height in inches "paperHeight": "11.7", # A4 height in inches
'marginTop': '0.4', "marginTop": "0.4",
'marginBottom': '0.4', "marginBottom": "0.4",
'marginLeft': '0.4', "marginLeft": "0.4",
'marginRight': '0.4', "marginRight": "0.4",
'printBackground': 'true', "printBackground": "true",
'preferCssPageSize': 'false', "preferCssPageSize": "false",
'waitDelay': '2s', # Wait for JavaScript to execute "waitDelay": "2s", # Wait for JavaScript to execute
} }
# Use Markdown route for markdown files # Use Markdown route for markdown files
elif (mime_type and mime_type in ['text/markdown', 'text/x-markdown']) or file_ext in ['.md', '.markdown']: elif (mime_type and mime_type in ["text/markdown", "text/x-markdown"]) or file_ext in [".md", ".markdown"]:
# For Markdown, we need both the markdown file and an HTML wrapper # For Markdown, we need both the markdown file and an HTML wrapper
endpoint = f"{gotenberg_url}/forms/chromium/convert/markdown" endpoint = f"{gotenberg_url}/forms/chromium/convert/markdown"
# Create a simple HTML wrapper for the markdown # Create a simple HTML wrapper for the markdown
# IMPORTANT: The filename in the template must match the key used in the files dictionary # IMPORTANT: The filename in the template must match the key used in the files dictionary
markdown_filename = os.path.basename(file_path) markdown_filename = os.path.basename(file_path)
@@ -125,35 +148,35 @@ def convert_to_pdf(self, file_path, original_filename=None):
{{{{ toHTML "{markdown_filename}" }}}} {{{{ toHTML "{markdown_filename}" }}}}
</body> </body>
</html>""" </html>"""
# Create a temporary HTML wrapper file # Create a temporary HTML wrapper file
wrapper_path = os.path.join(os.path.dirname(file_path), "md_wrapper.html") wrapper_path = os.path.join(os.path.dirname(file_path), "md_wrapper.html")
with open(wrapper_path, 'w') as f: with open(wrapper_path, "w") as f:
f.write(html_wrapper) f.write(html_wrapper)
try: try:
files = { files = {
'index.html': ('index.html', open(wrapper_path, 'rb')), "index.html": ("index.html", open(wrapper_path, "rb")),
markdown_filename: (markdown_filename, open(file_path, 'rb')) markdown_filename: (markdown_filename, open(file_path, "rb")),
} }
form_data = { form_data = {
'paperWidth': '8.27', # A4 width in inches "paperWidth": "8.27", # A4 width in inches
'paperHeight': '11.7', # A4 height in inches "paperHeight": "11.7", # A4 height in inches
'marginTop': '0.4', "marginTop": "0.4",
'marginBottom': '0.4', "marginBottom": "0.4",
'marginLeft': '0.4', "marginLeft": "0.4",
'marginRight': '0.4', "marginRight": "0.4",
} }
finally: finally:
# Clean up the temporary wrapper file after preparing the request # Clean up the temporary wrapper file after preparing the request
if os.path.exists(wrapper_path): if os.path.exists(wrapper_path):
os.remove(wrapper_path) os.remove(wrapper_path)
# Fallback to LibreOffice for everything else # Fallback to LibreOffice for everything else
else: else:
endpoint = f"{gotenberg_url}/forms/libreoffice/convert" endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
files = {'files': (os.path.basename(file_path), open(file_path, 'rb'))} files = {"files": (os.path.basename(file_path), open(file_path, "rb"))}
logger.warning(f"Using fallback conversion for unknown type: {mime_type} / {file_ext}") logger.warning(f"Using fallback conversion for unknown type: {mime_type} / {file_ext}")
if not endpoint: if not endpoint:
@@ -164,20 +187,22 @@ def convert_to_pdf(self, file_path, original_filename=None):
try: try:
logger.info(f"[{task_id}] Converting {file_path} using endpoint: {endpoint}") logger.info(f"[{task_id}] Converting {file_path} using endpoint: {endpoint}")
log_task_progress(task_id, "call_gotenberg", "in_progress", "Calling Gotenberg API") log_task_progress(task_id, "call_gotenberg", "in_progress", "Calling Gotenberg API")
# Send the conversion request to Gotenberg # Send the conversion request to Gotenberg
response = requests.post(endpoint, files=files, data=form_data) response = requests.post(endpoint, files=files, data=form_data)
if response.status_code == 200: if response.status_code == 200:
# Save the converted PDF # Save the converted PDF
converted_file_path = os.path.splitext(file_path)[0] + ".pdf" converted_file_path = os.path.splitext(file_path)[0] + ".pdf"
with open(converted_file_path, "wb") as out_file: with open(converted_file_path, "wb") as out_file:
out_file.write(response.content) out_file.write(response.content)
logger.info(f"[{task_id}] Converted file saved as PDF: {converted_file_path}") logger.info(f"[{task_id}] Converted file saved as PDF: {converted_file_path}")
log_task_progress(task_id, "call_gotenberg", "success", "PDF conversion successful") log_task_progress(task_id, "call_gotenberg", "success", "PDF conversion successful")
log_task_progress(task_id, "convert_to_pdf", "success", f"Converted to PDF: {os.path.basename(converted_file_path)}") log_task_progress(
task_id, "convert_to_pdf", "success", f"Converted to PDF: {os.path.basename(converted_file_path)}"
)
# Enqueue the PDF for further processing, preserving original filename if provided # Enqueue the PDF for further processing, preserving original filename if provided
if original_filename: if original_filename:
# Change extension to .pdf for the original filename # Change extension to .pdf for the original filename
@@ -186,7 +211,7 @@ def convert_to_pdf(self, file_path, original_filename=None):
process_document.delay(converted_file_path, original_filename=pdf_original_filename) process_document.delay(converted_file_path, original_filename=pdf_original_filename)
else: else:
process_document.delay(converted_file_path) process_document.delay(converted_file_path)
return converted_file_path return converted_file_path
else: else:
error_msg = f"Status code: {response.status_code}" error_msg = f"Status code: {response.status_code}"
app/tasks/process_document.py
+15 -24
View File
@@ -1,21 +1,22 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import os
import uuid
import shutil
import mimetypes
import logging import logging
import mimetypes
import os
import shutil
import uuid
import PyPDF2 # Replace fitz with PyPDF2 import PyPDF2 # Replace fitz with PyPDF2
from app.celery_app import celery
from app.config import settings from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry from app.database import SessionLocal
from app.models import FileRecord
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
from app.tasks.process_with_azure_document_intelligence import ( from app.tasks.process_with_azure_document_intelligence import (
process_with_azure_document_intelligence, process_with_azure_document_intelligence,
) )
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.database import SessionLocal
from app.models import FileRecord
from app.utils import hash_file, log_task_progress from app.utils import hash_file, log_task_progress
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -63,9 +64,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
if not mime_type: if not mime_type:
mime_type = "application/octet-stream" mime_type = "application/octet-stream"
logger.info( logger.info(f"[{task_id}] File hash: {filehash[:10]}..., Size: {file_size} bytes, MIME: {mime_type}")
f"[{task_id}] File hash: {filehash[:10]}..., Size: {file_size} bytes, MIME: {mime_type}"
)
log_task_progress( log_task_progress(
task_id, task_id,
"hash_file", "hash_file",
@@ -77,9 +76,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
with SessionLocal() as db: with SessionLocal() as db:
existing = db.query(FileRecord).filter_by(filehash=filehash).one_or_none() existing = db.query(FileRecord).filter_by(filehash=filehash).one_or_none()
if existing: if existing:
logger.info( logger.info(f"[{task_id}] Duplicate file detected (hash={filehash[:10]}...) Skipping processing.")
f"[{task_id}] Duplicate file detected (hash={filehash[:10]}...) Skipping processing."
)
log_task_progress( log_task_progress(
task_id, task_id,
"process_document", "process_document",
@@ -95,9 +92,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
# Not a duplicate -> insert a new record # Not a duplicate -> insert a new record
logger.info(f"[{task_id}] Creating new file record in database") logger.info(f"[{task_id}] Creating new file record in database")
log_task_progress( log_task_progress(task_id, "create_file_record", "in_progress", "Creating file record")
task_id, "create_file_record", "in_progress", "Creating file record"
)
new_record = FileRecord( new_record = FileRecord(
filehash=filehash, filehash=filehash,
original_filename=original_filename, original_filename=original_filename,
@@ -169,9 +164,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
break break
if has_text: if has_text:
logger.info( logger.info(f"[{task_id}] PDF {original_local_file} contains embedded text. Processing locally.")
f"[{task_id}] PDF {original_local_file} contains embedded text. Processing locally."
)
log_task_progress( log_task_progress(
task_id, task_id,
"check_text", "check_text",
@@ -221,9 +214,7 @@ def process_document(self, original_local_file: str, original_filename: str = No
} }
# 3. If no embedded text, queue Azure Document Intelligence processing # 3. If no embedded text, queue Azure Document Intelligence processing
logger.info( logger.info(f"[{task_id}] No embedded text found. Queueing Azure Document Intelligence processing")
f"[{task_id}] No embedded text found. Queueing Azure Document Intelligence processing"
)
log_task_progress( log_task_progress(
task_id, task_id,
"check_text", "check_text",
tests/test_original_filename_preservation.py
+16 -15
View File
@@ -5,13 +5,12 @@ These tests verify the fix for the issue where uploaded files do not maintain
their original file names. their original file names.
""" """
import os from unittest.mock import MagicMock, patch
import pytest
from unittest.mock import patch, MagicMock import pytest
from sqlalchemy.orm import Session
from app.tasks.process_document import process_document
from app.models import FileRecord from app.models import FileRecord
from app.tasks.process_document import process_document
@pytest.mark.unit @pytest.mark.unit
@@ -83,15 +82,16 @@ startxref
%%EOF %%EOF
""" """
test_pdf.write_bytes(pdf_content) test_pdf.write_bytes(pdf_content)
# The original filename that the user uploaded # The original filename that the user uploaded
original_filename = "Apostille Sverige.pdf" original_filename = "Apostille Sverige.pdf"
# Mock environment and dependencies # Mock environment and dependencies
with patch("app.tasks.process_document.SessionLocal") as mock_session_local, \ with patch("app.tasks.process_document.SessionLocal") as mock_session_local, patch(
patch("app.tasks.process_document.settings") as mock_settings, \ "app.tasks.process_document.settings"
patch("app.tasks.process_document.log_task_progress"), \ ) as mock_settings, patch("app.tasks.process_document.log_task_progress"), patch(
patch("app.tasks.process_document.extract_metadata_with_gpt") as mock_extract: "app.tasks.process_document.extract_metadata_with_gpt"
) as mock_extract:
# Setup mocks # Setup mocks
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)
@@ -109,7 +109,7 @@ startxref
# Verify that a FileRecord was created with the correct original filename # Verify that a FileRecord was created with the correct original filename
file_record = db_session.query(FileRecord).first() file_record = db_session.query(FileRecord).first()
assert file_record is not None assert file_record is not None
# This is the key assertion - the original filename should be preserved # This is the key assertion - the original filename should be preserved
assert file_record.original_filename == original_filename assert file_record.original_filename == original_filename
# The filename should NOT be the UUID-based filename # The filename should NOT be the UUID-based filename
@@ -187,10 +187,11 @@ startxref
test_pdf.write_bytes(pdf_content) test_pdf.write_bytes(pdf_content)
# Mock environment and dependencies # Mock environment and dependencies
with patch("app.tasks.process_document.SessionLocal") as mock_session_local, \ with patch("app.tasks.process_document.SessionLocal") as mock_session_local, patch(
patch("app.tasks.process_document.settings") as mock_settings, \ "app.tasks.process_document.settings"
patch("app.tasks.process_document.log_task_progress"), \ ) as mock_settings, patch("app.tasks.process_document.log_task_progress"), patch(
patch("app.tasks.process_document.extract_metadata_with_gpt") as mock_extract: "app.tasks.process_document.extract_metadata_with_gpt"
) as mock_extract:
# Setup mocks # Setup mocks
mock_settings.workdir = str(tmp_path) mock_settings.workdir = str(tmp_path)