""" File-related API endpoints """ from fastapi import APIRouter, Request, HTTPException, Depends, UploadFile, File, Query from sqlalchemy.orm import Session from sqlalchemy import desc, asc, or_, func from typing import Optional, List import logging import os import uuid import mimetypes from app.auth import require_login from app.models import FileRecord, ProcessingLog from app.config import settings from app.api.common import get_db from app.tasks.process_document import process_document from app.tasks.convert_to_pdf import convert_to_pdf from app.utils.file_status import get_file_processing_status, get_files_processing_status # Set up logging logger = logging.getLogger(__name__) router = APIRouter() @router.get("/files") @require_login def list_files_api( request: Request, db: Session = Depends(get_db), page: int = Query(1, ge=1, description="Page number"), per_page: int = Query(50, ge=1, le=200, description="Items per page"), sort_by: str = Query("created_at", description="Sort field: id, original_filename, file_size, mime_type, created_at, status"), sort_order: str = Query("desc", description="Sort order: asc or desc"), search: Optional[str] = Query(None, description="Search in filename"), mime_type: Optional[str] = Query(None, description="Filter by MIME type"), status: Optional[str] = Query(None, description="Filter by processing status") ): """ Returns a paginated JSON list of FileRecord entries with processing status. Supports server-side sorting, filtering, and searching. Query Parameters: - page: Page number (default: 1) - per_page: Items per page (default: 50, max: 200) - sort_by: Field to sort by (default: created_at) - sort_order: asc or desc (default: desc) - search: Search in filename - mime_type: Filter by MIME type - status: Filter by processing status (pending, processing, completed, failed) Example response: { "files": [...], "pagination": { "page": 1, "per_page": 50, "total_items": 150, "total_pages": 3 } } """ # Start with base query query = db.query(FileRecord) # Apply search filter if search: query = query.filter(FileRecord.original_filename.ilike(f"%{search}%")) # Apply MIME type filter if mime_type: query = query.filter(FileRecord.mime_type == mime_type) # Apply status filter (before pagination for correct counts) if status: # Subquery to get file IDs matching the status if status == "pending": # Files with no logs subq = db.query(ProcessingLog.file_id).distinct() query = query.filter(~FileRecord.id.in_(subq)) elif status == "processing": # Files with in_progress logs subq = db.query(ProcessingLog.file_id).filter( ProcessingLog.status == "in_progress" ).distinct() query = query.filter(FileRecord.id.in_(subq)) elif status == "failed": # Files with failure logs subq = db.query(ProcessingLog.file_id).filter( ProcessingLog.status == "failure" ).distinct() query = query.filter(FileRecord.id.in_(subq)) elif status == "completed": # Files with success logs but no failures or in_progress success_files = db.query(ProcessingLog.file_id).filter( ProcessingLog.status == "success" ).distinct().subquery() failed_files = db.query(ProcessingLog.file_id).filter( or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress") ).distinct().subquery() query = query.filter( FileRecord.id.in_(db.query(success_files.c.file_id)) ).filter( ~FileRecord.id.in_(db.query(failed_files.c.file_id)) ) # Get total count before pagination (after all filters) total_items = query.count() # Apply sorting sort_column = { "id": FileRecord.id, "original_filename": FileRecord.original_filename, "file_size": FileRecord.file_size, "mime_type": FileRecord.mime_type, "created_at": FileRecord.created_at }.get(sort_by, FileRecord.created_at) if sort_order == "asc": query = query.order_by(asc(sort_column)) else: query = query.order_by(desc(sort_column)) # Apply pagination offset = (page - 1) * per_page files = query.offset(offset).limit(per_page).all() # Get processing status for all files efficiently file_ids = [f.id for f in files] statuses = get_files_processing_status(db, file_ids) # Build result with processing status result = [] for f in files: result.append({ "id": f.id, "filehash": f.filehash, "original_filename": f.original_filename, "local_filename": f.local_filename, "file_size": f.file_size, "mime_type": f.mime_type, "created_at": f.created_at.isoformat() if f.created_at else None, "processing_status": statuses.get(f.id, { "status": "pending", "last_step": None, "has_errors": False, "total_steps": 0 }) }) # Calculate pagination info total_pages = (total_items + per_page - 1) // per_page return { "files": result, "pagination": { "page": page, "per_page": per_page, "total_items": total_items, "total_pages": total_pages } } def _get_file_processing_status(db: Session, file_id: int) -> dict: """ Deprecated: Use app.utils.file_status.get_file_processing_status instead. Kept for backward compatibility with file detail endpoint. """ from app.utils.file_status import get_file_processing_status return get_file_processing_status(db, file_id) @router.get("/files/{file_id}") @require_login def get_file_details(request: Request, file_id: int, db: Session = Depends(get_db)): """ Get detailed information about a specific file including processing history. """ # Find the file record file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: raise HTTPException( status_code=404, detail=f"File record with ID {file_id} not found" ) # Get processing logs logs = db.query(ProcessingLog).filter( ProcessingLog.file_id == file_id ).order_by(ProcessingLog.timestamp.desc()).all() # Build log list log_list = [] for log in logs: log_list.append({ "id": log.id, "task_id": log.task_id, "step_name": log.step_name, "status": log.status, "message": log.message, "timestamp": log.timestamp.isoformat() if log.timestamp else None }) # Get processing status processing_status = _get_file_processing_status(db, file_id) # Check if files exist on disk files_on_disk = { "original": os.path.exists(file_record.local_filename) if file_record.local_filename else False } return { "file": { "id": file_record.id, "filehash": file_record.filehash, "original_filename": file_record.original_filename, "local_filename": file_record.local_filename, "file_size": file_record.file_size, "mime_type": file_record.mime_type, "created_at": file_record.created_at.isoformat() if file_record.created_at else None }, "processing_status": processing_status, "logs": log_list, "files_on_disk": files_on_disk } @router.delete("/files/{file_id}") @require_login def delete_file_record(request: Request, file_id: int, db: Session = Depends(get_db)): """ Delete a file record from the database. This only removes the database entry, not the actual file. """ # Check if file deletion is allowed if not settings.allow_file_delete: raise HTTPException( status_code=403, detail="File deletion is disabled in the configuration" ) try: # Find the file record file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first() if not file_record: raise HTTPException( status_code=404, detail=f"File record with ID {file_id} not found" ) # Log the deletion logger.info(f"Deleting file record: ID={file_id}, Filename={file_record.original_filename}") # Delete the record db.delete(file_record) db.commit() return { "status": "success", "message": f"File record {file_id} deleted successfully" } except HTTPException: raise except Exception as e: db.rollback() logger.exception(f"Error deleting file record {file_id}: {str(e)}") raise HTTPException( status_code=500, detail=f"Error deleting file record: {str(e)}" ) @router.post("/files/bulk-delete") @require_login def bulk_delete_files(request: Request, file_ids: List[int], db: Session = Depends(get_db)): """ Delete multiple file records from the database. This only removes the database entries, not the actual files. """ # Check if file deletion is allowed if not settings.allow_file_delete: raise HTTPException( status_code=403, detail="File deletion is disabled in the configuration" ) try: # Find all file records file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() if not file_records: raise HTTPException( status_code=404, detail="No files found with the provided IDs" ) deleted_count = len(file_records) deleted_ids = [f.id for f in file_records] # Log the deletion logger.info(f"Bulk deleting {deleted_count} file records: IDs={deleted_ids}") # Delete all records for file_record in file_records: db.delete(file_record) db.commit() return { "status": "success", "message": f"Successfully deleted {deleted_count} file records", "deleted_ids": deleted_ids } except HTTPException: raise except Exception as e: db.rollback() logger.exception(f"Error bulk deleting file records: {str(e)}") raise HTTPException( status_code=500, detail=f"Error bulk deleting file records: {str(e)}" ) @router.post("/files/bulk-reprocess") @require_login def bulk_reprocess_files(request: Request, file_ids: List[int], db: Session = Depends(get_db)): """ Reprocess multiple files by queuing them for processing. """ try: # Find all file records file_records = db.query(FileRecord).filter(FileRecord.id.in_(file_ids)).all() if not file_records: raise HTTPException( status_code=404, detail="No files found with the provided IDs" ) task_ids = [] processed_files = [] errors = [] for file_record in file_records: try: # Check if local file exists if not file_record.local_filename or not os.path.exists(file_record.local_filename): errors.append({ "file_id": file_record.id, "filename": file_record.original_filename, "error": "Local file not found" }) continue # Queue the file for processing task = process_document.delay(file_record.local_filename) task_ids.append(task.id) processed_files.append({ "file_id": file_record.id, "filename": file_record.original_filename, "task_id": task.id }) logger.info(f"Reprocessing file: ID={file_record.id}, Filename={file_record.original_filename}, TaskID={task.id}") except Exception as e: logger.exception(f"Error reprocessing file {file_record.id}: {str(e)}") errors.append({ "file_id": file_record.id, "filename": file_record.original_filename, "error": str(e) }) return { "status": "success" if processed_files else "error", "message": f"Successfully queued {len(processed_files)} files for reprocessing", "processed_files": processed_files, "errors": errors if errors else None, "task_ids": task_ids } except HTTPException: raise except Exception as e: logger.exception(f"Error bulk reprocessing files: {str(e)}") raise HTTPException( status_code=500, detail=f"Error bulk reprocessing files: {str(e)}" ) @router.post("/ui-upload") @require_login async def ui_upload(request: Request, file: UploadFile = File(...)): """Endpoint to accept a user-uploaded file and enqueue it for processing.""" workdir = settings.workdir # Extract just the filename without any path components to prevent path traversal safe_filename = os.path.basename(file.filename) # Generate a unique filename with UUID to prevent overwriting and filename conflicts unique_id = str(uuid.uuid4()) # Keep the original extension if present if "." in safe_filename: file_extension = safe_filename.rsplit(".", 1)[1] target_filename = f"{unique_id}.{file_extension}" else: target_filename = unique_id # Store both the safe original name and the unique name target_path = os.path.join(workdir, target_filename) try: with open(target_path, "wb") as f: content = await file.read() f.write(content) except Exception as e: raise HTTPException( status_code=500, detail=f"Failed to save file: {e}" ) # Log the mapping between original and safe filename logger.info(f"Saved uploaded file '{safe_filename}' as '{target_filename}'") # Check file size file_size = os.path.getsize(target_path) max_size = 500 * 1024 * 1024 # 500MB if file_size > max_size: # Remove the file if it's too large os.remove(target_path) raise HTTPException( status_code=413, detail=f"File too large: {file_size} bytes (max {max_size} bytes)" ) # Same set of allowed file types as in the IMAP task ALLOWED_MIME_TYPES = { "application/pdf", "application/msword", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.ms-powerpoint", "application/vnd.openxmlformats-officedocument.presentationml.presentation", "text/plain", "text/csv", "application/rtf", "text/rtf", } # Image MIME types that need conversion IMAGE_MIME_TYPES = { 'image/jpeg', 'image/jpg', 'image/png', 'image/gif', 'image/bmp', 'image/tiff', 'image/webp', 'image/svg+xml' } # Determine if the file is a PDF or needs conversion mime_type, _ = mimetypes.guess_type(target_path) file_ext = os.path.splitext(target_path)[1].lower() # Check if it's a PDF by extension or MIME type is_pdf = file_ext == ".pdf" or mime_type == "application/pdf" if is_pdf: # If it's a PDF, process directly task = process_document.delay(target_path) logger.info(f"Enqueued PDF for processing: {target_path}") elif mime_type in IMAGE_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg']): # If it's an image, convert to PDF first task = convert_to_pdf.delay(target_path) logger.info(f"Enqueued image for PDF conversion: {target_path}") elif mime_type in ALLOWED_MIME_TYPES or any(file_ext.endswith(ext) for ext in ['.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.odt', '.ods', '.odp', '.rtf', '.txt', '.csv']): # If it's an office document, convert to PDF first task = convert_to_pdf.delay(target_path) logger.info(f"Enqueued office document for PDF conversion: {target_path}") else: # For any other file type, attempt conversion but log a warning logger.warning(f"Unsupported MIME type {mime_type} for {target_path}, attempting conversion") task = convert_to_pdf.delay(target_path) return { "task_id": task.id, "status": "queued", "original_filename": safe_filename, "stored_filename": target_filename }