diff --git a/app/api/files.py b/app/api/files.py index e8cd14b8..865250a0 100644 --- a/app/api/files.py +++ b/app/api/files.py @@ -16,6 +16,7 @@ from app.config import settings from app.api.common import get_db from app.tasks.process_document import process_document from app.tasks.convert_to_pdf import convert_to_pdf +from app.utils.file_status import get_file_processing_status, get_files_processing_status # Set up logging logger = logging.getLogger(__name__) @@ -70,7 +71,42 @@ def list_files_api( if mime_type: query = query.filter(FileRecord.mime_type == mime_type) - # Get total count before pagination + # Apply status filter (before pagination for correct counts) + if status: + # Subquery to get file IDs matching the status + if status == "pending": + # Files with no logs + subq = db.query(ProcessingLog.file_id).distinct() + query = query.filter(~FileRecord.id.in_(subq)) + elif status == "processing": + # Files with in_progress logs + subq = db.query(ProcessingLog.file_id).filter( + ProcessingLog.status == "in_progress" + ).distinct() + query = query.filter(FileRecord.id.in_(subq)) + elif status == "failed": + # Files with failure logs + subq = db.query(ProcessingLog.file_id).filter( + ProcessingLog.status == "failure" + ).distinct() + query = query.filter(FileRecord.id.in_(subq)) + elif status == "completed": + # Files with success logs but no failures or in_progress + success_files = db.query(ProcessingLog.file_id).filter( + ProcessingLog.status == "success" + ).distinct().subquery() + + failed_files = db.query(ProcessingLog.file_id).filter( + or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress") + ).distinct().subquery() + + query = query.filter( + FileRecord.id.in_(db.query(success_files.c.file_id)) + ).filter( + ~FileRecord.id.in_(db.query(failed_files.c.file_id)) + ) + + # Get total count before pagination (after all filters) total_items = query.count() # Apply sorting @@ -91,12 +127,13 @@ def list_files_api( offset = (page - 1) * per_page files = query.offset(offset).limit(per_page).all() + # Get processing status for all files efficiently + file_ids = [f.id for f in files] + statuses = get_files_processing_status(db, file_ids) + # Build result with processing status result = [] for f in files: - # Get processing status for this file - processing_status = _get_file_processing_status(db, f.id) - result.append({ "id": f.id, "filehash": f.filehash, @@ -105,14 +142,14 @@ def list_files_api( "file_size": f.file_size, "mime_type": f.mime_type, "created_at": f.created_at.isoformat() if f.created_at else None, - "processing_status": processing_status + "processing_status": statuses.get(f.id, { + "status": "pending", + "last_step": None, + "has_errors": False, + "total_steps": 0 + }) }) - # Filter by status if requested (after computing statuses) - if status: - result = [f for f in result if f["processing_status"]["status"] == status] - total_items = len(result) # Update total for filtered results - # Calculate pagination info total_pages = (total_items + per_page - 1) // per_page @@ -129,49 +166,11 @@ def list_files_api( def _get_file_processing_status(db: Session, file_id: int) -> dict: """ - Get the processing status for a file by checking its processing logs. - - Returns: - dict with status, last_step, and has_errors + Deprecated: Use app.utils.file_status.get_file_processing_status instead. + Kept for backward compatibility with file detail endpoint. """ - # Get all logs for this file - logs = db.query(ProcessingLog).filter( - ProcessingLog.file_id == file_id - ).order_by(ProcessingLog.timestamp.desc()).all() - - if not logs: - return { - "status": "pending", - "last_step": None, - "has_errors": False, - "total_steps": 0 - } - - # Check for failures - has_errors = any(log.status == "failure" for log in logs) - - # Check if any in progress - in_progress = any(log.status == "in_progress" for log in logs) - - # Get the latest log - latest_log = logs[0] - - # Determine overall status - if has_errors: - status = "failed" - elif in_progress: - status = "processing" - elif latest_log.status == "success": - status = "completed" - else: - status = "pending" - - return { - "status": status, - "last_step": latest_log.step_name, - "has_errors": has_errors, - "total_steps": len(logs) - } + from app.utils.file_status import get_file_processing_status + return get_file_processing_status(db, file_id) diff --git a/app/utils/file_status.py b/app/utils/file_status.py new file mode 100644 index 00000000..627533fd --- /dev/null +++ b/app/utils/file_status.py @@ -0,0 +1,102 @@ +""" +Utility functions for file processing status determination. +""" +from typing import Dict, List +from sqlalchemy.orm import Session +from app.models import ProcessingLog + + +def get_file_processing_status(db: Session, file_id: int) -> Dict: + """ + Get the processing status for a file by checking its processing logs. + + Args: + db: Database session + file_id: ID of the file + + Returns: + dict with status, last_step, and has_errors + """ + # Get all logs for this file + logs = db.query(ProcessingLog).filter( + ProcessingLog.file_id == file_id + ).order_by(ProcessingLog.timestamp.desc()).all() + + return _compute_status_from_logs(logs) + + +def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, Dict]: + """ + Get processing status for multiple files efficiently. + + Args: + db: Database session + file_ids: List of file IDs + + Returns: + dict mapping file_id to status dict + """ + # Get all logs for these files in one query + logs = db.query(ProcessingLog).filter( + ProcessingLog.file_id.in_(file_ids) + ).order_by(ProcessingLog.file_id, ProcessingLog.timestamp.desc()).all() + + # Group logs by file_id + logs_by_file = {} + for log in logs: + if log.file_id not in logs_by_file: + logs_by_file[log.file_id] = [] + logs_by_file[log.file_id].append(log) + + # Compute status for each file + result = {} + for file_id in file_ids: + file_logs = logs_by_file.get(file_id, []) + result[file_id] = _compute_status_from_logs(file_logs) + + return result + + +def _compute_status_from_logs(logs: List[ProcessingLog]) -> Dict: + """ + Compute processing status from a list of processing logs. + + Args: + logs: List of ProcessingLog objects (should be ordered by timestamp desc) + + Returns: + dict with status, last_step, has_errors, and total_steps + """ + if not logs: + return { + "status": "pending", + "last_step": None, + "has_errors": False, + "total_steps": 0 + } + + # Check for failures + has_errors = any(log.status == "failure" for log in logs) + + # Check if any in progress + in_progress = any(log.status == "in_progress" for log in logs) + + # Get the latest log + latest_log = logs[0] + + # Determine overall status + if has_errors: + status = "failed" + elif in_progress: + status = "processing" + elif latest_log.status == "success": + status = "completed" + else: + status = "pending" + + return { + "status": status, + "last_step": latest_log.step_name, + "has_errors": has_errors, + "total_steps": len(logs) + } diff --git a/app/views/files.py b/app/views/files.py index dccf2110..8da5c545 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -6,6 +6,7 @@ from sqlalchemy.orm import Session from typing import Optional from app.views.base import APIRouter, templates, require_login, get_db, logger +from app.utils.file_status import get_files_processing_status router = APIRouter() @@ -62,33 +63,14 @@ def files_page( offset = (page - 1) * per_page files = query.offset(offset).limit(per_page).all() - # Add processing status to each file + # Get processing status for all files efficiently (avoids N+1) + file_ids = [f.id for f in files] + statuses = get_files_processing_status(db, file_ids) + + # Add status to each file files_with_status = [] for file in files: - # Get processing logs for this file - logs = db.query(ProcessingLog).filter( - ProcessingLog.file_id == file.id - ).order_by(ProcessingLog.timestamp.desc()).all() - - # Determine status - if not logs: - processing_status = "pending" - else: - has_errors = any(log.status == "failure" for log in logs) - in_progress = any(log.status == "in_progress" for log in logs) - latest_log = logs[0] - - if has_errors: - processing_status = "failed" - elif in_progress: - processing_status = "processing" - elif latest_log.status == "success": - processing_status = "completed" - else: - processing_status = "pending" - - # Add status to file object as an attribute - file.processing_status = processing_status + file.processing_status = statuses.get(file.id, {}).get("status", "pending") files_with_status.append(file) # Calculate pagination info diff --git a/frontend/templates/files.html b/frontend/templates/files.html index 1174cc24..5fcfcc22 100644 --- a/frontend/templates/files.html +++ b/frontend/templates/files.html @@ -308,7 +308,7 @@ {% if sort_by == 'id' %} {% if sort_order == 'asc' %}▲{% else %}▼{% endif %} {% else %} - ⬍ + ↕ {% endif %} @@ -318,7 +318,7 @@ {% if sort_by == 'original_filename' %} {% if sort_order == 'asc' %}▲{% else %}▼{% endif %} {% else %} - ⬍ + ↕ {% endif %} @@ -328,7 +328,7 @@ {% if sort_by == 'file_size' %} {% if sort_order == 'asc' %}▲{% else %}▼{% endif %} {% else %} - ⬍ + ↕ {% endif %} @@ -338,7 +338,7 @@ {% if sort_by == 'mime_type' %} {% if sort_order == 'asc' %}▲{% else %}▼{% endif %} {% else %} - ⬍ + ↕ {% endif %} @@ -349,7 +349,7 @@ {% if sort_by == 'created_at' %} {% if sort_order == 'asc' %}▲{% else %}▼{% endif %} {% else %} - ⬍ + ↕ {% endif %}