Address code review feedback

- Fixed status filtering to occur before pagination for correct counts
- Resolved N+1 query problem by batch-fetching processing statuses
- Extracted status computation logic to shared utility function
- Changed sort indicator from ⬍ to ↕ for better browser compatibility
- Updated both API and view layers to use shared status utilities

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/api/files.py

@@ -16,6 +16,7 @@ from app.config import settings
 from app.api.common import get_db
 from app.tasks.process_document import process_document
 from app.tasks.convert_to_pdf import convert_to_pdf
+from app.utils.file_status import get_file_processing_status, get_files_processing_status
 
 # Set up logging
 logger = logging.getLogger(__name__)
@@ -70,7 +71,42 @@ def list_files_api(
     if mime_type:
         query = query.filter(FileRecord.mime_type == mime_type)
     
-    # Get total count before pagination
+    # Apply status filter (before pagination for correct counts)
+    if status:
+        # Subquery to get file IDs matching the status
+        if status == "pending":
+            # Files with no logs
+            subq = db.query(ProcessingLog.file_id).distinct()
+            query = query.filter(~FileRecord.id.in_(subq))
+        elif status == "processing":
+            # Files with in_progress logs
+            subq = db.query(ProcessingLog.file_id).filter(
+                ProcessingLog.status == "in_progress"
+            ).distinct()
+            query = query.filter(FileRecord.id.in_(subq))
+        elif status == "failed":
+            # Files with failure logs
+            subq = db.query(ProcessingLog.file_id).filter(
+                ProcessingLog.status == "failure"
+            ).distinct()
+            query = query.filter(FileRecord.id.in_(subq))
+        elif status == "completed":
+            # Files with success logs but no failures or in_progress
+            success_files = db.query(ProcessingLog.file_id).filter(
+                ProcessingLog.status == "success"
+            ).distinct().subquery()
+            
+            failed_files = db.query(ProcessingLog.file_id).filter(
+                or_(ProcessingLog.status == "failure", ProcessingLog.status == "in_progress")
+            ).distinct().subquery()
+            
+            query = query.filter(
+                FileRecord.id.in_(db.query(success_files.c.file_id))
+            ).filter(
+                ~FileRecord.id.in_(db.query(failed_files.c.file_id))
+            )
+    
+    # Get total count before pagination (after all filters)
     total_items = query.count()
     
     # Apply sorting
@@ -91,12 +127,13 @@ def list_files_api(
     offset = (page - 1) * per_page
     files = query.offset(offset).limit(per_page).all()
     
+    # Get processing status for all files efficiently
+    file_ids = [f.id for f in files]
+    statuses = get_files_processing_status(db, file_ids)
+    
     # Build result with processing status
     result = []
     for f in files:
-        # Get processing status for this file
-        processing_status = _get_file_processing_status(db, f.id)
-        
         result.append({
             "id": f.id,
             "filehash": f.filehash,
@@ -105,14 +142,14 @@ def list_files_api(
             "file_size": f.file_size,
             "mime_type": f.mime_type,
             "created_at": f.created_at.isoformat() if f.created_at else None,
-            "processing_status": processing_status
+            "processing_status": statuses.get(f.id, {
+                "status": "pending",
+                "last_step": None,
+                "has_errors": False,
+                "total_steps": 0
+            })
         })
     
-    # Filter by status if requested (after computing statuses)
-    if status:
-        result = [f for f in result if f["processing_status"]["status"] == status]
-        total_items = len(result)  # Update total for filtered results
-    
     # Calculate pagination info
     total_pages = (total_items + per_page - 1) // per_page
     
@@ -129,49 +166,11 @@ def list_files_api(
 
 def _get_file_processing_status(db: Session, file_id: int) -> dict:
     """
-    Get the processing status for a file by checking its processing logs.
+    Deprecated: Use app.utils.file_status.get_file_processing_status instead.
-    
+    Kept for backward compatibility with file detail endpoint.
-    Returns:
-        dict with status, last_step, and has_errors
     """
-    # Get all logs for this file
+    from app.utils.file_status import get_file_processing_status
-    logs = db.query(ProcessingLog).filter(
+    return get_file_processing_status(db, file_id)
-        ProcessingLog.file_id == file_id
-    ).order_by(ProcessingLog.timestamp.desc()).all()
-    
-    if not logs:
-        return {
-            "status": "pending",
-            "last_step": None,
-            "has_errors": False,
-            "total_steps": 0
-        }
-    
-    # Check for failures
-    has_errors = any(log.status == "failure" for log in logs)
-    
-    # Check if any in progress
-    in_progress = any(log.status == "in_progress" for log in logs)
-    
-    # Get the latest log
-    latest_log = logs[0]
-    
-    # Determine overall status
-    if has_errors:
-        status = "failed"
-    elif in_progress:
-        status = "processing"
-    elif latest_log.status == "success":
-        status = "completed"
-    else:
-        status = "pending"
-    
-    return {
-        "status": status,
-        "last_step": latest_log.step_name,
-        "has_errors": has_errors,
-        "total_steps": len(logs)
-    }
 
 
 

app/utils/file_status.py

@@ -0,0 +1,102 @@
+"""
+Utility functions for file processing status determination.
+"""
+from typing import Dict, List
+from sqlalchemy.orm import Session
+from app.models import ProcessingLog
+
+
+def get_file_processing_status(db: Session, file_id: int) -> Dict:
+    """
+    Get the processing status for a file by checking its processing logs.
+    
+    Args:
+        db: Database session
+        file_id: ID of the file
+    
+    Returns:
+        dict with status, last_step, and has_errors
+    """
+    # Get all logs for this file
+    logs = db.query(ProcessingLog).filter(
+        ProcessingLog.file_id == file_id
+    ).order_by(ProcessingLog.timestamp.desc()).all()
+    
+    return _compute_status_from_logs(logs)
+
+
+def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, Dict]:
+    """
+    Get processing status for multiple files efficiently.
+    
+    Args:
+        db: Database session
+        file_ids: List of file IDs
+    
+    Returns:
+        dict mapping file_id to status dict
+    """
+    # Get all logs for these files in one query
+    logs = db.query(ProcessingLog).filter(
+        ProcessingLog.file_id.in_(file_ids)
+    ).order_by(ProcessingLog.file_id, ProcessingLog.timestamp.desc()).all()
+    
+    # Group logs by file_id
+    logs_by_file = {}
+    for log in logs:
+        if log.file_id not in logs_by_file:
+            logs_by_file[log.file_id] = []
+        logs_by_file[log.file_id].append(log)
+    
+    # Compute status for each file
+    result = {}
+    for file_id in file_ids:
+        file_logs = logs_by_file.get(file_id, [])
+        result[file_id] = _compute_status_from_logs(file_logs)
+    
+    return result
+
+
+def _compute_status_from_logs(logs: List[ProcessingLog]) -> Dict:
+    """
+    Compute processing status from a list of processing logs.
+    
+    Args:
+        logs: List of ProcessingLog objects (should be ordered by timestamp desc)
+    
+    Returns:
+        dict with status, last_step, has_errors, and total_steps
+    """
+    if not logs:
+        return {
+            "status": "pending",
+            "last_step": None,
+            "has_errors": False,
+            "total_steps": 0
+        }
+    
+    # Check for failures
+    has_errors = any(log.status == "failure" for log in logs)
+    
+    # Check if any in progress
+    in_progress = any(log.status == "in_progress" for log in logs)
+    
+    # Get the latest log
+    latest_log = logs[0]
+    
+    # Determine overall status
+    if has_errors:
+        status = "failed"
+    elif in_progress:
+        status = "processing"
+    elif latest_log.status == "success":
+        status = "completed"
+    else:
+        status = "pending"
+    
+    return {
+        "status": status,
+        "last_step": latest_log.step_name,
+        "has_errors": has_errors,
+        "total_steps": len(logs)
+    }

app/views/files.py

@@ -6,6 +6,7 @@ from sqlalchemy.orm import Session
 from typing import Optional
 
 from app.views.base import APIRouter, templates, require_login, get_db, logger
+from app.utils.file_status import get_files_processing_status
 
 router = APIRouter()
 
@@ -62,33 +63,14 @@ def files_page(
         offset = (page - 1) * per_page
         files = query.offset(offset).limit(per_page).all()
         
-        # Add processing status to each file
+        # Get processing status for all files efficiently (avoids N+1)
+        file_ids = [f.id for f in files]
+        statuses = get_files_processing_status(db, file_ids)
+        
+        # Add status to each file
         files_with_status = []
         for file in files:
-            # Get processing logs for this file
+            file.processing_status = statuses.get(file.id, {}).get("status", "pending")
-            logs = db.query(ProcessingLog).filter(
-                ProcessingLog.file_id == file.id
-            ).order_by(ProcessingLog.timestamp.desc()).all()
-            
-            # Determine status
-            if not logs:
-                processing_status = "pending"
-            else:
-                has_errors = any(log.status == "failure" for log in logs)
-                in_progress = any(log.status == "in_progress" for log in logs)
-                latest_log = logs[0]
-                
-                if has_errors:
-                    processing_status = "failed"
-                elif in_progress:
-                    processing_status = "processing"
-                elif latest_log.status == "success":
-                    processing_status = "completed"
-                else:
-                    processing_status = "pending"
-            
-            # Add status to file object as an attribute
-            file.processing_status = processing_status
             files_with_status.append(file)
         
         # Calculate pagination info

frontend/templates/files.html

@@ -308,7 +308,7 @@
               {% if sort_by == 'id' %}
                 {% if sort_order == 'asc' %}▲{% else %}▼{% endif %}
               {% else %}
-                ⬍
+                ↕
               {% endif %}
             </span>
           </th>
@@ -318,7 +318,7 @@
               {% if sort_by == 'original_filename' %}
                 {% if sort_order == 'asc' %}▲{% else %}▼{% endif %}
               {% else %}
-                ⬍
+                ↕
               {% endif %}
             </span>
           </th>
@@ -328,7 +328,7 @@
               {% if sort_by == 'file_size' %}
                 {% if sort_order == 'asc' %}▲{% else %}▼{% endif %}
               {% else %}
-                ⬍
+                ↕
               {% endif %}
             </span>
           </th>
@@ -338,7 +338,7 @@
               {% if sort_by == 'mime_type' %}
                 {% if sort_order == 'asc' %}▲{% else %}▼{% endif %}
               {% else %}
-                ⬍
+                ↕
               {% endif %}
             </span>
           </th>
@@ -349,7 +349,7 @@
               {% if sort_by == 'created_at' %}
                 {% if sort_order == 'asc' %}▲{% else %}▼{% endif %}
               {% else %}
-                ⬍
+                ↕
               {% endif %}
             </span>
           </th>