fix: correct file status and metrics to use latest status per step

- Fix _compute_status_from_logs to track latest status per unique step
- Fix _compute_step_summary to count only latest status per step
- Add comprehensive tests for both fixes
- Resolves issue where completed files showed as "Processing"
- Resolves issue where metrics showed incorrect counts

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/utils/file_status.py

@@ -76,13 +76,20 @@ def _compute_status_from_logs(logs: List[ProcessingLog]) -> Dict:
     if not logs:
         return {"status": "pending", "last_step": None, "has_errors": False, "total_steps": 0}
 
-    # Check for failures
-    has_errors = any(log.status == "failure" for log in logs)
+    # Get the latest status for each unique step
+    # Since logs are ordered by timestamp desc, the first occurrence is the latest
+    latest_by_step = {}
+    for log in logs:
+        if log.step_name not in latest_by_step:
+            latest_by_step[log.step_name] = log
 
-    # Check if any in progress
-    in_progress = any(log.status == "in_progress" for log in logs)
+    # Check for failures in latest statuses
+    has_errors = any(log.status == "failure" for log in latest_by_step.values())
 
-    # Get the latest log
+    # Check if any step is currently in progress (based on latest status per step)
+    in_progress = any(log.status == "in_progress" for log in latest_by_step.values())
+
+    # Get the overall latest log
     latest_log = logs[0]
 
     # Determine overall status

app/views/files.py

@@ -347,8 +347,8 @@ def _compute_step_summary(logs):
     main_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0}
     upload_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0}
 
-    # Track which steps we've seen
-    main_steps_seen = set()
+    # Track latest status for each step (logs are ordered by timestamp desc)
+    main_steps_seen = {}
     upload_tasks_seen = {}
 
     for log in logs:
@@ -363,15 +363,20 @@ def _compute_step_summary(logs):
         is_upload = any(step_name.startswith(prefix) for prefix in upload_prefixes)
 
         if is_upload:
-            # Track latest status for each unique upload task
-            upload_tasks_seen[step_name] = status
+            # Track latest status for each unique upload task (first seen is latest)
+            if step_name not in upload_tasks_seen:
+                upload_tasks_seen[step_name] = status
         elif step_name in main_steps:
-            # Track latest status for main steps
-            main_steps_seen.add(step_name)
-            if status in main_counts:
-                main_counts[status] += 1
+            # Track latest status for main steps (first seen is latest)
+            if step_name not in main_steps_seen:
+                main_steps_seen[step_name] = status
 
-    # Count upload task statuses
+    # Count main step statuses from latest status per step
+    for task_status in main_steps_seen.values():
+        if task_status in main_counts:
+            main_counts[task_status] += 1
+
+    # Count upload task statuses from latest status per task
     for task_status in upload_tasks_seen.values():
         if task_status in upload_counts:
             upload_counts[task_status] += 1