feat(duplication): implement duplicate file handling and detection across processing steps

app/utils/file_queries.py

@@ -100,6 +100,9 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
         query = query.filter(FileRecord.id.in_(db.query(subq.c.file_id)))
     elif status == "completed":
         # Files where all real steps are either success or skipped (no failures or in_progress)
+        # Exclude duplicates from completed
+        query = query.filter(FileRecord.is_duplicate.is_(False))
+        
         # Get files that have real steps
         files_with_real_steps = real_steps_subq.distinct().subquery()
 
@@ -115,5 +118,8 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
         query = query.filter(FileRecord.id.in_(db.query(files_with_real_steps.c.file_id))).filter(
             ~FileRecord.id.in_(db.query(files_with_issues.c.file_id))
         )
+    elif status == "duplicate":
+        # Files marked as duplicates
+        query = query.filter(FileRecord.is_duplicate.is_(True))
 
     return query

app/utils/file_status.py

@@ -6,7 +6,7 @@ from typing import Dict, List
 
 from sqlalchemy.orm import Session
 
-from app.models import FileProcessingStep, ProcessingLog
+from app.models import FileProcessingStep, FileRecord, ProcessingLog
 from app.utils.step_manager import get_file_overall_status, get_step_summary
 
 
@@ -23,6 +23,15 @@ def get_file_processing_status(db: Session, file_id: int) -> Dict:
     Returns:
         dict with status, last_step, and has_errors
     """
+    file_record = db.query(FileRecord).filter(FileRecord.id == file_id).one_or_none()
+    if file_record and file_record.is_duplicate:
+        return {
+            "status": "duplicate",
+            "last_step": "check_for_duplicates",
+            "has_errors": False,
+            "total_steps": 0,
+        }
+
     # Use the new status table approach
     overall_status = get_file_overall_status(db, file_id)
 
@@ -61,6 +70,12 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
     Returns:
         dict mapping file_id to status dict
     """
+    # Preload duplicate flags for all files
+    duplicate_flags = {
+        record.id: record.is_duplicate
+        for record in db.query(FileRecord.id, FileRecord.is_duplicate).filter(FileRecord.id.in_(file_ids)).all()
+    }
+
     # Define which steps are "real" status-determining steps
     from app.config import settings
     
@@ -107,6 +122,14 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
     # Compute status for each file
     result = {}
     for file_id in file_ids:
+        if duplicate_flags.get(file_id):
+            result[file_id] = {
+                "status": "duplicate",
+                "last_step": "check_for_duplicates",
+                "has_errors": False,
+                "total_steps": 0,
+            }
+            continue
         file_steps = steps_by_file.get(file_id, [])
         if not file_steps:
             result[file_id] = {"status": "pending", "last_step": None, "has_errors": False, "total_steps": 0}

app/utils/step_manager.py

@@ -11,7 +11,7 @@ from typing import Dict, List, Optional
 from sqlalchemy.orm import Session
 
 from app.config import settings
-from app.models import FileProcessingStep
+from app.models import FileProcessingStep, FileRecord
 
 # Define the expected processing steps for a standard file workflow
 # The "check_for_duplicates" step is conditionally included based on enable_deduplication setting
@@ -186,6 +186,19 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict:
             "in_progress_steps": 2
         }
     """
+    # If file is marked as duplicate, return duplicate status immediately
+    file_record = db.query(FileRecord).filter(FileRecord.id == file_id).one_or_none()
+    if file_record and file_record.is_duplicate:
+        return {
+            "status": "duplicate",
+            "has_errors": False,
+            "total_steps": 0,
+            "completed_steps": 0,
+            "failed_steps": 0,
+            "in_progress_steps": 0,
+            "skipped_steps": 0,
+        }
+
     # Define which steps are "real" status-determining steps
     # Only high-level logical steps, not implementation sub-steps
     REAL_MAIN_STEPS = {