feat(deduplication): implement configurable duplicate file detection

- Add enable_deduplication and show_deduplication_step config options
- Rename hash_file step to check_for_duplicates
- Make deduplication step conditional based on configuration
- Add is_duplicate and duplicate_of_id fields to FileRecord model
- Create database migration for new deduplication fields
- Update process_document task to log deduplication results
- Update step visualization to show/hide step based on config
- Update status calculations to include deduplication step conditionally
- Default: deduplication enabled, step displayed
- Can be configured to hide from UI while still processing

app/utils/file_queries.py

@@ -52,6 +52,8 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
 
     # Define which steps are "real" status-determining steps
     # Only high-level logical steps and actual upload destinations (not queue_* steps)
+    from app.config import settings
+    
     REAL_STEPS = {
         "create_file_record",
         "check_text",
@@ -73,6 +75,10 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
         "upload_to_email",
         "upload_to_s3",
     }
+    
+    # Add check_for_duplicates if deduplication is enabled
+    if settings.enable_deduplication:
+        REAL_STEPS.add("check_for_duplicates")
 
     # Filter to only real steps
     real_steps_subq = (

app/utils/file_status.py

@@ -62,6 +62,8 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
         dict mapping file_id to status dict
     """
     # Define which steps are "real" status-determining steps
+    from app.config import settings
+    
     REAL_STEPS = {
         "create_file_record",
         "check_text",
@@ -83,6 +85,10 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
         "upload_to_email",
         "upload_to_s3",
     }
+    
+    # Add check_for_duplicates if deduplication is enabled
+    if settings.enable_deduplication:
+        REAL_STEPS.add("check_for_duplicates")
 
     # Get all REAL steps for these files in one query
     steps = (

app/utils/step_manager.py

@@ -10,11 +10,12 @@ from typing import Dict, List, Optional
 
 from sqlalchemy.orm import Session
 
+from app.config import settings
 from app.models import FileProcessingStep
 
 # Define the expected processing steps for a standard file workflow
-MAIN_PROCESSING_STEPS = [
-    "hash_file",
+# The "check_for_duplicates" step is conditionally included based on enable_deduplication setting
+BASE_MAIN_PROCESSING_STEPS = [
     "create_file_record",
     "check_text",
     "extract_text",  # Or "process_with_azure_document_intelligence"
@@ -24,6 +25,16 @@ MAIN_PROCESSING_STEPS = [
     "send_to_all_destinations",
 ]
 
+OPTIONAL_PROCESSING_STEPS = {
+    "check_for_duplicates": settings.enable_deduplication,  # Only if deduplication is enabled
+}
+
+# Combine steps based on configuration
+MAIN_PROCESSING_STEPS = []
+if settings.enable_deduplication:
+    MAIN_PROCESSING_STEPS.append("check_for_duplicates")
+MAIN_PROCESSING_STEPS.extend(BASE_MAIN_PROCESSING_STEPS)
+
 
 def initialize_file_steps(db: Session, file_id: int, include_uploads: bool = False) -> None:
     """
@@ -188,6 +199,10 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict:
         "send_to_all_destinations",
     }
     
+    # Add check_for_duplicates if deduplication is enabled
+    if settings.enable_deduplication:
+        REAL_MAIN_STEPS.add("check_for_duplicates")
+    
     all_steps = db.query(FileProcessingStep).filter(FileProcessingStep.file_id == file_id).all()
     
     # Filter to only real steps
@@ -270,6 +285,10 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
         "send_to_all_destinations",
     }
     
+    # Add check_for_duplicates if deduplication is enabled
+    if settings.enable_deduplication:
+        REAL_MAIN_STEPS.add("check_for_duplicates")
+    
     steps = db.query(FileProcessingStep).filter(FileProcessingStep.file_id == file_id).all()
 
     main_counts = {"queued": 0, "in_progress": 0, "success": 0, "failure": 0, "skipped": 0}