feat(deduplication): implement configurable duplicate file detection

- Add enable_deduplication and show_deduplication_step config options
- Rename hash_file step to check_for_duplicates
- Make deduplication step conditional based on configuration
- Add is_duplicate and duplicate_of_id fields to FileRecord model
- Create database migration for new deduplication fields
- Update process_document task to log deduplication results
- Update step visualization to show/hide step based on config
- Update status calculations to include deduplication step conditionally
- Default: deduplication enabled, step displayed
- Can be configured to hide from UI while still processing

app/utils/file_queries.py

@@ -52,6 +52,8 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
 
     # Define which steps are "real" status-determining steps
     # Only high-level logical steps and actual upload destinations (not queue_* steps)
+    from app.config import settings
+    
     REAL_STEPS = {
         "create_file_record",
         "check_text",
@@ -73,6 +75,10 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
         "upload_to_email",
         "upload_to_s3",
     }
+    
+    # Add check_for_duplicates if deduplication is enabled
+    if settings.enable_deduplication:
+        REAL_STEPS.add("check_for_duplicates")
 
     # Filter to only real steps
     real_steps_subq = (