feat(deduplication): implement configurable duplicate file detection

- Add enable_deduplication and show_deduplication_step config options
- Rename hash_file step to check_for_duplicates
- Make deduplication step conditional based on configuration
- Add is_duplicate and duplicate_of_id fields to FileRecord model
- Create database migration for new deduplication fields
- Update process_document task to log deduplication results
- Update step visualization to show/hide step based on config
- Update status calculations to include deduplication step conditionally
- Default: deduplication enabled, step displayed
- Can be configured to hide from UI while still processing

app/models.py

@@ -1,6 +1,6 @@
 # app/models.py
 
-from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func, Boolean
 
 from app.database import Base
 
@@ -43,6 +43,13 @@ class FileRecord(Base):
 
     # MIME type or extension (optional)
     mime_type = Column(String)
+    
+    # Deduplication tracking: True if this file is a duplicate of another file
+    # When a duplicate is detected, this file record is created but marked as duplicate
+    is_duplicate = Column(Boolean, default=False, nullable=False, index=True)
+    
+    # If this is a duplicate, record the ID of the original file for reference
+    duplicate_of_id = Column(Integer, ForeignKey("files.id"), nullable=True)
 
     # Timestamp when we inserted this record
     created_at = Column(DateTime(timezone=True), server_default=func.now())