feat(storage): implement immutable original/processed file storage and collision handling

- Add original_file_path and processed_file_path columns to FileRecord model
- Create database migration for new fields
- Implement get_unique_filepath_with_counter() with -0001 suffix format
- Update process_document to save immutable copy to /workdir/original
- Add force_cloud_ocr parameter to process_document for forced OCR
- Update embed_metadata to use new collision handling
- Update metadata JSON to include file path references
- Add /files/{file_id}/reprocess-with-cloud-ocr API endpoint
- Update processed_file_path in database during embedding

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/utils/__init__.py

@@ -4,7 +4,8 @@ Utility functions and helpers for the document processor application.
 
 # Import functions to make them available through the package
 from app.utils.file_operations import hash_file
+from app.utils.filename_utils import get_unique_filepath_with_counter, sanitize_filename
 from app.utils.logging import log_task_progress
 
 # Export all the functions that should be available when importing from app.utils
-__all__ = ["hash_file", "log_task_progress"]
+__all__ = ["hash_file", "log_task_progress", "get_unique_filepath_with_counter", "sanitize_filename"]

app/utils/filename_utils.py

@@ -69,6 +69,57 @@ def get_unique_filename(original_path, check_exists_func=None):
     return new_path
 
 
+def get_unique_filepath_with_counter(directory, base_filename, extension=".pdf"):
+    """
+    Returns a unique filepath in the specified directory using a numeric counter suffix.
+    If 'base_filename.pdf' exists, it will append '-0001', '-0002', etc.
+    
+    This function implements robust collision handling with zero-padded numeric suffixes
+    as required for document storage organization.
+    
+    Args:
+        directory (str): Directory path where the file will be stored
+        base_filename (str): Base name for the file (without extension)
+        extension (str): File extension including the dot (default: ".pdf")
+        
+    Returns:
+        str: Full path to a unique filename
+        
+    Examples:
+        >>> get_unique_filepath_with_counter("/workdir/original", "2024-01-01_Invoice")
+        "/workdir/original/2024-01-01_Invoice.pdf"  # If doesn't exist
+        
+        >>> get_unique_filepath_with_counter("/workdir/original", "2024-01-01_Invoice")
+        "/workdir/original/2024-01-01_Invoice-0001.pdf"  # If original exists
+    """
+    # Try the base filename first
+    candidate = os.path.join(directory, base_filename + extension)
+    if not os.path.exists(candidate):
+        return candidate
+    
+    # If base exists, try with counter suffix
+    counter = 1
+    while True:
+        # Use zero-padded 4-digit counter: -0001, -0002, etc.
+        suffix = f"-{counter:04d}"
+        candidate = os.path.join(directory, f"{base_filename}{suffix}{extension}")
+        if not os.path.exists(candidate):
+            return candidate
+        counter += 1
+        
+        # Sanity check to prevent infinite loops (very unlikely to reach)
+        if counter > 9999:
+            # Fall back to timestamp + UUID if somehow we have 10000 collisions
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            uuid_str = str(uuid.uuid4())[:8]
+            candidate = os.path.join(directory, f"{base_filename}-{timestamp}-{uuid_str}{extension}")
+            logger.warning(
+                f"Exceeded 9999 file collisions for {base_filename}, "
+                f"using timestamp+UUID: {os.path.basename(candidate)}"
+            )
+            return candidate
+
+
 def sanitize_filename(filename):
     r"""
     Sanitize a filename to ensure it's valid across different file systems