Merge pull request #403 from christianlouis/copilot/fix-file-status-sync

Fix file detail workflow steps out of sync after OCR provider abstraction

app/api/files.py

@@ -520,7 +520,8 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
 
     Supports restarting from intermediate pipeline steps:
     - process_document: Full reprocessing (skips duplicate check)
-    - process_with_azure_document_intelligence: OCR processing
+    - process_with_ocr: OCR processing (multi-provider)
+    - process_with_azure_document_intelligence: OCR processing (legacy alias for process_with_ocr)
     - extract_metadata_with_gpt: Metadata extraction
     - embed_metadata_into_pdf: Metadata embedding
 
@@ -554,16 +555,11 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
             original_filename=file_record.original_filename,
             file_id=file_id,
         )
-    elif step_name == "process_with_azure_document_intelligence":
-        from app.tasks.process_with_azure_document_intelligence import (
-            process_with_azure_document_intelligence,
-        )
+    elif step_name in ("process_with_ocr", "process_with_azure_document_intelligence"):
+        from app.tasks.process_with_ocr import process_with_ocr
 
         # OCR needs the file in workdir/tmp
-        logger.info(
-            f"Retrying process_with_azure_document_intelligence for file {file_id}: "
-            f"local_filename={file_record.local_filename!r}"
-        )
+        logger.info(f"Retrying process_with_ocr for file {file_id}: local_filename={file_record.local_filename!r}")
         if not file_record.local_filename:
             logger.error(f"OCR retry failed for file {file_id}: local_filename is None")
             raise HTTPException(status_code=400, detail="Local file path is None. Cannot retry OCR.")
@@ -577,7 +573,7 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) -
 
         logger.info(f"Found file for OCR retry at: {file_record.local_filename!r}")
         filename = os.path.basename(file_record.local_filename)
-        task = process_with_azure_document_intelligence.delay(filename, file_id)
+        task = process_with_ocr.delay(filename, file_id)
     elif step_name == "extract_metadata_with_gpt":
         from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
 
@@ -701,7 +697,7 @@ def retry_subtask(
     Retry a specific failed subtask for a file.
 
     Supports both upload tasks (e.g., upload_to_dropbox) and pipeline processing
-    steps (e.g., process_with_azure_document_intelligence, extract_metadata_with_gpt,
+    steps (e.g., process_with_ocr, extract_metadata_with_gpt,
     embed_metadata_into_pdf).
 
     Args:
@@ -721,6 +717,7 @@ def retry_subtask(
         # Pipeline processing steps that can be retried from the failed step
         pipeline_step_names = {
             "process_document",
+            "process_with_ocr",
             "process_with_azure_document_intelligence",
             "extract_metadata_with_gpt",
             "embed_metadata_into_pdf",

app/tasks/process_document.py

@@ -476,7 +476,7 @@ def process_document(
         # Mark OCR as skipped since we extracted text locally
         log_task_progress(
             task_id,
-            "process_with_azure_document_intelligence",
+            "process_with_ocr",
             "skipped",
             "Local text extraction succeeded, OCR not needed",
             file_id=file_id,

app/utils/file_queries.py

@@ -21,9 +21,9 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
     processing status by examining associated FileProcessingStep entries.
 
     Only tracks "real" processing steps that represent user-facing status:
-    - Main steps: create_file_record, check_text, extract_text, process_with_azure_document_intelligence,
-                  extract_metadata_with_gpt, embed_metadata_into_pdf, finalize_document_storage,
-                  send_to_all_destinations
+    - Main steps: create_file_record, check_text, extract_text, process_with_ocr,
+                  process_with_azure_document_intelligence (legacy), extract_metadata_with_gpt,
+                  embed_metadata_into_pdf, finalize_document_storage, send_to_all_destinations
     - Upload steps: queue_*, upload_to_*
 
     Diagnostic/internal steps (poll_task, upload_file, set_custom_fields, etc.) are ignored
@@ -58,6 +58,7 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que
         "create_file_record",
         "check_text",
         "extract_text",
+        "process_with_ocr",
         "process_with_azure_document_intelligence",
         "extract_metadata_with_gpt",
         "embed_metadata_into_pdf",

app/utils/file_status.py

@@ -56,9 +56,9 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
     Get processing status for multiple files efficiently.
 
     Only counts "real" processing steps that represent user-facing status:
-    - Main steps: create_file_record, check_text, extract_text, process_with_azure_document_intelligence,
-                  extract_metadata_with_gpt, embed_metadata_into_pdf, finalize_document_storage,
-                  send_to_all_destinations
+    - Main steps: create_file_record, check_text, extract_text, process_with_ocr,
+                  process_with_azure_document_intelligence (legacy), extract_metadata_with_gpt,
+                  embed_metadata_into_pdf, finalize_document_storage, send_to_all_destinations
     - Upload steps: upload_to_*
 
     Diagnostic/internal steps (poll_task, upload_file, set_custom_fields, etc.) are ignored.
@@ -83,6 +83,7 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D
         "create_file_record",
         "check_text",
         "extract_text",
+        "process_with_ocr",
         "process_with_azure_document_intelligence",
         "extract_metadata_with_gpt",
         "embed_metadata_into_pdf",

app/utils/step_manager.py

@@ -201,10 +201,13 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict:
 
     # Define which steps are "real" status-determining steps
     # Only high-level logical steps, not implementation sub-steps
+    # Both process_with_ocr (current) and process_with_azure_document_intelligence (legacy)
+    # are included to correctly count steps for files processed before the OCR abstraction.
     REAL_MAIN_STEPS = {
         "create_file_record",
         "check_text",
         "extract_text",
+        "process_with_ocr",
         "process_with_azure_document_intelligence",
         "extract_metadata_with_gpt",
         "embed_metadata_into_pdf",
@@ -286,10 +289,13 @@ def get_step_summary(db: Session, file_id: int) -> Dict:
     """
     # Define which steps are "real" status-determining steps
     # Only high-level logical steps, not implementation sub-steps
+    # Both process_with_ocr (current) and process_with_azure_document_intelligence (legacy)
+    # are included to correctly count steps for files processed before the OCR abstraction.
     REAL_MAIN_STEPS = {
         "create_file_record",
         "check_text",
         "extract_text",
+        "process_with_ocr",
         "process_with_azure_document_intelligence",
         "extract_metadata_with_gpt",
         "embed_metadata_into_pdf",

app/views/files.py

@@ -222,11 +222,11 @@ def _compute_processing_flow(logs):
         "create_file_record": {"label": "Create File Record", "next": ["check_text"]},
         "check_text": {
             "label": "Check Embedded Text",
-            "next": ["extract_text", "process_with_azure_document_intelligence"],
+            "next": ["extract_text", "process_with_ocr"],
         },
         "extract_text": {"label": "Extract Text (Local)", "next": ["extract_metadata_with_gpt"]},
-        "process_with_azure_document_intelligence": {
-            "label": "OCR Processing (Azure)",
+        "process_with_ocr": {
+            "label": "OCR Processing",
             "next": ["extract_metadata_with_gpt"],
         },
         "extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]},
@@ -285,6 +285,9 @@ def _compute_processing_flow(logs):
                 {"status": log.status, "message": log.message, "timestamp": log.timestamp, "task_id": log.task_id}
             )
         else:
+            # Normalize legacy OCR step name for backward compatibility with old log entries
+            if step_name == "process_with_azure_document_intelligence":
+                step_name = "process_with_ocr"
             # Regular processing step
             if step_name not in step_map:
                 step_map[step_name] = []
@@ -366,7 +369,7 @@ def _compute_step_summary(logs):
             "create_file_record",
             "check_text",
             "extract_text",
-            "process_with_azure_document_intelligence",
+            "process_with_ocr",
             "extract_metadata_with_gpt",
             "embed_metadata_into_pdf",
             "finalize_document_storage",
@@ -391,6 +394,10 @@ def _compute_step_summary(logs):
         if status == "pending":
             status = "queued"
 
+        # Normalize legacy OCR step name for backward compatibility
+        if step_name == "process_with_azure_document_intelligence":
+            step_name = "process_with_ocr"
+
         # Check if it's an upload task
         is_upload = any(step_name.startswith(prefix) for prefix in upload_prefixes)
 

tests/test_api_files_comprehensive.py

@@ -950,9 +950,7 @@ class TestRetryPipelineSteps:
         db_session.add(file)
         db_session.commit()
 
-        with patch(
-            "app.tasks.process_with_azure_document_intelligence.process_with_azure_document_intelligence"
-        ) as mock_task:
+        with patch("app.tasks.process_with_ocr.process_with_ocr") as mock_task:
             mock_task.delay.return_value = Mock(id="task-azure")
             result = _retry_pipeline_step(file, "process_with_azure_document_intelligence", db_session)
             assert result["task_id"] == "task-azure"