diff --git a/app/api/files.py b/app/api/files.py index 6803b1d8..d4014f55 100644 --- a/app/api/files.py +++ b/app/api/files.py @@ -520,7 +520,8 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) - Supports restarting from intermediate pipeline steps: - process_document: Full reprocessing (skips duplicate check) - - process_with_azure_document_intelligence: OCR processing + - process_with_ocr: OCR processing (multi-provider) + - process_with_azure_document_intelligence: OCR processing (legacy alias for process_with_ocr) - extract_metadata_with_gpt: Metadata extraction - embed_metadata_into_pdf: Metadata embedding @@ -554,16 +555,11 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) - original_filename=file_record.original_filename, file_id=file_id, ) - elif step_name == "process_with_azure_document_intelligence": - from app.tasks.process_with_azure_document_intelligence import ( - process_with_azure_document_intelligence, - ) + elif step_name in ("process_with_ocr", "process_with_azure_document_intelligence"): + from app.tasks.process_with_ocr import process_with_ocr # OCR needs the file in workdir/tmp - logger.info( - f"Retrying process_with_azure_document_intelligence for file {file_id}: " - f"local_filename={file_record.local_filename!r}" - ) + logger.info(f"Retrying process_with_ocr for file {file_id}: local_filename={file_record.local_filename!r}") if not file_record.local_filename: logger.error(f"OCR retry failed for file {file_id}: local_filename is None") raise HTTPException(status_code=400, detail="Local file path is None. Cannot retry OCR.") @@ -577,7 +573,7 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) - logger.info(f"Found file for OCR retry at: {file_record.local_filename!r}") filename = os.path.basename(file_record.local_filename) - task = process_with_azure_document_intelligence.delay(filename, file_id) + task = process_with_ocr.delay(filename, file_id) elif step_name == "extract_metadata_with_gpt": from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt @@ -701,7 +697,7 @@ def retry_subtask( Retry a specific failed subtask for a file. Supports both upload tasks (e.g., upload_to_dropbox) and pipeline processing - steps (e.g., process_with_azure_document_intelligence, extract_metadata_with_gpt, + steps (e.g., process_with_ocr, extract_metadata_with_gpt, embed_metadata_into_pdf). Args: @@ -721,6 +717,7 @@ def retry_subtask( # Pipeline processing steps that can be retried from the failed step pipeline_step_names = { "process_document", + "process_with_ocr", "process_with_azure_document_intelligence", "extract_metadata_with_gpt", "embed_metadata_into_pdf", diff --git a/app/tasks/process_document.py b/app/tasks/process_document.py index 4226f0b7..0d3b044d 100644 --- a/app/tasks/process_document.py +++ b/app/tasks/process_document.py @@ -476,7 +476,7 @@ def process_document( # Mark OCR as skipped since we extracted text locally log_task_progress( task_id, - "process_with_azure_document_intelligence", + "process_with_ocr", "skipped", "Local text extraction succeeded, OCR not needed", file_id=file_id, diff --git a/app/utils/file_queries.py b/app/utils/file_queries.py index f3ced1a7..4fa79e9f 100644 --- a/app/utils/file_queries.py +++ b/app/utils/file_queries.py @@ -21,9 +21,9 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que processing status by examining associated FileProcessingStep entries. Only tracks "real" processing steps that represent user-facing status: - - Main steps: create_file_record, check_text, extract_text, process_with_azure_document_intelligence, - extract_metadata_with_gpt, embed_metadata_into_pdf, finalize_document_storage, - send_to_all_destinations + - Main steps: create_file_record, check_text, extract_text, process_with_ocr, + process_with_azure_document_intelligence (legacy), extract_metadata_with_gpt, + embed_metadata_into_pdf, finalize_document_storage, send_to_all_destinations - Upload steps: queue_*, upload_to_* Diagnostic/internal steps (poll_task, upload_file, set_custom_fields, etc.) are ignored @@ -58,6 +58,7 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que "create_file_record", "check_text", "extract_text", + "process_with_ocr", "process_with_azure_document_intelligence", "extract_metadata_with_gpt", "embed_metadata_into_pdf", diff --git a/app/utils/file_status.py b/app/utils/file_status.py index 18a20658..5a278a7c 100644 --- a/app/utils/file_status.py +++ b/app/utils/file_status.py @@ -56,9 +56,9 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D Get processing status for multiple files efficiently. Only counts "real" processing steps that represent user-facing status: - - Main steps: create_file_record, check_text, extract_text, process_with_azure_document_intelligence, - extract_metadata_with_gpt, embed_metadata_into_pdf, finalize_document_storage, - send_to_all_destinations + - Main steps: create_file_record, check_text, extract_text, process_with_ocr, + process_with_azure_document_intelligence (legacy), extract_metadata_with_gpt, + embed_metadata_into_pdf, finalize_document_storage, send_to_all_destinations - Upload steps: upload_to_* Diagnostic/internal steps (poll_task, upload_file, set_custom_fields, etc.) are ignored. @@ -83,6 +83,7 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D "create_file_record", "check_text", "extract_text", + "process_with_ocr", "process_with_azure_document_intelligence", "extract_metadata_with_gpt", "embed_metadata_into_pdf", diff --git a/app/utils/step_manager.py b/app/utils/step_manager.py index 022d97be..0d84ee09 100644 --- a/app/utils/step_manager.py +++ b/app/utils/step_manager.py @@ -201,10 +201,13 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict: # Define which steps are "real" status-determining steps # Only high-level logical steps, not implementation sub-steps + # Both process_with_ocr (current) and process_with_azure_document_intelligence (legacy) + # are included to correctly count steps for files processed before the OCR abstraction. REAL_MAIN_STEPS = { "create_file_record", "check_text", "extract_text", + "process_with_ocr", "process_with_azure_document_intelligence", "extract_metadata_with_gpt", "embed_metadata_into_pdf", @@ -286,10 +289,13 @@ def get_step_summary(db: Session, file_id: int) -> Dict: """ # Define which steps are "real" status-determining steps # Only high-level logical steps, not implementation sub-steps + # Both process_with_ocr (current) and process_with_azure_document_intelligence (legacy) + # are included to correctly count steps for files processed before the OCR abstraction. REAL_MAIN_STEPS = { "create_file_record", "check_text", "extract_text", + "process_with_ocr", "process_with_azure_document_intelligence", "extract_metadata_with_gpt", "embed_metadata_into_pdf", diff --git a/app/views/files.py b/app/views/files.py index 5d7d0c9f..d2395dc7 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -222,11 +222,11 @@ def _compute_processing_flow(logs): "create_file_record": {"label": "Create File Record", "next": ["check_text"]}, "check_text": { "label": "Check Embedded Text", - "next": ["extract_text", "process_with_azure_document_intelligence"], + "next": ["extract_text", "process_with_ocr"], }, "extract_text": {"label": "Extract Text (Local)", "next": ["extract_metadata_with_gpt"]}, - "process_with_azure_document_intelligence": { - "label": "OCR Processing (Azure)", + "process_with_ocr": { + "label": "OCR Processing", "next": ["extract_metadata_with_gpt"], }, "extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]}, @@ -285,6 +285,9 @@ def _compute_processing_flow(logs): {"status": log.status, "message": log.message, "timestamp": log.timestamp, "task_id": log.task_id} ) else: + # Normalize legacy OCR step name for backward compatibility with old log entries + if step_name == "process_with_azure_document_intelligence": + step_name = "process_with_ocr" # Regular processing step if step_name not in step_map: step_map[step_name] = [] @@ -366,7 +369,7 @@ def _compute_step_summary(logs): "create_file_record", "check_text", "extract_text", - "process_with_azure_document_intelligence", + "process_with_ocr", "extract_metadata_with_gpt", "embed_metadata_into_pdf", "finalize_document_storage", @@ -391,6 +394,10 @@ def _compute_step_summary(logs): if status == "pending": status = "queued" + # Normalize legacy OCR step name for backward compatibility + if step_name == "process_with_azure_document_intelligence": + step_name = "process_with_ocr" + # Check if it's an upload task is_upload = any(step_name.startswith(prefix) for prefix in upload_prefixes) diff --git a/tests/test_api_files_comprehensive.py b/tests/test_api_files_comprehensive.py index c5323e10..655a9335 100644 --- a/tests/test_api_files_comprehensive.py +++ b/tests/test_api_files_comprehensive.py @@ -950,9 +950,7 @@ class TestRetryPipelineSteps: db_session.add(file) db_session.commit() - with patch( - "app.tasks.process_with_azure_document_intelligence.process_with_azure_document_intelligence" - ) as mock_task: + with patch("app.tasks.process_with_ocr.process_with_ocr") as mock_task: mock_task.delay.return_value = Mock(id="task-azure") result = _retry_pipeline_step(file, "process_with_azure_document_intelligence", db_session) assert result["task_id"] == "task-azure"