From 37a3f7aae7b1691915c65c481dbca061061d610a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Feb 2026 22:22:06 +0000 Subject: [PATCH] fix(ocr): sync workflow steps with process_with_ocr replacing legacy azure step - Update _compute_processing_flow to recognize process_with_ocr as the OCR stage and remap legacy process_with_azure_document_intelligence log entries for backward compatibility - Normalize legacy OCR step name in _compute_step_summary log fallback - Add process_with_ocr to REAL_MAIN_STEPS/REAL_STEPS in step_manager, file_status, and file_queries (keeping legacy name for old DB entries) - Update retry logic in api/files.py to retry failed OCR via process_with_ocr (handles both step names as aliases) - Fix process_document.py to log process_with_ocr as skipped (not azure step) for the local text extraction path Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com> --- app/api/files.py | 19 ++++++++----------- app/tasks/process_document.py | 2 +- app/utils/file_queries.py | 7 ++++--- app/utils/file_status.py | 7 ++++--- app/utils/step_manager.py | 6 ++++++ app/views/files.py | 15 +++++++++++---- 6 files changed, 34 insertions(+), 22 deletions(-) diff --git a/app/api/files.py b/app/api/files.py index 6803b1d8..d4014f55 100644 --- a/app/api/files.py +++ b/app/api/files.py @@ -520,7 +520,8 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) - Supports restarting from intermediate pipeline steps: - process_document: Full reprocessing (skips duplicate check) - - process_with_azure_document_intelligence: OCR processing + - process_with_ocr: OCR processing (multi-provider) + - process_with_azure_document_intelligence: OCR processing (legacy alias for process_with_ocr) - extract_metadata_with_gpt: Metadata extraction - embed_metadata_into_pdf: Metadata embedding @@ -554,16 +555,11 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) - original_filename=file_record.original_filename, file_id=file_id, ) - elif step_name == "process_with_azure_document_intelligence": - from app.tasks.process_with_azure_document_intelligence import ( - process_with_azure_document_intelligence, - ) + elif step_name in ("process_with_ocr", "process_with_azure_document_intelligence"): + from app.tasks.process_with_ocr import process_with_ocr # OCR needs the file in workdir/tmp - logger.info( - f"Retrying process_with_azure_document_intelligence for file {file_id}: " - f"local_filename={file_record.local_filename!r}" - ) + logger.info(f"Retrying process_with_ocr for file {file_id}: local_filename={file_record.local_filename!r}") if not file_record.local_filename: logger.error(f"OCR retry failed for file {file_id}: local_filename is None") raise HTTPException(status_code=400, detail="Local file path is None. Cannot retry OCR.") @@ -577,7 +573,7 @@ def _retry_pipeline_step(file_record: FileRecord, step_name: str, db: Session) - logger.info(f"Found file for OCR retry at: {file_record.local_filename!r}") filename = os.path.basename(file_record.local_filename) - task = process_with_azure_document_intelligence.delay(filename, file_id) + task = process_with_ocr.delay(filename, file_id) elif step_name == "extract_metadata_with_gpt": from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt @@ -701,7 +697,7 @@ def retry_subtask( Retry a specific failed subtask for a file. Supports both upload tasks (e.g., upload_to_dropbox) and pipeline processing - steps (e.g., process_with_azure_document_intelligence, extract_metadata_with_gpt, + steps (e.g., process_with_ocr, extract_metadata_with_gpt, embed_metadata_into_pdf). Args: @@ -721,6 +717,7 @@ def retry_subtask( # Pipeline processing steps that can be retried from the failed step pipeline_step_names = { "process_document", + "process_with_ocr", "process_with_azure_document_intelligence", "extract_metadata_with_gpt", "embed_metadata_into_pdf", diff --git a/app/tasks/process_document.py b/app/tasks/process_document.py index 4226f0b7..0d3b044d 100644 --- a/app/tasks/process_document.py +++ b/app/tasks/process_document.py @@ -476,7 +476,7 @@ def process_document( # Mark OCR as skipped since we extracted text locally log_task_progress( task_id, - "process_with_azure_document_intelligence", + "process_with_ocr", "skipped", "Local text extraction succeeded, OCR not needed", file_id=file_id, diff --git a/app/utils/file_queries.py b/app/utils/file_queries.py index f3ced1a7..4fa79e9f 100644 --- a/app/utils/file_queries.py +++ b/app/utils/file_queries.py @@ -21,9 +21,9 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que processing status by examining associated FileProcessingStep entries. Only tracks "real" processing steps that represent user-facing status: - - Main steps: create_file_record, check_text, extract_text, process_with_azure_document_intelligence, - extract_metadata_with_gpt, embed_metadata_into_pdf, finalize_document_storage, - send_to_all_destinations + - Main steps: create_file_record, check_text, extract_text, process_with_ocr, + process_with_azure_document_intelligence (legacy), extract_metadata_with_gpt, + embed_metadata_into_pdf, finalize_document_storage, send_to_all_destinations - Upload steps: queue_*, upload_to_* Diagnostic/internal steps (poll_task, upload_file, set_custom_fields, etc.) are ignored @@ -58,6 +58,7 @@ def apply_status_filter(query: Query, db: Session, status: Optional[str]) -> Que "create_file_record", "check_text", "extract_text", + "process_with_ocr", "process_with_azure_document_intelligence", "extract_metadata_with_gpt", "embed_metadata_into_pdf", diff --git a/app/utils/file_status.py b/app/utils/file_status.py index 18a20658..5a278a7c 100644 --- a/app/utils/file_status.py +++ b/app/utils/file_status.py @@ -56,9 +56,9 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D Get processing status for multiple files efficiently. Only counts "real" processing steps that represent user-facing status: - - Main steps: create_file_record, check_text, extract_text, process_with_azure_document_intelligence, - extract_metadata_with_gpt, embed_metadata_into_pdf, finalize_document_storage, - send_to_all_destinations + - Main steps: create_file_record, check_text, extract_text, process_with_ocr, + process_with_azure_document_intelligence (legacy), extract_metadata_with_gpt, + embed_metadata_into_pdf, finalize_document_storage, send_to_all_destinations - Upload steps: upload_to_* Diagnostic/internal steps (poll_task, upload_file, set_custom_fields, etc.) are ignored. @@ -83,6 +83,7 @@ def get_files_processing_status(db: Session, file_ids: List[int]) -> Dict[int, D "create_file_record", "check_text", "extract_text", + "process_with_ocr", "process_with_azure_document_intelligence", "extract_metadata_with_gpt", "embed_metadata_into_pdf", diff --git a/app/utils/step_manager.py b/app/utils/step_manager.py index 022d97be..0d84ee09 100644 --- a/app/utils/step_manager.py +++ b/app/utils/step_manager.py @@ -201,10 +201,13 @@ def get_file_overall_status(db: Session, file_id: int) -> Dict: # Define which steps are "real" status-determining steps # Only high-level logical steps, not implementation sub-steps + # Both process_with_ocr (current) and process_with_azure_document_intelligence (legacy) + # are included to correctly count steps for files processed before the OCR abstraction. REAL_MAIN_STEPS = { "create_file_record", "check_text", "extract_text", + "process_with_ocr", "process_with_azure_document_intelligence", "extract_metadata_with_gpt", "embed_metadata_into_pdf", @@ -286,10 +289,13 @@ def get_step_summary(db: Session, file_id: int) -> Dict: """ # Define which steps are "real" status-determining steps # Only high-level logical steps, not implementation sub-steps + # Both process_with_ocr (current) and process_with_azure_document_intelligence (legacy) + # are included to correctly count steps for files processed before the OCR abstraction. REAL_MAIN_STEPS = { "create_file_record", "check_text", "extract_text", + "process_with_ocr", "process_with_azure_document_intelligence", "extract_metadata_with_gpt", "embed_metadata_into_pdf", diff --git a/app/views/files.py b/app/views/files.py index 5d7d0c9f..d2395dc7 100644 --- a/app/views/files.py +++ b/app/views/files.py @@ -222,11 +222,11 @@ def _compute_processing_flow(logs): "create_file_record": {"label": "Create File Record", "next": ["check_text"]}, "check_text": { "label": "Check Embedded Text", - "next": ["extract_text", "process_with_azure_document_intelligence"], + "next": ["extract_text", "process_with_ocr"], }, "extract_text": {"label": "Extract Text (Local)", "next": ["extract_metadata_with_gpt"]}, - "process_with_azure_document_intelligence": { - "label": "OCR Processing (Azure)", + "process_with_ocr": { + "label": "OCR Processing", "next": ["extract_metadata_with_gpt"], }, "extract_metadata_with_gpt": {"label": "Extract Metadata (GPT)", "next": ["embed_metadata_into_pdf"]}, @@ -285,6 +285,9 @@ def _compute_processing_flow(logs): {"status": log.status, "message": log.message, "timestamp": log.timestamp, "task_id": log.task_id} ) else: + # Normalize legacy OCR step name for backward compatibility with old log entries + if step_name == "process_with_azure_document_intelligence": + step_name = "process_with_ocr" # Regular processing step if step_name not in step_map: step_map[step_name] = [] @@ -366,7 +369,7 @@ def _compute_step_summary(logs): "create_file_record", "check_text", "extract_text", - "process_with_azure_document_intelligence", + "process_with_ocr", "extract_metadata_with_gpt", "embed_metadata_into_pdf", "finalize_document_storage", @@ -391,6 +394,10 @@ def _compute_step_summary(logs): if status == "pending": status = "queued" + # Normalize legacy OCR step name for backward compatibility + if step_name == "process_with_azure_document_intelligence": + step_name = "process_with_ocr" + # Check if it's an upload task is_upload = any(step_name.startswith(prefix) for prefix in upload_prefixes)