diff --git a/app/tasks/process_with_azure_document_intelligence.py b/app/tasks/process_with_azure_document_intelligence.py index 652f68f4..c89c1c9b 100644 --- a/app/tasks/process_with_azure_document_intelligence.py +++ b/app/tasks/process_with_azure_document_intelligence.py @@ -11,6 +11,7 @@ from app.celery_app import celery from app.config import settings from app.tasks.retry_config import BaseTaskWithRetry from app.tasks.rotate_pdf_pages import rotate_pdf_pages +from app.utils import log_task_progress logger = logging.getLogger(__name__) @@ -56,30 +57,30 @@ def check_page_rotation(result, filename): Returns: dict: Dictionary mapping page indices (integers) to rotation angles """ - logger.error(f"Checking rotation for document: {filename}") + logger.info(f"Checking rotation for document: {filename}") rotation_data = {} if not hasattr(result, "pages") or not result.pages: - logger.error(f"No page information available for rotation check: {filename}") + logger.warning(f"No page information available for rotation check: {filename}") return rotation_data for i, page in enumerate(result.pages): if hasattr(page, "angle"): rotation_angle = page.angle if rotation_angle != 0: - logger.error(f"Page {i+1} is rotated by {rotation_angle} degrees") + logger.info(f"Page {i+1} is rotated by {rotation_angle} degrees") # Store page index as integer, not string rotation_data[i] = rotation_angle else: - logger.error(f"Page {i+1} has no rotation (0 degrees)") + logger.info(f"Page {i+1} has no rotation (0 degrees)") else: - logger.error(f"Page {i+1} rotation information not available") + logger.info(f"Page {i+1} rotation information not available") return rotation_data -@celery.task(base=BaseTaskWithRetry) -def process_with_azure_document_intelligence(filename: str, file_id: int = None): +@celery.task(base=BaseTaskWithRetry, bind=True) +def process_with_azure_document_intelligence(self, filename: str, file_id: int = None): """ Processes a PDF document using Azure Document Intelligence and overlays OCR text onto the local temporary file (stored under /tmp). @@ -96,6 +97,11 @@ def process_with_azure_document_intelligence(filename: str, file_id: int = None) filename: Name of the file to process file_id: Optional file ID to pass through to subsequent tasks """ + task_id = self.request.id + log_task_progress( + task_id, "process_with_azure_document_intelligence", "in_progress", + f"Starting OCR for {filename}", file_id=file_id, + ) try: tmp_file_path = os.path.join(settings.workdir, "tmp", filename) if not os.path.exists(tmp_file_path): @@ -107,7 +113,11 @@ def process_with_azure_document_intelligence(filename: str, file_id: int = None) error_msg = ( f"File size ({file_size / (1024 * 1024):.2f} MB) exceeds Azure Document Intelligence limit of 500 MB" ) - logger.error(error_msg) + logger.error(f"[{task_id}] {error_msg}") + log_task_progress( + task_id, "validate_file", "failure", + f"File too large: {filename}", file_id=file_id, detail=error_msg, + ) return {"error": error_msg, "file": filename, "status": "Failed - Size limit exceeded"} # For PDF files, check page count against service limits @@ -116,12 +126,27 @@ def process_with_azure_document_intelligence(filename: str, file_id: int = None) page_count = get_pdf_page_count(tmp_file_path) if page_count is not None and page_count > AZURE_DOC_INTELLIGENCE_LIMITS["max_pages"]: error_msg = f"PDF page count ({page_count}) exceeds Azure Document Intelligence limit of 2000 pages" - logger.error(error_msg) + logger.error(f"[{task_id}] {error_msg}") + log_task_progress( + task_id, "validate_file", "failure", + f"Too many pages: {filename}", file_id=file_id, detail=error_msg, + ) return {"error": error_msg, "file": filename, "status": "Failed - Page limit exceeded"} if page_count is None: - logger.warning(f"Could not determine page count for {filename}, proceeding with processing anyway") + logger.warning( + f"[{task_id}] Could not determine page count for {filename}, proceeding with processing anyway" + ) - logger.info(f"Processing {filename} with Azure Document Intelligence OCR.") + log_task_progress( + task_id, "validate_file", "success", + f"File validation passed for {filename}", file_id=file_id, + ) + + logger.info(f"[{task_id}] Processing {filename} with Azure Document Intelligence OCR.") + log_task_progress( + task_id, "call_azure_ocr", "in_progress", + f"Sending {filename} to Azure Document Intelligence", file_id=file_id, + ) # Open and send the document for processing with open(tmp_file_path, "rb") as f: @@ -139,16 +164,32 @@ def process_with_azure_document_intelligence(filename: str, file_id: int = None) searchable_pdf_path = tmp_file_path # Overwrite the original PDF location with open(searchable_pdf_path, "wb") as writer: writer.writelines(response) - logger.info(f"Searchable PDF saved at: {searchable_pdf_path}") + logger.info(f"[{task_id}] Searchable PDF saved at: {searchable_pdf_path}") # Extract raw text content from the result extracted_text = result.content if result.content else "" - logger.info(f"Extracted text for {filename}: {len(extracted_text)} characters") + logger.info(f"[{task_id}] Extracted text for {filename}: {len(extracted_text)} characters") + + log_task_progress( + task_id, "call_azure_ocr", "success", + f"Azure OCR completed for {filename}", file_id=file_id, + detail=f"Extracted {len(extracted_text)} characters, {len(rotation_data)} rotated pages detected", + ) # Trigger page rotation task if rotation is detected, otherwise proceed to metadata extraction rotate_pdf_pages.delay(filename, extracted_text, rotation_data, file_id) + log_task_progress( + task_id, "process_with_azure_document_intelligence", "success", + f"OCR processing complete for {filename}", file_id=file_id, + detail=f"Searchable PDF saved, {len(extracted_text)} characters extracted", + ) + return {"file": filename, "searchable_pdf": searchable_pdf_path, "cleaned_text": extracted_text} except Exception as e: - logger.error(f"Error processing {filename} with Azure Document Intelligence: {e}") + logger.error(f"[{task_id}] Error processing {filename} with Azure Document Intelligence: {e}") + log_task_progress( + task_id, "process_with_azure_document_intelligence", "failure", + f"OCR failed for {filename}", file_id=file_id, detail=str(e), + ) raise