feat(tasks): add detailed logging to Azure Document Intelligence task

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>

app/tasks/process_with_azure_document_intelligence.py

@@ -11,6 +11,7 @@ from app.celery_app import celery
 from app.config import settings
 from app.tasks.retry_config import BaseTaskWithRetry
 from app.tasks.rotate_pdf_pages import rotate_pdf_pages
+from app.utils import log_task_progress
 
 logger = logging.getLogger(__name__)
 
@@ -56,30 +57,30 @@ def check_page_rotation(result, filename):
     Returns:
         dict: Dictionary mapping page indices (integers) to rotation angles
     """
-    logger.error(f"Checking rotation for document: {filename}")
+    logger.info(f"Checking rotation for document: {filename}")
     rotation_data = {}
 
     if not hasattr(result, "pages") or not result.pages:
-        logger.error(f"No page information available for rotation check: {filename}")
+        logger.warning(f"No page information available for rotation check: {filename}")
         return rotation_data
 
     for i, page in enumerate(result.pages):
         if hasattr(page, "angle"):
             rotation_angle = page.angle
             if rotation_angle != 0:
-                logger.error(f"Page {i+1} is rotated by {rotation_angle} degrees")
+                logger.info(f"Page {i+1} is rotated by {rotation_angle} degrees")
                 # Store page index as integer, not string
                 rotation_data[i] = rotation_angle
             else:
-                logger.error(f"Page {i+1} has no rotation (0 degrees)")
+                logger.info(f"Page {i+1} has no rotation (0 degrees)")
         else:
-            logger.error(f"Page {i+1} rotation information not available")
+            logger.info(f"Page {i+1} rotation information not available")
 
     return rotation_data
 
 
-@celery.task(base=BaseTaskWithRetry)
+@celery.task(base=BaseTaskWithRetry, bind=True)
-def process_with_azure_document_intelligence(filename: str, file_id: int = None):
+def process_with_azure_document_intelligence(self, filename: str, file_id: int = None):
     """
     Processes a PDF document using Azure Document Intelligence and overlays OCR text onto
     the local temporary file (stored under <workdir>/tmp).
@@ -96,6 +97,11 @@ def process_with_azure_document_intelligence(filename: str, file_id: int = None)
         filename: Name of the file to process
         file_id: Optional file ID to pass through to subsequent tasks
     """
+    task_id = self.request.id
+    log_task_progress(
+        task_id, "process_with_azure_document_intelligence", "in_progress",
+        f"Starting OCR for {filename}", file_id=file_id,
+    )
     try:
         tmp_file_path = os.path.join(settings.workdir, "tmp", filename)
         if not os.path.exists(tmp_file_path):
@@ -107,7 +113,11 @@ def process_with_azure_document_intelligence(filename: str, file_id: int = None)
             error_msg = (
                 f"File size ({file_size / (1024 * 1024):.2f} MB) exceeds Azure Document Intelligence limit of 500 MB"
             )
-            logger.error(error_msg)
+            logger.error(f"[{task_id}] {error_msg}")
+            log_task_progress(
+                task_id, "validate_file", "failure",
+                f"File too large: {filename}", file_id=file_id, detail=error_msg,
+            )
             return {"error": error_msg, "file": filename, "status": "Failed - Size limit exceeded"}
 
         # For PDF files, check page count against service limits
@@ -116,12 +126,27 @@ def process_with_azure_document_intelligence(filename: str, file_id: int = None)
             page_count = get_pdf_page_count(tmp_file_path)
             if page_count is not None and page_count > AZURE_DOC_INTELLIGENCE_LIMITS["max_pages"]:
                 error_msg = f"PDF page count ({page_count}) exceeds Azure Document Intelligence limit of 2000 pages"
-                logger.error(error_msg)
+                logger.error(f"[{task_id}] {error_msg}")
+                log_task_progress(
+                    task_id, "validate_file", "failure",
+                    f"Too many pages: {filename}", file_id=file_id, detail=error_msg,
+                )
                 return {"error": error_msg, "file": filename, "status": "Failed - Page limit exceeded"}
             if page_count is None:
-                logger.warning(f"Could not determine page count for {filename}, proceeding with processing anyway")
+                logger.warning(
+                    f"[{task_id}] Could not determine page count for {filename}, proceeding with processing anyway"
+                )
 
-        logger.info(f"Processing {filename} with Azure Document Intelligence OCR.")
+        log_task_progress(
+            task_id, "validate_file", "success",
+            f"File validation passed for {filename}", file_id=file_id,
+        )
+
+        logger.info(f"[{task_id}] Processing {filename} with Azure Document Intelligence OCR.")
+        log_task_progress(
+            task_id, "call_azure_ocr", "in_progress",
+            f"Sending {filename} to Azure Document Intelligence", file_id=file_id,
+        )
 
         # Open and send the document for processing
         with open(tmp_file_path, "rb") as f:
@@ -139,16 +164,32 @@ def process_with_azure_document_intelligence(filename: str, file_id: int = None)
         searchable_pdf_path = tmp_file_path  # Overwrite the original PDF location
         with open(searchable_pdf_path, "wb") as writer:
             writer.writelines(response)
-        logger.info(f"Searchable PDF saved at: {searchable_pdf_path}")
+        logger.info(f"[{task_id}] Searchable PDF saved at: {searchable_pdf_path}")
 
         # Extract raw text content from the result
         extracted_text = result.content if result.content else ""
-        logger.info(f"Extracted text for {filename}: {len(extracted_text)} characters")
+        logger.info(f"[{task_id}] Extracted text for {filename}: {len(extracted_text)} characters")
+
+        log_task_progress(
+            task_id, "call_azure_ocr", "success",
+            f"Azure OCR completed for {filename}", file_id=file_id,
+            detail=f"Extracted {len(extracted_text)} characters, {len(rotation_data)} rotated pages detected",
+        )
 
         # Trigger page rotation task if rotation is detected, otherwise proceed to metadata extraction
         rotate_pdf_pages.delay(filename, extracted_text, rotation_data, file_id)
 
+        log_task_progress(
+            task_id, "process_with_azure_document_intelligence", "success",
+            f"OCR processing complete for {filename}", file_id=file_id,
+            detail=f"Searchable PDF saved, {len(extracted_text)} characters extracted",
+        )
+
         return {"file": filename, "searchable_pdf": searchable_pdf_path, "cleaned_text": extracted_text}
     except Exception as e:
-        logger.error(f"Error processing {filename} with Azure Document Intelligence: {e}")
+        logger.error(f"[{task_id}] Error processing {filename} with Azure Document Intelligence: {e}")
+        log_task_progress(
+            task_id, "process_with_azure_document_intelligence", "failure",
+            f"OCR failed for {filename}", file_id=file_id, detail=str(e),
+        )
         raise