From 98df430f8f8a4f99a02caa1cb23e1a9814fcb5e4 Mon Sep 17 00:00:00 2001 From: Christian Krakau-Louis Date: Fri, 4 Apr 2025 01:30:14 +0200 Subject: [PATCH] feat: add PDF rotation functionality and integrate with Azure Document Intelligence processing --- VERSION | 2 +- app/celery_worker.py | 1 + ...rocess_with_azure_document_intelligence.py | 22 ++- app/tasks/rotate_pdf_pages.py | 133 ++++++++++++++++++ requirements.txt | 2 +- 5 files changed, 151 insertions(+), 9 deletions(-) create mode 100644 app/tasks/rotate_pdf_pages.py diff --git a/VERSION b/VERSION index 8cf9df5e..7017c79a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.3-dev +0.4.5-dev diff --git a/app/celery_worker.py b/app/celery_worker.py index c2ea5f0c..5c9aff15 100644 --- a/app/celery_worker.py +++ b/app/celery_worker.py @@ -11,6 +11,7 @@ from app import tasks # <— This imports app/tasks.py so Celery can register t # **Ensure all tasks are imported before Celery starts** from app.tasks.process_document import process_document from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence +from app.tasks.rotate_pdf_pages import rotate_pdf_pages from app.tasks.refine_text_with_gpt import refine_text_with_gpt from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf diff --git a/app/tasks/process_with_azure_document_intelligence.py b/app/tasks/process_with_azure_document_intelligence.py index 80f2c978..ecf9ee2f 100644 --- a/app/tasks/process_with_azure_document_intelligence.py +++ b/app/tasks/process_with_azure_document_intelligence.py @@ -7,7 +7,7 @@ from azure.ai.documentintelligence.models import AnalyzeOutputOption, AnalyzeRes from app.config import settings from app.tasks.retry_config import BaseTaskWithRetry -from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt +from app.tasks.rotate_pdf_pages import rotate_pdf_pages from app.celery_app import celery logger = logging.getLogger(__name__) @@ -41,22 +41,30 @@ def check_page_rotation(result, filename): Args: result: The AnalyzeResult from Azure Document Intelligence API filename: The name of the file being processed + + Returns: + dict: Dictionary mapping page indices (integers) to rotation angles """ logger.error(f"Checking rotation for document: {filename}") + rotation_data = {} if not hasattr(result, 'pages') or not result.pages: logger.error(f"No page information available for rotation check: {filename}") - return + return rotation_data for i, page in enumerate(result.pages): if hasattr(page, 'angle'): rotation_angle = page.angle if rotation_angle != 0: logger.error(f"Page {i+1} is rotated by {rotation_angle} degrees") + # Store page index as integer, not string + rotation_data[i] = rotation_angle else: logger.error(f"Page {i+1} has no rotation (0 degrees)") else: logger.error(f"Page {i+1} rotation information not available") + + return rotation_data @celery.task(base=BaseTaskWithRetry) def process_with_azure_document_intelligence(filename: str): @@ -69,8 +77,8 @@ def process_with_azure_document_intelligence(filename: str): 1. Uploads the document for OCR using Azure Document Intelligence. 2. Retrieves the processed PDF with embedded text. 3. Saves the OCR-processed PDF locally in the same location as before. - 4. Extracts the text content for metadata processing. - 5. Triggers downstream metadata extraction by calling extract_metadata_with_gpt. + 4. Checks for page rotation and triggers page rotation if needed. + 5. Triggers downstream metadata extraction. """ try: tmp_file_path = os.path.join(settings.workdir, "tmp", filename) @@ -106,7 +114,7 @@ def process_with_azure_document_intelligence(filename: str): operation_id = poller.details["operation_id"] # Check and log page rotation information - check_page_rotation(result, filename) + rotation_data = check_page_rotation(result, filename) # Retrieve the processed searchable PDF response = document_intelligence_client.get_analyze_result_pdf( @@ -121,8 +129,8 @@ def process_with_azure_document_intelligence(filename: str): extracted_text = result.content if result.content else "" logger.info(f"Extracted text for {filename}: {len(extracted_text)} characters") - # Trigger downstream metadata extraction - extract_metadata_with_gpt.delay(filename, extracted_text) + # Trigger page rotation task if rotation is detected, otherwise proceed to metadata extraction + rotate_pdf_pages.delay(filename, extracted_text, rotation_data) return {"file": filename, "searchable_pdf": searchable_pdf_path, "cleaned_text": extracted_text} except Exception as e: diff --git a/app/tasks/rotate_pdf_pages.py b/app/tasks/rotate_pdf_pages.py new file mode 100644 index 00000000..6ac38d4e --- /dev/null +++ b/app/tasks/rotate_pdf_pages.py @@ -0,0 +1,133 @@ +import os +import logging +import PyPDF2 +import math +import json + +from app.config import settings +from app.tasks.retry_config import BaseTaskWithRetry +from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt +from app.celery_app import celery + +logger = logging.getLogger(__name__) + +def determine_rotation_angle(detected_angle): + """ + Determine the optimal rotation angle based on detected angle. + + Args: + detected_angle: The angle detected by Azure Document Intelligence + + Returns: + int: The angle to rotate the page in PyPDF2 (must be multiple of 90 degrees) + """ + # Normalize angle to be between 0 and 360 + normalized_angle = detected_angle % 360 + if normalized_angle < 0: + normalized_angle += 360 + + # If angle is very small (< 1 degree), don't rotate + if abs(normalized_angle) < 1 or abs(normalized_angle - 360) < 1: + return 0 + + # For angles close to 90, 180, or 270 degrees (±5°), round to nearest 90° increment + for target in [90, 180, 270]: + if abs(normalized_angle - target) < 5: + # PyPDF2 uses clockwise rotation, so we need to use the complementary angle + rotation_value = (360 - target) % 360 + logger.info(f"Detected angle {detected_angle}° is close to {target}°, will rotate by {rotation_value}°") + return rotation_value + + # For other significant angles, round to nearest 90° increment + # (PyPDF2 only supports rotations in 90-degree increments) + closest_90_multiple = round(normalized_angle / 90) * 90 + # Convert to PyPDF2 rotation value (clockwise) + rotation_value = (360 - closest_90_multiple) % 360 + logger.info(f"Detected angle {detected_angle}° rounded to {closest_90_multiple}°, will rotate by {rotation_value}°") + return rotation_value + +@celery.task(base=BaseTaskWithRetry) +def rotate_pdf_pages(filename: str, extracted_text: str, rotation_data=None): + """ + Rotates pages in a PDF document based on detected rotation angles. + + Args: + filename: The name of the file to rotate + extracted_text: The extracted text from the document + rotation_data: Optional rotation data dictionary {page_index: angle} + """ + try: + pdf_path = os.path.join(settings.workdir, "tmp", filename) + if not os.path.exists(pdf_path): + raise FileNotFoundError(f"PDF file not found: {pdf_path}") + + # Skip rotation if no rotation data provided + if not rotation_data: + logger.info(f"No rotation data provided for {filename}, proceeding with metadata extraction") + extract_metadata_with_gpt.delay(filename, extracted_text) + return {"file": filename, "status": "no_rotation_needed"} + + # Standardize rotation_data keys to integers + normalized_rotation_data = {} + for key, value in rotation_data.items(): + try: + normalized_rotation_data[int(key)] = float(value) + except (ValueError, TypeError): + logger.warning(f"Invalid rotation data key-value: {key}:{value}") + + if not any(abs(angle) > 0 for angle in normalized_rotation_data.values()): + logger.info(f"No significant rotations detected in {filename}, proceeding with metadata extraction") + extract_metadata_with_gpt.delay(filename, extracted_text) + return {"file": filename, "status": "no_rotation_needed"} + + logger.info(f"Rotating {len(normalized_rotation_data)} pages in {filename}") + applied_rotations = {} + + # Load the PDF + with open(pdf_path, 'rb') as file: + pdf_reader = PyPDF2.PdfReader(file) + pdf_writer = PyPDF2.PdfWriter() + + # Process each page + for page_idx in range(len(pdf_reader.pages)): + page = pdf_reader.pages[page_idx] + + # Apply rotation if this page has rotation data + if page_idx in normalized_rotation_data and abs(normalized_rotation_data[page_idx]) > 0: + detected_angle = normalized_rotation_data[page_idx] + rotation_angle = determine_rotation_angle(detected_angle) + + if rotation_angle > 0: + # PyPDF2 uses clockwise rotation in 90-degree increments + page.rotate(rotation_angle) + logger.info(f"Page {page_idx+1} rotated by {rotation_angle}° (from detected {detected_angle}°)") + applied_rotations[str(page_idx)] = rotation_angle + else: + logger.info(f"Page {page_idx+1} had detected angle {detected_angle}° but determined it doesn't need rotation") + + pdf_writer.add_page(page) + + # Save the rotated PDF + with open(pdf_path, 'wb') as output_file: + pdf_writer.write(output_file) + + if applied_rotations: + logger.info(f"Successfully rotated PDF: {filename} with rotations: {json.dumps(applied_rotations)}") + else: + logger.info(f"Detected rotations in {filename} but no rotations were actually applied (angles too small or not multiples of 90°)") + + # Continue with metadata extraction + extract_metadata_with_gpt.delay(filename, extracted_text) + + return { + "file": filename, + "status": "rotated" if applied_rotations else "no_rotation_needed", + "detected_rotations": rotation_data, + "applied_rotations": applied_rotations + } + + except Exception as e: + logger.error(f"Error rotating PDF {filename}: {e}") + # Continue with metadata extraction despite rotation failure + extract_metadata_with_gpt.delay(filename, extracted_text) + return {"file": filename, "status": "rotation_failed", "error": str(e)} diff --git a/requirements.txt b/requirements.txt index 9394c34a..eebca058 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ sqlalchemy # Database ORM pydantic # Data validation openai # GPT integration for metadata extraction pymupdf # PDF processing, text extraction, and detection (imported as 'fitz') -PyPDF2 # PDF processing for page counting +PyPDF2 # PDF processing for page counting and now also for rotation requests # HTTP client dropbox>=11.36.0 # Dropbox integration azure-ai-documentintelligence # Azure OCR service