Merge branch 'main' into copilot/add-file-processing-notifications

app/tasks/finalize_document_storage.py

@@ -1,28 +1,52 @@
 #!/usr/bin/env python3
 
+import logging
 import os
 from app.config import settings
 from app.tasks.retry_config import BaseTaskWithRetry
 # Import the shared Celery instance
 from app.celery_app import celery
 
-# 1) Import the aggregator task
+# Import the aggregator task and validator
 from app.tasks.send_to_all import send_to_all_destinations, get_configured_services_from_validator
 
 # Import notification utility
 from app.utils.notification import notify_file_processed
 
+# Import database and logging utils from main
+from app.utils import log_task_progress
+from app.database import SessionLocal
+from app.models import FileRecord
 
-@celery.task(base=BaseTaskWithRetry)
-def finalize_document_storage(original_file: str, processed_file: str, metadata: dict):
+logger = logging.getLogger(__name__)
+
+
+@celery.task(base=BaseTaskWithRetry, bind=True)
+def finalize_document_storage(self, original_file: str, processed_file: str, metadata: dict, file_id: int = None):
     """
     Final storage step after embedding metadata.
     We will now call 'send_to_all_destinations' to push the final PDF to Dropbox/Nextcloud/Paperless.
     After uploading, send a notification about the processed file.
     """
-    print(f"[INFO] Finalizing document storage for {processed_file}")
+    task_id = self.request.id
+    logger.info(f"[{task_id}] Finalizing document storage for {processed_file}")
+    
+    # 1. Update Database Status (From Main)
+    log_task_progress(task_id, "finalize_document_storage", "in_progress", f"Finalizing: {os.path.basename(processed_file)}", file_id=file_id)
+    
+    # Get file_id from database if not provided (fallback logic from Main)
+    if file_id is None:
+        with SessionLocal() as db:
+            # Only as a last resort, try to find by exact match on local_filename
+            tmp_path = os.path.join(settings.workdir, "tmp", os.path.basename(original_file))
+            file_record = db.query(FileRecord).filter(
+                FileRecord.local_filename == tmp_path
+            ).first()
+            if file_record:
+                file_id = file_record.id
 
-    # Determine which destinations are configured
+    # 2. Determine Configured Destinations (From Copilot)
+    # This is needed for the notification message later
     configured_destinations = []
     try:
         configured_services = get_configured_services_from_validator()
@@ -33,16 +57,21 @@ def finalize_document_storage(original_file: str, processed_file: str, metadata:
                 display_name = service_name.replace('_', ' ').title()
                 configured_destinations.append(display_name)
     except Exception as e:
-        print(f"[WARNING] Could not determine configured destinations: {e}")
+        logger.warning(f"[WARNING] Could not determine configured destinations: {e}")
         configured_destinations = ["configured destinations"]
 
-    # 2) Enqueue uploads to all destinations (Dropbox, Nextcloud, Paperless)
+    # 3. Queue Uploads (Merged)
+    # Uses Main branch signature to ensure file_id is passed, but keeps logic structure
+    logger.info(f"[{task_id}] Queueing uploads to all destinations")
+    log_task_progress(task_id, "finalize_document_storage", "success", "Queuing uploads to destinations", file_id=file_id)
+    
     # Note: send_to_all_destinations is asynchronous and queues upload tasks
-    send_to_all_destinations.delay(processed_file)
+    # We pass 'True' (delete_after) and 'file_id' as per Main branch requirements
+    send_to_all_destinations.delay(processed_file, True, file_id)
 
-    # 3) Send notification about successful file processing
+    # 4. Send Notification (From Copilot)
     # Note: This notification is sent after processing is complete but while uploads
-    # are being queued. The message reflects that uploads are being initiated.
+    # are being queued.
     try:
         # Get file information
         file_size = os.path.getsize(processed_file) if os.path.exists(processed_file) else 0
@@ -55,9 +84,9 @@ def finalize_document_storage(original_file: str, processed_file: str, metadata:
             destinations=configured_destinations
         )
     except Exception as e:
-        print(f"[WARNING] Failed to send file processed notification: {e}")
+        logger.warning(f"[WARNING] Failed to send file processed notification: {e}")
 
     return {
         "status": "Completed",
         "file": processed_file
-    }
+    }