Merge pull request #77 from christianlouis/copilot/add-file-processing-notifications

Add file processing completion notifications
2026-02-07 15:37:59 +01:00
parent e8c091897a e45eb04366
commit 9672cb234b
6 changed files with 301 additions and 6 deletions
@@ -146,6 +146,7 @@ NOTIFY_ON_TASK_FAILURE=True
 NOTIFY_ON_CREDENTIAL_FAILURE=True
 NOTIFY_ON_STARTUP=True
 NOTIFY_ON_SHUTDOWN=False
+NOTIFY_ON_FILE_PROCESSED=True

 # Uptime Kuma
 UPTIME_KUMA_URL=https://status.example.com/api/push/abcdef123456?status=up
@@ -156,6 +156,10 @@ class Settings(BaseSettings):
        default=False,
        description="Send notifications when application shuts down"
    )
+    notify_on_file_processed: bool = Field(
+        default=True,
+        description="Send notifications when files are successfully processed"
+    )
    
    @validator('notification_urls', pre=True)
    def parse_notification_urls(cls, v):
@@ -7,8 +7,13 @@ from app.tasks.retry_config import BaseTaskWithRetry
 # Import the shared Celery instance
 from app.celery_app import celery

-# 1) Import the aggregator task
-from app.tasks.send_to_all import send_to_all_destinations
+# Import the aggregator task and validator
+from app.tasks.send_to_all import send_to_all_destinations, get_configured_services_from_validator
+
+# Import notification utility
+from app.utils.notification import notify_file_processed
+
+# Import database and logging utils from main
 from app.utils import log_task_progress
 from app.database import SessionLocal
 from app.models import FileRecord
@@ -21,16 +26,18 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
    """
    Final storage step after embedding metadata.
    We will now call 'send_to_all_destinations' to push the final PDF to Dropbox/Nextcloud/Paperless.
+    After uploading, send a notification about the processed file.
    """
    task_id = self.request.id
    logger.info(f"[{task_id}] Finalizing document storage for {processed_file}")
+    
+    # 1. Update Database Status (From Main)
    log_task_progress(task_id, "finalize_document_storage", "in_progress", f"Finalizing: {os.path.basename(processed_file)}", file_id=file_id)
    
-    # Get file_id from database if not provided (fallback only, prefer passing file_id explicitly)
+    # Get file_id from database if not provided (fallback logic from Main)
    if file_id is None:
        with SessionLocal() as db:
            # Only as a last resort, try to find by exact match on local_filename
-            # This should not be needed if file_id is passed correctly through the chain
            tmp_path = os.path.join(settings.workdir, "tmp", os.path.basename(original_file))
            file_record = db.query(FileRecord).filter(
                FileRecord.local_filename == tmp_path
@@ -38,12 +45,48 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
            if file_record:
                file_id = file_record.id

-    # 2) Enqueue uploads to all destinations (Dropbox, Nextcloud, Paperless)
+    # 2. Determine Configured Destinations (From Copilot)
+    # This is needed for the notification message later
+    configured_destinations = []
+    try:
+        configured_services = get_configured_services_from_validator()
+        # Get list of service names that are configured
+        for service_name, is_configured in configured_services.items():
+            if is_configured:
+                # Format service names for display
+                display_name = service_name.replace('_', ' ').title()
+                configured_destinations.append(display_name)
+    except Exception as e:
+        logger.warning(f"[WARNING] Could not determine configured destinations: {e}")
+        configured_destinations = ["configured destinations"]
+
+    # 3. Queue Uploads (Merged)
+    # Uses Main branch signature to ensure file_id is passed, but keeps logic structure
    logger.info(f"[{task_id}] Queueing uploads to all destinations")
    log_task_progress(task_id, "finalize_document_storage", "success", "Queuing uploads to destinations", file_id=file_id)
+    
+    # Note: send_to_all_destinations is asynchronous and queues upload tasks
+    # We pass 'True' (delete_after) and 'file_id' as per Main branch requirements
    send_to_all_destinations.delay(processed_file, True, file_id)

+    # 4. Send Notification (From Copilot)
+    # Note: This notification is sent after processing is complete but while uploads
+    # are being queued.
+    try:
+        # Get file information
+        file_size = os.path.getsize(processed_file) if os.path.exists(processed_file) else 0
+        filename = os.path.basename(processed_file)
+        
+        notify_file_processed(
+            filename=filename,
+            file_size=file_size,
+            metadata=metadata,
+            destinations=configured_destinations
+        )
+    except Exception as e:
+        logger.warning(f"[WARNING] Failed to send file processed notification: {e}")
+
    return {
        "status": "Completed",
        "file": processed_file
-    }
+    }
@@ -183,3 +183,38 @@ def notify_shutdown() -> bool:
        notification_type="info",
        tags=["system", "shutdown"]
    )
+
+def notify_file_processed(filename: str, file_size: int, metadata: dict, destinations: list) -> bool:
+    """Send a notification that a file has been successfully processed"""
+    if not settings.notify_on_file_processed:
+        return False
+    
+    # Format file size for display
+    size_mb = file_size / (1024 * 1024)
+    size_str = f"{size_mb:.2f} MB" if size_mb >= 1 else f"{file_size / 1024:.2f} KB"
+    
+    # Extract key metadata fields
+    doc_type = metadata.get('document_type', 'Unknown')
+    tags = metadata.get('tags', [])
+    tags_str = ', '.join(tags) if tags else 'None'
+    
+    # Format destinations
+    destinations_str = ', '.join(destinations) if destinations else 'None configured'
+    
+    title = f"File Processed: {filename}"
+    message = f"""
+File: {filename}
+Size: {size_str}
+Document Type: {doc_type}
+Tags: {tags_str}
+Destinations: {destinations_str}
+
+The file has been successfully processed and is being uploaded to all configured destinations.
+"""
+    
+    return send_notification(
+        title=title,
+        message=message.strip(),
+        notification_type="success",
+        tags=["document", "processed", "success"]
+    )
@@ -11,6 +11,7 @@ This guide explains how to set up the notification system for DocuElevate, which
 | `NOTIFY_ON_CREDENTIAL_FAILURE` | Send notifications on credential failures (`True`/`False`) |
 | `NOTIFY_ON_STARTUP`        | Send notification when system starts (`True`/`False`)    |
 | `NOTIFY_ON_SHUTDOWN`       | Send notification when system shuts down (`True`/`False`)|
+| `NOTIFY_ON_FILE_PROCESSED` | Send notifications when files are processed (`True`/`False`)|

 For a complete list of configuration options, see the [Configuration Guide](ConfigurationGuide.md).

@@ -61,6 +62,42 @@ NOTIFY_ON_TASK_FAILURE=True        # Notify when background tasks fail
 NOTIFY_ON_CREDENTIAL_FAILURE=True  # Notify when service credentials fail (e.g. API token expired)
 NOTIFY_ON_STARTUP=True             # Notify when the system starts
 NOTIFY_ON_SHUTDOWN=False           # Notify when the system shuts down
+NOTIFY_ON_FILE_PROCESSED=True      # Notify when files are successfully processed
+```
+
+## File Processing Notifications
+
+DocuElevate can notify you whenever a document is successfully processed and uploaded to your configured destinations. When enabled, you'll receive a notification containing:
+
+- **File name**: The name of the processed document
+- **File size**: Size of the processed file
+- **Document type**: The type of document detected by AI (invoice, receipt, contract, etc.)
+- **Tags**: Any tags extracted from the document
+- **Destinations**: List of destinations where the file was uploaded
+
+This feature is particularly useful for:
+- Tracking document processing in real-time
+- Confirming important documents have been processed
+- Monitoring the document processing workflow
+- Receiving immediate feedback when uploading documents via email or API
+
+To enable file processing notifications:
+
+```dotenv
+NOTIFY_ON_FILE_PROCESSED=True
+```
+
+Example notification:
+```
+Subject: File Processed: Invoice_2024-01-15.pdf
+
+File: Invoice_2024-01-15.pdf
+Size: 1.23 MB
+Document Type: Invoice
+Tags: financial, vendor-acme, urgent
+Destinations: Dropbox, Google Drive, Paperless-ngx
+
+The file has been successfully processed and is being uploaded to all configured destinations.
 ```

 ## Automated Credential Checking
@@ -0,0 +1,175 @@
+import os
+import unittest
+from unittest.mock import patch, MagicMock
+from app.utils.notification import notify_file_processed, send_notification
+from app.config import settings
+
+
+class TestFileProcessedNotification(unittest.TestCase):
+    """Test file processing notification functionality"""
+    
+    @patch('app.utils.notification.send_notification')
+    def test_notify_file_processed_when_enabled(self, mock_send):
+        """Test that notification is sent when NOTIFY_ON_FILE_PROCESSED is True"""
+        # Arrange
+        mock_send.return_value = True
+        original_value = settings.notify_on_file_processed
+        settings.notify_on_file_processed = True
+        
+        filename = "test_document.pdf"
+        file_size = 1024 * 1024  # 1 MB
+        metadata = {
+            'document_type': 'Invoice',
+            'tags': ['financial', 'urgent']
+        }
+        destinations = ['Dropbox', 'Google Drive']
+        
+        try:
+            # Act
+            result = notify_file_processed(filename, file_size, metadata, destinations)
+            
+            # Assert
+            self.assertTrue(result)
+            mock_send.assert_called_once()
+            
+            # Check the call arguments
+            call_args = mock_send.call_args
+            self.assertIn("test_document.pdf", call_args[1]['title'])
+            self.assertIn("Invoice", call_args[1]['message'])
+            self.assertIn("financial, urgent", call_args[1]['message'])
+            self.assertIn("Dropbox, Google Drive", call_args[1]['message'])
+            self.assertEqual(call_args[1]['notification_type'], "success")
+        finally:
+            settings.notify_on_file_processed = original_value
+    
+    @patch('app.utils.notification.send_notification')
+    def test_notify_file_processed_when_disabled(self, mock_send):
+        """Test that notification is not sent when NOTIFY_ON_FILE_PROCESSED is False"""
+        # Arrange
+        original_value = settings.notify_on_file_processed
+        settings.notify_on_file_processed = False
+        
+        filename = "test_document.pdf"
+        file_size = 1024 * 1024
+        metadata = {'document_type': 'Invoice', 'tags': []}
+        destinations = ['Dropbox']
+        
+        try:
+            # Act
+            result = notify_file_processed(filename, file_size, metadata, destinations)
+            
+            # Assert
+            self.assertFalse(result)
+            mock_send.assert_not_called()
+        finally:
+            settings.notify_on_file_processed = original_value
+    
+    @patch('app.utils.notification.send_notification')
+    def test_notify_file_processed_with_no_destinations(self, mock_send):
+        """Test notification when no destinations are configured"""
+        # Arrange
+        mock_send.return_value = True
+        original_value = settings.notify_on_file_processed
+        settings.notify_on_file_processed = True
+        
+        filename = "test_document.pdf"
+        file_size = 512 * 1024  # 512 KB
+        metadata = {
+            'document_type': 'Receipt',
+            'tags': []
+        }
+        destinations = []
+        
+        try:
+            # Act
+            result = notify_file_processed(filename, file_size, metadata, destinations)
+            
+            # Assert
+            self.assertTrue(result)
+            mock_send.assert_called_once()
+            
+            # Check that message indicates no destinations
+            call_args = mock_send.call_args
+            self.assertIn("None configured", call_args[1]['message'])
+        finally:
+            settings.notify_on_file_processed = original_value
+    
+    @patch('app.utils.notification.send_notification')
+    def test_notify_file_processed_formats_file_size_mb(self, mock_send):
+        """Test that file size is formatted correctly for MB"""
+        # Arrange
+        mock_send.return_value = True
+        original_value = settings.notify_on_file_processed
+        settings.notify_on_file_processed = True
+        
+        filename = "large_document.pdf"
+        file_size = 5 * 1024 * 1024  # 5 MB
+        metadata = {'document_type': 'Contract', 'tags': []}
+        destinations = ['Dropbox']
+        
+        try:
+            # Act
+            result = notify_file_processed(filename, file_size, metadata, destinations)
+            
+            # Assert
+            self.assertTrue(result)
+            call_args = mock_send.call_args
+            self.assertIn("5.00 MB", call_args[1]['message'])
+        finally:
+            settings.notify_on_file_processed = original_value
+    
+    @patch('app.utils.notification.send_notification')
+    def test_notify_file_processed_formats_file_size_kb(self, mock_send):
+        """Test that file size is formatted correctly for KB"""
+        # Arrange
+        mock_send.return_value = True
+        original_value = settings.notify_on_file_processed
+        settings.notify_on_file_processed = True
+        
+        filename = "small_document.pdf"
+        file_size = 512 * 1024  # 512 KB
+        metadata = {'document_type': 'Note', 'tags': []}
+        destinations = ['Dropbox']
+        
+        try:
+            # Act
+            result = notify_file_processed(filename, file_size, metadata, destinations)
+            
+            # Assert
+            self.assertTrue(result)
+            call_args = mock_send.call_args
+            self.assertIn("512.00 KB", call_args[1]['message'])
+        finally:
+            settings.notify_on_file_processed = original_value
+    
+    @patch('app.utils.notification.send_notification')
+    def test_notify_file_processed_with_missing_metadata_fields(self, mock_send):
+        """Test that notification handles missing metadata fields gracefully"""
+        # Arrange
+        mock_send.return_value = True
+        original_value = settings.notify_on_file_processed
+        settings.notify_on_file_processed = True
+        
+        filename = "document.pdf"
+        file_size = 1024 * 1024
+        metadata = {}  # Empty metadata
+        destinations = ['Dropbox']
+        
+        try:
+            # Act
+            result = notify_file_processed(filename, file_size, metadata, destinations)
+            
+            # Assert
+            self.assertTrue(result)
+            mock_send.assert_called_once()
+            
+            # Check that defaults are used
+            call_args = mock_send.call_args
+            self.assertIn("Unknown", call_args[1]['message'])  # Default document type
+            self.assertIn("None", call_args[1]['message'])  # No tags
+        finally:
+            settings.notify_on_file_processed = original_value
+
+
+if __name__ == '__main__':
+    unittest.main()