feat(tasks): dynamic routing to user-specific destination integrations
- Add app/tasks/upload_to_user_integration.py: new Celery task that uploads a processed document to a specific UserIntegration using its own per-user config and Fernet-decrypted credentials. Supports all DESTINATION types: Dropbox, S3, Google Drive, OneDrive, WebDAV, Nextcloud, FTP, SFTP, Paperless-ngx, Email (SMTP), and Rclone. - Extend app/tasks/send_to_all.py: add send_to_user_destinations task (queries active DESTINATION UserIntegrations for an owner and dispatches one upload_to_user_integration task per integration) and get_user_destination_count helper used by finalize_document_storage. - Refactor app/tasks/finalize_document_storage.py: after processing, look up the document owner; if the owner has active DESTINATION integrations route exclusively to those (user-specific routing), otherwise fall back to the global send_to_all_destinations. - Update tests/test_finalize_storage.py: add autouse fixture to prevent Redis hangs, update all existing tests with new mock parameters, add TestFinalizeDocumentStorageUserRouting class with four new tests that validate user-specific vs global routing decisions. - Add tests/test_user_integration_upload.py: 14 new unit tests covering upload_to_user_integration (handler dispatch, error persistence, last_used_at update, credential decryption, skip for unknown types) and send_to_user_destinations / get_user_destination_count. - Update docs/StorageArchitecture.md: document the user-specific destination routing feature, supported types, multiple-destination behaviour, and global fallback semantics. Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -10,8 +10,13 @@ from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
|
||||
# Import the aggregator task and validator
|
||||
from app.tasks.send_to_all import get_configured_services_from_validator, send_to_all_destinations
|
||||
# Import the aggregator tasks and validator
|
||||
from app.tasks.send_to_all import (
|
||||
get_configured_services_from_validator,
|
||||
get_user_destination_count,
|
||||
send_to_all_destinations,
|
||||
send_to_user_destinations,
|
||||
)
|
||||
|
||||
# Import database and logging utils from main
|
||||
from app.utils import log_task_progress
|
||||
@@ -26,8 +31,16 @@ logger = logging.getLogger(__name__)
|
||||
def finalize_document_storage(self, original_file: str, processed_file: str, metadata: dict, file_id: int = None):
|
||||
"""
|
||||
Final storage step after embedding metadata.
|
||||
We will now call 'send_to_all_destinations' to push the final PDF to Dropbox/Nextcloud/Paperless.
|
||||
After uploading, send a notification about the processed file.
|
||||
Routes the processed document to the appropriate destination(s):
|
||||
|
||||
1. If the document has an identified owner and that owner has active
|
||||
DESTINATION UserIntegrations, the file is uploaded to each of those
|
||||
integrations (user-specific routing).
|
||||
2. Otherwise the file is forwarded to the globally-configured destinations
|
||||
via :func:`send_to_all_destinations` (system-wide fallback).
|
||||
|
||||
After queuing uploads, optional PDF/A archival conversion and embedding
|
||||
computation are triggered, and a completion notification is sent.
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Finalizing document storage for {processed_file}")
|
||||
@@ -41,7 +54,8 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Get file_id from database if not provided (fallback logic)
|
||||
# 2. Resolve file_id and owner_id from the database
|
||||
owner_id = None
|
||||
if file_id is None:
|
||||
with SessionLocal() as db:
|
||||
# Only as a last resort, try to find by exact match on local_filename
|
||||
@@ -49,32 +63,52 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
|
||||
file_record = db.query(FileRecord).filter(FileRecord.local_filename == tmp_path).first()
|
||||
if file_record:
|
||||
file_id = file_record.id
|
||||
owner_id = file_record.owner_id
|
||||
else:
|
||||
with SessionLocal() as db:
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
if file_record:
|
||||
owner_id = file_record.owner_id
|
||||
|
||||
# 2. Determine Configured Destinations
|
||||
# This is needed for the notification message later
|
||||
# 3. Determine configured destinations for notification
|
||||
configured_destinations = []
|
||||
try:
|
||||
configured_services = get_configured_services_from_validator()
|
||||
# Get list of service names that are configured
|
||||
for service_name, is_configured in configured_services.items():
|
||||
if is_configured:
|
||||
# Format service names for display
|
||||
display_name = service_name.replace("_", " ").title()
|
||||
configured_destinations.append(display_name)
|
||||
except Exception as e:
|
||||
logger.warning(f"[WARNING] Could not determine configured destinations: {e}")
|
||||
configured_destinations = ["configured destinations"]
|
||||
|
||||
# 3. Queue Uploads
|
||||
logger.info(f"[{task_id}] Queueing uploads to all destinations")
|
||||
# 4. Queue Uploads — prefer user-specific destinations when available
|
||||
log_task_progress(
|
||||
task_id, "finalize_document_storage", "success", "Queuing uploads to destinations", file_id=file_id
|
||||
)
|
||||
|
||||
# Note: send_to_all_destinations is asynchronous and queues upload tasks
|
||||
send_to_all_destinations.delay(processed_file, True, file_id)
|
||||
user_dest_count = 0
|
||||
if owner_id:
|
||||
try:
|
||||
user_dest_count = get_user_destination_count(owner_id)
|
||||
except Exception as e:
|
||||
logger.warning("[%s] Could not query user destination count for owner=%s: %s", task_id, owner_id, e)
|
||||
|
||||
# 3a. Trigger PDF/A archival conversion if enabled (from feature branch)
|
||||
if owner_id and user_dest_count > 0:
|
||||
# User has configured their own destinations → use those exclusively
|
||||
logger.info(
|
||||
"[%s] Routing to %d user-specific destination(s) for owner=%s",
|
||||
task_id,
|
||||
user_dest_count,
|
||||
owner_id,
|
||||
)
|
||||
send_to_user_destinations.delay(processed_file, owner_id, file_id)
|
||||
else:
|
||||
# No user-specific destinations → fall back to global configuration
|
||||
logger.info("[%s] No user-specific destinations found; using global destinations", task_id)
|
||||
send_to_all_destinations.delay(processed_file, True, file_id)
|
||||
|
||||
# 4a. Trigger PDF/A archival conversion if enabled
|
||||
if settings.enable_pdfa_conversion:
|
||||
try:
|
||||
from app.tasks.convert_to_pdfa import convert_to_pdfa
|
||||
@@ -84,7 +118,7 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
|
||||
except Exception as e:
|
||||
logger.warning(f"[{task_id}] Could not queue PDF/A conversion: {e}")
|
||||
|
||||
# 3b. Queue embedding computation (from main branch)
|
||||
# 4b. Queue embedding computation
|
||||
if file_id is not None:
|
||||
try:
|
||||
from app.tasks.compute_embedding import compute_document_embedding
|
||||
@@ -94,9 +128,8 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
|
||||
except Exception as e:
|
||||
logger.warning(f"[{task_id}] Could not queue embedding task: {e}")
|
||||
|
||||
# 4. Send Notification
|
||||
# 5. Send Notification
|
||||
try:
|
||||
# Get file information
|
||||
file_size = os.path.getsize(processed_file) if os.path.exists(processed_file) else 0
|
||||
filename = os.path.basename(processed_file)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user