copilot-swe-agent[bot]
a08b103271
Merges origin/main (v0.156.0) into the classification feature branch, properly resolving all 23 merge conflicts: - Auto-generated files (BUILD_DATE, VERSION, etc.): accept main's version - Non-classification files (SharePoint, QR auth, session mgmt, mobile): accept main's version - Classification files (api/__init__.py, models.py, migrations/env.py, conftest.py): keep classification additions alongside main's content Previously the branch was incorrectly removing files from main (SharePoint integration, QR scanner, session management). This merge properly preserves all main branch content while maintaining the classification feature additions. Migration chain validated: 038_add_classification_rules chains from 037_add_user_sessions_and_qr_challenges. Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
452 lines
17 KiB
Python
452 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import logging
|
|
import os
|
|
|
|
from app.celery_app import celery
|
|
from app.config import settings
|
|
from app.database import SessionLocal
|
|
from app.models import FileRecord, IntegrationDirection, UserIntegration
|
|
from app.tasks.retry_config import BaseTaskWithRetry
|
|
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
|
from app.tasks.upload_to_email import upload_to_email
|
|
from app.tasks.upload_to_ftp import upload_to_ftp
|
|
from app.tasks.upload_to_google_drive import upload_to_google_drive
|
|
from app.tasks.upload_to_icloud import upload_to_icloud
|
|
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
|
from app.tasks.upload_to_onedrive import upload_to_onedrive
|
|
from app.tasks.upload_to_paperless import upload_to_paperless
|
|
from app.tasks.upload_to_s3 import upload_to_s3
|
|
from app.tasks.upload_to_sftp import upload_to_sftp
|
|
from app.tasks.upload_to_sharepoint import upload_to_sharepoint
|
|
from app.tasks.upload_to_webdav import upload_to_webdav
|
|
from app.utils.config_validator import get_provider_status
|
|
from app.utils.logging import log_task_progress
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _should_upload_to_dropbox():
|
|
return bool(
|
|
getattr(settings, "dropbox_enabled", True)
|
|
and settings.dropbox_app_key
|
|
and settings.dropbox_app_secret
|
|
and settings.dropbox_refresh_token
|
|
)
|
|
|
|
|
|
def _should_upload_to_nextcloud():
|
|
return bool(
|
|
getattr(settings, "nextcloud_enabled", True)
|
|
and settings.nextcloud_upload_url
|
|
and settings.nextcloud_username
|
|
and settings.nextcloud_password
|
|
)
|
|
|
|
|
|
def _should_upload_to_paperless():
|
|
return bool(
|
|
getattr(settings, "paperless_enabled", True) and settings.paperless_ngx_api_token and settings.paperless_host
|
|
)
|
|
|
|
|
|
def _should_upload_to_google_drive():
|
|
if not getattr(settings, "google_drive_enabled", True):
|
|
return False
|
|
# Check for OAuth configuration
|
|
if getattr(settings, "google_drive_use_oauth", False):
|
|
return bool(
|
|
settings.google_drive_client_id
|
|
and settings.google_drive_client_secret
|
|
and settings.google_drive_refresh_token
|
|
and settings.google_drive_folder_id
|
|
)
|
|
# Or check for service account configuration
|
|
else:
|
|
return bool(settings.google_drive_credentials_json and settings.google_drive_folder_id)
|
|
|
|
|
|
def _should_upload_to_webdav():
|
|
return bool(
|
|
getattr(settings, "webdav_enabled", True)
|
|
and settings.webdav_url
|
|
and settings.webdav_username
|
|
and settings.webdav_password
|
|
)
|
|
|
|
|
|
def _should_upload_to_ftp():
|
|
return bool(
|
|
getattr(settings, "ftp_enabled", True) and settings.ftp_host and settings.ftp_username and settings.ftp_password
|
|
)
|
|
|
|
|
|
def _should_upload_to_sftp():
|
|
return bool(
|
|
getattr(settings, "sftp_enabled", True)
|
|
and settings.sftp_host
|
|
and settings.sftp_username
|
|
and (settings.sftp_password or settings.sftp_private_key)
|
|
)
|
|
|
|
|
|
def _should_upload_to_email():
|
|
return bool(
|
|
getattr(settings, "dest_email_enabled", True)
|
|
and settings.dest_email_host
|
|
and settings.dest_email_username
|
|
and settings.dest_email_password
|
|
and settings.dest_email_default_recipient
|
|
)
|
|
|
|
|
|
def _should_upload_to_onedrive():
|
|
return bool(
|
|
getattr(settings, "onedrive_enabled", True)
|
|
and settings.onedrive_client_id
|
|
and settings.onedrive_client_secret
|
|
and settings.onedrive_refresh_token
|
|
)
|
|
|
|
|
|
def _should_upload_to_s3():
|
|
return bool(
|
|
getattr(settings, "s3_enabled", True)
|
|
and settings.s3_bucket_name
|
|
and settings.aws_access_key_id
|
|
and settings.aws_secret_access_key
|
|
)
|
|
|
|
|
|
def _should_upload_to_icloud():
|
|
return bool(getattr(settings, "icloud_enabled", True) and settings.icloud_username and settings.icloud_password)
|
|
|
|
|
|
def _should_upload_to_sharepoint():
|
|
return bool(
|
|
settings.sharepoint_client_id
|
|
and settings.sharepoint_client_secret
|
|
and settings.sharepoint_site_url
|
|
and (
|
|
settings.sharepoint_refresh_token
|
|
or (settings.sharepoint_tenant_id and settings.sharepoint_tenant_id != "common")
|
|
)
|
|
)
|
|
|
|
|
|
def get_configured_services_from_validator():
|
|
"""
|
|
Use the config validator to determine which services are configured and enabled.
|
|
Returns a dictionary with service names as keys and boolean values indicating
|
|
whether they're properly configured AND explicitly enabled.
|
|
"""
|
|
providers = get_provider_status()
|
|
|
|
service_map = {
|
|
"Dropbox": "dropbox",
|
|
"NextCloud": "nextcloud",
|
|
"Paperless-ngx": "paperless",
|
|
"Google Drive": "google_drive",
|
|
"WebDAV": "webdav",
|
|
"FTP Storage": "ftp",
|
|
"SFTP Storage": "sftp",
|
|
"Email": "email",
|
|
"OneDrive": "onedrive",
|
|
"S3 Storage": "s3",
|
|
"SharePoint": "sharepoint",
|
|
"iCloud Drive": "icloud",
|
|
}
|
|
|
|
result = {}
|
|
for provider_name, internal_name in service_map.items():
|
|
if provider_name in providers:
|
|
provider = providers[provider_name]
|
|
result[internal_name] = provider.get("configured", False) and provider.get("enabled", True)
|
|
|
|
return result
|
|
|
|
|
|
@celery.task(base=BaseTaskWithRetry, bind=True)
|
|
def send_to_all_destinations(self, file_path: str, use_validator=True, file_id: int = None, folder_overrides=None):
|
|
"""
|
|
Distribute a file to all configured storage destinations.
|
|
|
|
Args:
|
|
file_path: Path to the file to distribute
|
|
use_validator: Whether to use the config validator to determine enabled services
|
|
(if False, falls back to individual checks)
|
|
file_id: Optional file ID to associate with logs
|
|
folder_overrides: Optional dict mapping provider names to folder override strings.
|
|
When set, the override is passed to the upload task which uses it
|
|
instead of the provider's default folder. Example:
|
|
{"dropbox": "/Documents/pdfa", "s3": "docs/pdfa/"}
|
|
"""
|
|
task_id = self.request.id
|
|
|
|
if not os.path.exists(file_path):
|
|
logger.error(f"[{task_id}] File not found: {file_path}")
|
|
log_task_progress(task_id, "send_to_all_destinations", "failure", "File not found", file_id=file_id)
|
|
raise FileNotFoundError(f"File not found: {file_path}")
|
|
|
|
logger.info(f"[{task_id}] Sending {file_path} to all configured destinations")
|
|
log_task_progress(
|
|
task_id,
|
|
"send_to_all_destinations",
|
|
"in_progress",
|
|
f"Distributing: {os.path.basename(file_path)}",
|
|
file_id=file_id,
|
|
)
|
|
|
|
# Get file_id from database if not provided (fallback only, prefer passing file_id explicitly)
|
|
if file_id is None:
|
|
with SessionLocal() as db:
|
|
# Only as a last resort, try to find by basename match
|
|
# This should not be needed if file_id is passed correctly through the chain
|
|
file_record = (
|
|
db.query(FileRecord)
|
|
.filter(FileRecord.local_filename == os.path.join(settings.workdir, "tmp", os.path.basename(file_path)))
|
|
.first()
|
|
)
|
|
if file_record:
|
|
file_id = file_record.id
|
|
|
|
results = {}
|
|
|
|
# Define service configurations
|
|
services = [
|
|
{
|
|
"name": "dropbox",
|
|
"should_upload": _should_upload_to_dropbox,
|
|
"upload_func": upload_to_dropbox,
|
|
},
|
|
{
|
|
"name": "nextcloud",
|
|
"should_upload": _should_upload_to_nextcloud,
|
|
"upload_func": upload_to_nextcloud,
|
|
},
|
|
{
|
|
"name": "paperless",
|
|
"should_upload": _should_upload_to_paperless,
|
|
"upload_func": upload_to_paperless,
|
|
},
|
|
{
|
|
"name": "google_drive",
|
|
"should_upload": _should_upload_to_google_drive,
|
|
"upload_func": upload_to_google_drive,
|
|
},
|
|
{
|
|
"name": "webdav",
|
|
"should_upload": _should_upload_to_webdav,
|
|
"upload_func": upload_to_webdav,
|
|
},
|
|
{
|
|
"name": "ftp",
|
|
"should_upload": _should_upload_to_ftp,
|
|
"upload_func": upload_to_ftp,
|
|
},
|
|
{
|
|
"name": "sftp",
|
|
"should_upload": _should_upload_to_sftp,
|
|
"upload_func": upload_to_sftp,
|
|
},
|
|
{
|
|
"name": "email",
|
|
"should_upload": _should_upload_to_email,
|
|
"upload_func": upload_to_email,
|
|
},
|
|
{
|
|
"name": "onedrive",
|
|
"should_upload": _should_upload_to_onedrive,
|
|
"upload_func": upload_to_onedrive,
|
|
},
|
|
{
|
|
"name": "s3",
|
|
"should_upload": _should_upload_to_s3,
|
|
"upload_func": upload_to_s3,
|
|
},
|
|
{
|
|
"name": "sharepoint",
|
|
"should_upload": _should_upload_to_sharepoint,
|
|
"upload_func": upload_to_sharepoint,
|
|
},
|
|
{
|
|
"name": "icloud",
|
|
"should_upload": _should_upload_to_icloud,
|
|
"upload_func": upload_to_icloud,
|
|
},
|
|
]
|
|
|
|
# Optionally get configuration status from validator
|
|
configured_services = {}
|
|
if use_validator:
|
|
try:
|
|
configured_services = get_configured_services_from_validator()
|
|
logger.info(f"[{task_id}] Configured services according to validator: {configured_services}")
|
|
except Exception as e:
|
|
logger.warning(f"[{task_id}] Failed to get configuration from validator: {str(e)}")
|
|
use_validator = False
|
|
|
|
# Process each service
|
|
queued_count = 0
|
|
for service in services:
|
|
service_name = service["name"]
|
|
|
|
# Determine if service is configured
|
|
is_configured = False
|
|
if use_validator and service_name in configured_services:
|
|
is_configured = configured_services[service_name]
|
|
logger.debug(f"[{task_id}] {service_name} configuration from validator: {is_configured}")
|
|
else:
|
|
try:
|
|
is_configured = service["should_upload"]()
|
|
logger.debug(f"[{task_id}] {service_name} configuration from function: {is_configured}")
|
|
except Exception as e:
|
|
logger.error(f"[{task_id}] Error checking configuration for {service_name}: {str(e)}")
|
|
is_configured = False
|
|
|
|
# Queue the upload task if service is configured
|
|
if is_configured:
|
|
logger.info(f"[{task_id}] Queueing {file_path} for {service_name} upload")
|
|
log_task_progress(
|
|
task_id, f"queue_{service_name}", "in_progress", f"Queueing upload to {service_name}", file_id=file_id
|
|
)
|
|
try:
|
|
kwargs = {"file_id": file_id}
|
|
if folder_overrides and service_name in folder_overrides:
|
|
kwargs["folder_override"] = folder_overrides[service_name]
|
|
task = service["upload_func"].delay(file_path, **kwargs)
|
|
results[f"{service_name}_task_id"] = task.id
|
|
queued_count += 1
|
|
log_task_progress(
|
|
task_id, f"queue_{service_name}", "success", f"Queued for {service_name}", file_id=file_id
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"[{task_id}] Failed to queue {service_name} task: {str(e)}")
|
|
results[f"{service_name}_error"] = str(e)
|
|
log_task_progress(task_id, f"queue_{service_name}", "failure", f"Failed: {str(e)}", file_id=file_id)
|
|
|
|
logger.info(f"[{task_id}] Queued {queued_count} upload tasks")
|
|
log_task_progress(task_id, "send_to_all_destinations", "success", f"Queued {queued_count} uploads", file_id=file_id)
|
|
|
|
return {"status": "Queued", "file_path": file_path, "tasks": results}
|
|
|
|
|
|
@celery.task(base=BaseTaskWithRetry, bind=True)
|
|
def send_to_user_destinations(self, file_path: str, owner_id: str, file_id: int | None = None):
|
|
"""Dispatch uploads to all active DESTINATION UserIntegrations for *owner_id*.
|
|
|
|
This is the user-specific counterpart of :func:`send_to_all_destinations`.
|
|
It queries the ``user_integrations`` table for records where:
|
|
|
|
* ``owner_id`` matches the document owner,
|
|
* ``direction == "DESTINATION"``, and
|
|
* ``is_active == True``.
|
|
|
|
One :func:`upload_to_user_integration` Celery task is queued for each
|
|
matching integration so that uploads proceed asynchronously and
|
|
independently.
|
|
|
|
Args:
|
|
file_path: Absolute path to the processed document file.
|
|
owner_id: The stable user identifier from ``FileRecord.owner_id``.
|
|
file_id: Optional ``FileRecord.id`` used for progress logging.
|
|
|
|
Returns:
|
|
A dict summarising how many integrations were queued.
|
|
"""
|
|
from app.tasks.upload_to_user_integration import upload_to_user_integration
|
|
|
|
task_id = self.request.id
|
|
filename = os.path.basename(file_path)
|
|
|
|
if not os.path.exists(file_path):
|
|
error_msg = f"File not found: {file_path}"
|
|
logger.error("[%s] %s", task_id, error_msg)
|
|
log_task_progress(task_id, "send_to_user_destinations", "failure", error_msg, file_id=file_id)
|
|
raise FileNotFoundError(error_msg)
|
|
|
|
logger.info("[%s] Sending %s to user destinations for owner=%s", task_id, filename, owner_id)
|
|
log_task_progress(
|
|
task_id,
|
|
"send_to_user_destinations",
|
|
"in_progress",
|
|
f"Distributing {filename} to user integrations",
|
|
file_id=file_id,
|
|
)
|
|
|
|
with SessionLocal() as db:
|
|
integrations = (
|
|
db.query(UserIntegration)
|
|
.filter(
|
|
UserIntegration.owner_id == owner_id,
|
|
UserIntegration.direction == IntegrationDirection.DESTINATION,
|
|
UserIntegration.is_active.is_(True),
|
|
)
|
|
.all()
|
|
)
|
|
# Snapshot the IDs so we don't keep the session open
|
|
integration_ids = [(i.id, i.name, i.integration_type) for i in integrations]
|
|
|
|
queued = 0
|
|
task_results: dict[str, str] = {}
|
|
|
|
for int_id, int_name, int_type in integration_ids:
|
|
logger.info("[%s] Queueing upload for integration %d (%s '%s')", task_id, int_id, int_type, int_name)
|
|
log_task_progress(
|
|
task_id,
|
|
f"queue_user_integration_{int_id}",
|
|
"in_progress",
|
|
f"Queueing upload to {int_type} '{int_name}'",
|
|
file_id=file_id,
|
|
)
|
|
try:
|
|
celery_task = upload_to_user_integration.delay(file_path, int_id, file_id)
|
|
task_results[f"integration_{int_id}_task_id"] = celery_task.id
|
|
queued += 1
|
|
log_task_progress(
|
|
task_id,
|
|
f"queue_user_integration_{int_id}",
|
|
"success",
|
|
f"Queued upload to {int_type} '{int_name}'",
|
|
file_id=file_id,
|
|
)
|
|
except Exception as exc: # noqa: BLE001
|
|
error_msg = str(exc)
|
|
logger.error("[%s] Failed to queue upload for integration %d: %s", task_id, int_id, error_msg)
|
|
task_results[f"integration_{int_id}_error"] = error_msg
|
|
log_task_progress(
|
|
task_id,
|
|
f"queue_user_integration_{int_id}",
|
|
"failure",
|
|
f"Failed to queue {int_type} '{int_name}': {error_msg}",
|
|
file_id=file_id,
|
|
)
|
|
|
|
logger.info("[%s] Queued %d user-integration upload(s) for owner=%s", task_id, queued, owner_id)
|
|
log_task_progress(
|
|
task_id,
|
|
"send_to_user_destinations",
|
|
"success",
|
|
f"Queued {queued} user-integration upload(s)",
|
|
file_id=file_id,
|
|
)
|
|
return {"status": "Queued", "file_path": file_path, "queued": queued, "tasks": task_results}
|
|
|
|
|
|
def get_user_destination_count(owner_id: str) -> int:
|
|
"""Return the number of active DESTINATION integrations for *owner_id*.
|
|
|
|
A count of zero means no user-specific destinations are configured and
|
|
the caller should fall back to the global :func:`send_to_all_destinations`.
|
|
"""
|
|
with SessionLocal() as db:
|
|
return (
|
|
db.query(UserIntegration)
|
|
.filter(
|
|
UserIntegration.owner_id == owner_id,
|
|
UserIntegration.direction == IntegrationDirection.DESTINATION,
|
|
UserIntegration.is_active.is_(True),
|
|
)
|
|
.count()
|
|
)
|