feat: implement Google Drive integration and enhance documentation for new features
This commit is contained in:
Christian Krakau-Louis
+89
-12
@@ -1,21 +1,98 @@
|
||||
# app/tasks/send_to_all.py
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from app.celery_app import celery
|
||||
import os
|
||||
import logging
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||
from app.tasks.upload_to_paperless import upload_to_paperless
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive
|
||||
from app.tasks.upload_to_webdav import upload_to_webdav
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp
|
||||
from app.tasks.upload_to_sftp import upload_to_sftp
|
||||
from app.tasks.upload_to_email import upload_to_email
|
||||
from app.tasks.upload_to_onedrive import upload_to_onedrive
|
||||
from app.tasks.upload_to_s3 import upload_to_s3
|
||||
from app.celery_app import celery
|
||||
|
||||
@celery.task
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
def send_to_all_destinations(file_path: str):
|
||||
"""
|
||||
Fires off tasks to upload a single file to Dropbox, Nextcloud, and Paperless.
|
||||
These tasks run in parallel (Celery returns immediately from each .delay()).
|
||||
"""
|
||||
upload_to_dropbox.delay(file_path)
|
||||
upload_to_nextcloud.delay(file_path)
|
||||
upload_to_paperless.delay(file_path)
|
||||
"""Distribute a file to all configured storage destinations."""
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
logger.info(f"Sending {file_path} to all configured destinations")
|
||||
results = {}
|
||||
|
||||
# Send to Dropbox if configured
|
||||
if settings.dropbox_app_key and settings.dropbox_app_secret and settings.dropbox_refresh_token:
|
||||
logger.info(f"Queueing {file_path} for Dropbox upload")
|
||||
task = upload_to_dropbox.delay(file_path)
|
||||
results["dropbox_task_id"] = task.id
|
||||
|
||||
# Send to Nextcloud if configured
|
||||
if settings.nextcloud_upload_url and settings.nextcloud_username and settings.nextcloud_password:
|
||||
logger.info(f"Queueing {file_path} for Nextcloud upload")
|
||||
task = upload_to_nextcloud.delay(file_path)
|
||||
results["nextcloud_task_id"] = task.id
|
||||
|
||||
# Send to Paperless if configured
|
||||
if settings.paperless_ngx_api_token and settings.paperless_host:
|
||||
logger.info(f"Queueing {file_path} for Paperless upload")
|
||||
task = upload_to_paperless.delay(file_path)
|
||||
results["paperless_task_id"] = task.id
|
||||
|
||||
# Send to Google Drive if configured
|
||||
if settings.google_drive_credentials_json:
|
||||
logger.info(f"Queueing {file_path} for Google Drive upload")
|
||||
task = upload_to_google_drive.delay(file_path)
|
||||
results["google_drive_task_id"] = task.id
|
||||
|
||||
# Send to WebDAV if configured
|
||||
if settings.webdav_url and settings.webdav_username and settings.webdav_password:
|
||||
logger.info(f"Queueing {file_path} for WebDAV upload")
|
||||
task = upload_to_webdav.delay(file_path)
|
||||
results["webdav_task_id"] = task.id
|
||||
|
||||
# Send to FTP if configured
|
||||
if settings.ftp_host and settings.ftp_username and settings.ftp_password:
|
||||
logger.info(f"Queueing {file_path} for FTP upload")
|
||||
task = upload_to_ftp.delay(file_path)
|
||||
results["ftp_task_id"] = task.id
|
||||
|
||||
# Send to SFTP if configured
|
||||
if settings.sftp_host and settings.sftp_username and (settings.sftp_password or settings.sftp_private_key):
|
||||
logger.info(f"Queueing {file_path} for SFTP upload")
|
||||
task = upload_to_sftp.delay(file_path)
|
||||
results["sftp_task_id"] = task.id
|
||||
|
||||
# Send via email if configured
|
||||
if settings.email_host and settings.email_username and settings.email_password and settings.email_default_recipient:
|
||||
logger.info(f"Queueing {file_path} for email delivery")
|
||||
task = upload_to_email.delay(file_path)
|
||||
results["email_task_id"] = task.id
|
||||
|
||||
# Send to OneDrive if configured
|
||||
if settings.onedrive_client_id and settings.onedrive_client_secret and (
|
||||
settings.onedrive_refresh_token or
|
||||
(settings.onedrive_tenant_id and settings.onedrive_tenant_id != "common")
|
||||
):
|
||||
logger.info(f"Queueing {file_path} for OneDrive upload")
|
||||
task = upload_to_onedrive.delay(file_path)
|
||||
results["onedrive_task_id"] = task.id
|
||||
|
||||
# Send to Amazon S3 if configured
|
||||
if settings.s3_bucket_name and settings.aws_access_key_id and settings.aws_secret_access_key:
|
||||
logger.info(f"Queueing {file_path} for S3 upload")
|
||||
task = upload_to_s3.delay(file_path)
|
||||
results["s3_task_id"] = task.id
|
||||
|
||||
return {
|
||||
"status": "All upload tasks enqueued",
|
||||
"file_path": file_path
|
||||
"status": "Queued",
|
||||
"file_path": file_path,
|
||||
"tasks": results
|
||||
}
|
||||
|
||||
+111
-33
@@ -1,20 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import logging
|
||||
import requests
|
||||
import dropbox
|
||||
from dropbox.exceptions import ApiError, AuthError
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.celery_app import celery
|
||||
from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def get_dropbox_access_token():
|
||||
"""Refresh the Dropbox access token using the stored refresh token from ENV."""
|
||||
|
||||
# Check if needed settings are available
|
||||
if not hasattr(settings, 'dropbox_refresh_token') or not settings.dropbox_refresh_token:
|
||||
logger.error("Cannot refresh Dropbox token: Missing refresh token")
|
||||
return None
|
||||
|
||||
if not hasattr(settings, 'dropbox_app_key') or not settings.dropbox_app_key:
|
||||
logger.error("Cannot refresh Dropbox token: Missing app key")
|
||||
return None
|
||||
|
||||
if not hasattr(settings, 'dropbox_app_secret') or not settings.dropbox_app_secret:
|
||||
logger.error("Cannot refresh Dropbox token: Missing app secret")
|
||||
return None
|
||||
|
||||
token_url = "https://api.dropbox.com/oauth2/token"
|
||||
headers = {"Content-Type": "application/x-www-form-urlencoded"}
|
||||
data = {
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": settings.dropbox_refresh_token, # Now using ENV
|
||||
"refresh_token": settings.dropbox_refresh_token,
|
||||
"client_id": settings.dropbox_app_key,
|
||||
"client_secret": settings.dropbox_app_secret,
|
||||
}
|
||||
@@ -25,50 +43,110 @@ def get_dropbox_access_token():
|
||||
return response.json()["access_token"]
|
||||
else:
|
||||
error_msg = f"Failed to refresh Dropbox token: {response.status_code} - {response.text}"
|
||||
print(f"[ERROR] {error_msg}")
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
def upload_to_dropbox(file_path: str):
|
||||
"""Uploads a file to Dropbox using the API."""
|
||||
|
||||
"""
|
||||
Upload a file to Dropbox.
|
||||
"""
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
# Extract filename and set target path
|
||||
error_msg = f"File not found: {file_path}"
|
||||
logger.error(error_msg)
|
||||
raise FileNotFoundError(error_msg)
|
||||
|
||||
# Check if Dropbox is properly configured
|
||||
if not (hasattr(settings, 'dropbox_app_key') and settings.dropbox_app_key and
|
||||
hasattr(settings, 'dropbox_app_secret') and settings.dropbox_app_secret and
|
||||
hasattr(settings, 'dropbox_refresh_token') and settings.dropbox_refresh_token):
|
||||
logger.info("Dropbox upload skipped: Missing configuration")
|
||||
return {"status": "Skipped", "reason": "Dropbox settings not configured"}
|
||||
|
||||
filename = os.path.basename(file_path)
|
||||
dropbox_path = f"{settings.dropbox_folder}/{filename}"
|
||||
|
||||
sanitized_filename = sanitize_filename(filename)
|
||||
|
||||
try:
|
||||
# Get fresh access token
|
||||
# Get access token from refresh token
|
||||
access_token = get_dropbox_access_token()
|
||||
if not access_token:
|
||||
return {"status": "Failed", "reason": "Could not obtain access token"}
|
||||
|
||||
dbx = dropbox.Dropbox(access_token)
|
||||
|
||||
file_size = os.path.getsize(file_path)
|
||||
chunk_size = 4 * 1024 * 1024 # 4MB chunk size
|
||||
|
||||
with open(file_path, "rb") as file_data:
|
||||
if file_size <= chunk_size:
|
||||
dbx.files_upload(file_data.read(), dropbox_path)
|
||||
else:
|
||||
upload_session_start_result = dbx.files_upload_session_start(file_data.read(chunk_size))
|
||||
cursor = dropbox.files.UploadSessionCursor(
|
||||
session_id=upload_session_start_result.session_id,
|
||||
offset=file_data.tell(),
|
||||
)
|
||||
commit = dropbox.files.CommitInfo(path=dropbox_path)
|
||||
|
||||
|
||||
# Calculate remote path based on local file structure
|
||||
remote_base = settings.dropbox_folder or ""
|
||||
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
|
||||
|
||||
# Function to check if file exists in Dropbox
|
||||
def check_exists_in_dropbox(path):
|
||||
try:
|
||||
dbx.files_get_metadata(path)
|
||||
return True
|
||||
except ApiError as e:
|
||||
if e.error.is_path() and e.error.get_path().is_not_found():
|
||||
return False
|
||||
raise
|
||||
|
||||
# Get a unique path in case of collision
|
||||
remote_full_path = f"/{remote_path}" # Dropbox paths should start with /
|
||||
remote_full_path = remote_full_path.replace('//', '/') # Clean double slashes
|
||||
|
||||
# Check for potential file collision and get a unique name if needed
|
||||
dropbox_path = get_unique_filename(remote_full_path, check_exists_in_dropbox)
|
||||
|
||||
# Upload the file
|
||||
logger.info(f"Uploading {filename} to Dropbox at {dropbox_path}")
|
||||
with open(file_path, 'rb') as file_data:
|
||||
# Use files_upload_session for large files to avoid timeouts
|
||||
file_size = os.path.getsize(file_path)
|
||||
if file_size > 10 * 1024 * 1024: # 10 MB threshold for chunked upload
|
||||
cursor = None
|
||||
chunk_size = 4 * 1024 * 1024 # 4 MB chunks
|
||||
file_data.seek(0)
|
||||
|
||||
# Start upload session
|
||||
session_start = dbx.files_upload_session_start(file_data.read(chunk_size))
|
||||
cursor = dropbox.files.UploadSessionCursor(session_start.session_id, file_data.tell())
|
||||
|
||||
# Upload chunks until we reach the end
|
||||
while file_data.tell() < file_size:
|
||||
if (file_size - file_data.tell()) <= chunk_size:
|
||||
dbx.files_upload_session_finish(file_data.read(chunk_size), cursor, commit)
|
||||
# Last chunk
|
||||
dbx.files_upload_session_finish(
|
||||
file_data.read(chunk_size),
|
||||
cursor,
|
||||
dropbox.files.CommitInfo(path=dropbox_path, mode=dropbox.files.WriteMode.overwrite)
|
||||
)
|
||||
else:
|
||||
# More chunks to upload
|
||||
dbx.files_upload_session_append_v2(file_data.read(chunk_size), cursor)
|
||||
cursor.offset = file_data.tell()
|
||||
|
||||
print(f"[INFO] Successfully uploaded {filename} to Dropbox at {dropbox_path}.")
|
||||
return {"status": "Completed", "file": file_path}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"[ERROR] Failed to upload {filename} to Dropbox: {str(e)}"
|
||||
print(error_msg)
|
||||
else:
|
||||
# Small file, direct upload
|
||||
file_data.seek(0)
|
||||
dbx.files_upload(
|
||||
file_data.read(),
|
||||
dropbox_path,
|
||||
mode=dropbox.files.WriteMode.overwrite
|
||||
)
|
||||
|
||||
logger.info(f"Successfully uploaded {filename} to Dropbox at {dropbox_path}")
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file_path": file_path,
|
||||
"dropbox_path": dropbox_path
|
||||
}
|
||||
|
||||
except AuthError:
|
||||
error_msg = f"[ERROR] Authentication failed while uploading {filename} to Dropbox. Check token."
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
except ApiError as e:
|
||||
error_msg = f"[ERROR] Failed to upload {filename} to Dropbox: {e}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
except Exception as e:
|
||||
error_msg = f"[ERROR] Unexpected error uploading {filename} to Dropbox: {e}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import json
|
||||
import smtplib
|
||||
import socket
|
||||
import logging
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.application import MIMEApplication
|
||||
from email.mime.image import MIMEImage
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.celery_app import celery
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def get_email_template(template_name="default.html"):
|
||||
"""
|
||||
Load email template from one of these locations in order of precedence:
|
||||
1. Custom template from workdir/templates/email/
|
||||
2. Default template from app/templates/email/
|
||||
"""
|
||||
# First try to load from workdir (user customizable location)
|
||||
try:
|
||||
workdir_template_path = os.path.join(settings.workdir, "templates", "email")
|
||||
if os.path.exists(workdir_template_path):
|
||||
env = Environment(
|
||||
loader=FileSystemLoader(workdir_template_path),
|
||||
autoescape=select_autoescape(['html', 'xml'])
|
||||
)
|
||||
env.globals['now'] = datetime.now # Add the now function to the Jinja environment
|
||||
template = env.get_template(template_name)
|
||||
logger.info(f"Using custom email template from workdir: {template_name}")
|
||||
return template
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load custom email template: {str(e)}")
|
||||
|
||||
# Fallback to built-in template
|
||||
try:
|
||||
# Get the app directory path (where this file is)
|
||||
current_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
app_template_path = os.path.join(current_dir, "templates", "email")
|
||||
env = Environment(
|
||||
loader=FileSystemLoader(app_template_path),
|
||||
autoescape=select_autoescape(['html', 'xml'])
|
||||
)
|
||||
env.globals['now'] = datetime.now # Add the now function to the Jinja environment
|
||||
template = env.get_template(template_name)
|
||||
logger.info(f"Using built-in email template: {template_name}")
|
||||
return template
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load built-in email template: {str(e)}")
|
||||
raise ValueError(f"Could not find any valid email template: {str(e)}")
|
||||
|
||||
def extract_metadata_from_file(file_path):
|
||||
"""
|
||||
Try to extract metadata from a file using several methods:
|
||||
1. Check for a .json metadata file with the same name
|
||||
2. Extract metadata from PDF if it's embedded
|
||||
|
||||
Returns a dictionary of metadata or None if not found
|
||||
"""
|
||||
metadata = {}
|
||||
|
||||
# Check for separate metadata JSON file
|
||||
metadata_path = os.path.splitext(file_path)[0] + '.json'
|
||||
if os.path.exists(metadata_path):
|
||||
try:
|
||||
with open(metadata_path, 'r', encoding='utf-8') as f:
|
||||
metadata = json.load(f)
|
||||
logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
|
||||
return metadata
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load metadata from JSON file: {str(e)}")
|
||||
|
||||
# TODO: For PDF files, try to extract embedded metadata using PyPDF2
|
||||
# This would require additional dependencies, so for now we'll just check for external JSON
|
||||
|
||||
return metadata
|
||||
|
||||
def attach_logo(msg):
|
||||
"""Attach the DocuNova logo to the email with proper Content-ID."""
|
||||
try:
|
||||
# Try to find logo in workdir first (for customization)
|
||||
custom_logo_path = os.path.join(settings.workdir, "templates", "email", "logo.png")
|
||||
if os.path.exists(custom_logo_path):
|
||||
logo_path = custom_logo_path
|
||||
else:
|
||||
# Use built-in logo
|
||||
app_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
logo_path = os.path.join(app_dir, "static", "logo.png")
|
||||
# Fallback to logo in frontend/static if app/static doesn't exist
|
||||
if not os.path.exists(logo_path):
|
||||
logo_path = os.path.join(app_dir, "..", "frontend", "static", "logo.png")
|
||||
|
||||
if os.path.exists(logo_path):
|
||||
with open(logo_path, 'rb') as img:
|
||||
logo_data = img.read()
|
||||
|
||||
# Determine image MIME type based on extension
|
||||
mimetype = 'image/svg+xml' if logo_path.endswith('.svg') else 'image/png'
|
||||
logo_attach = MIMEImage(logo_data, mimetype)
|
||||
logo_attach.add_header('Content-ID', '<logo>')
|
||||
logo_attach.add_header('Content-Disposition', 'inline', filename='logo.png')
|
||||
msg.attach(logo_attach)
|
||||
logger.info(f"Logo attached from {logo_path}")
|
||||
return True
|
||||
else:
|
||||
logger.warning("Could not find logo file")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Error attaching logo: {str(e)}")
|
||||
return False
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
def upload_to_email(file_path: str, recipients=None, subject=None, message=None, template_name="default.html", include_metadata=True):
|
||||
"""
|
||||
Sends a file via email to the specified recipients.
|
||||
If recipients is None, uses the configured default email recipient.
|
||||
"""
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
# Extract filename
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
# Check if email settings are configured
|
||||
if not settings.email_host:
|
||||
error_msg = "Email host is not configured"
|
||||
logger.error(error_msg)
|
||||
return {"status": "Skipped", "reason": error_msg}
|
||||
|
||||
# Log email configuration for debugging
|
||||
logger.debug(f"Email config - Host: {settings.email_host}, Port: {settings.email_port}, "
|
||||
f"Username: {settings.email_username}, TLS: {settings.email_use_tls}")
|
||||
|
||||
# Use provided recipients or fall back to default
|
||||
if not recipients:
|
||||
if not settings.email_default_recipient:
|
||||
error_msg = "No recipients specified and no default recipient configured"
|
||||
logger.error(error_msg)
|
||||
return {"status": "Skipped", "reason": error_msg}
|
||||
recipients = [settings.email_default_recipient]
|
||||
elif isinstance(recipients, str):
|
||||
recipients = [recipients] # Convert single email to list
|
||||
|
||||
# Use provided subject or create default
|
||||
if not subject:
|
||||
subject = f"DocuNova Document: {filename}"
|
||||
|
||||
# Extract document metadata if available
|
||||
metadata = {}
|
||||
if include_metadata:
|
||||
metadata = extract_metadata_from_file(file_path)
|
||||
|
||||
try:
|
||||
# Create the email
|
||||
msg = MIMEMultipart('related') # Changed to 'related' to properly handle inline images
|
||||
msg['From'] = settings.email_sender or settings.email_username
|
||||
msg['To'] = ", ".join(recipients)
|
||||
msg['Subject'] = subject
|
||||
|
||||
# Create alternative part for HTML content
|
||||
alt_part = MIMEMultipart('alternative')
|
||||
msg.attach(alt_part)
|
||||
|
||||
# Attach logo to the email
|
||||
has_logo = attach_logo(msg)
|
||||
|
||||
# Load and render template
|
||||
template = get_email_template(template_name)
|
||||
|
||||
# Context data for the template
|
||||
context = {
|
||||
"filename": filename,
|
||||
"message": message or f"Attached is the document: {filename}",
|
||||
"app_name": "DocuNova",
|
||||
"app_url": f"https://{settings.external_hostname}" if settings.external_hostname else None,
|
||||
"custom_message": message,
|
||||
"metadata": metadata,
|
||||
"has_metadata": bool(metadata),
|
||||
"has_logo": has_logo,
|
||||
"current_year": datetime.now().year
|
||||
}
|
||||
|
||||
# Render HTML body
|
||||
html_content = template.render(**context)
|
||||
alt_part.attach(MIMEText(html_content, 'html'))
|
||||
|
||||
# Attach the file
|
||||
with open(file_path, "rb") as file:
|
||||
attachment = MIMEApplication(file.read(), _subtype="pdf")
|
||||
attachment.add_header('Content-Disposition', f'attachment; filename="{filename}"')
|
||||
msg.attach(attachment)
|
||||
|
||||
try:
|
||||
# First try to resolve the hostname
|
||||
socket.gethostbyname(settings.email_host)
|
||||
|
||||
# Connect to the SMTP server
|
||||
with smtplib.SMTP(settings.email_host, settings.email_port, timeout=30) as server:
|
||||
# Use TLS if specified
|
||||
if settings.email_use_tls:
|
||||
server.starttls()
|
||||
|
||||
# Login if credentials are provided
|
||||
if settings.email_username and settings.email_password:
|
||||
server.login(settings.email_username, settings.email_password)
|
||||
|
||||
# Send the email
|
||||
server.send_message(msg)
|
||||
|
||||
logger.info(f"Successfully sent {filename} via email to {', '.join(recipients)}")
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file": file_path,
|
||||
"recipients": recipients,
|
||||
"subject": subject,
|
||||
"metadata_included": bool(metadata),
|
||||
"logo_included": has_logo
|
||||
}
|
||||
except socket.gaierror as e:
|
||||
error_msg = f"Failed to resolve email host: {settings.email_host} - {str(e)}"
|
||||
logger.error(error_msg)
|
||||
return {"status": "Failed", "reason": error_msg, "error": str(e)}
|
||||
except ConnectionRefusedError as e:
|
||||
error_msg = f"Connection refused to SMTP server {settings.email_host}:{settings.email_port} - {str(e)}"
|
||||
logger.error(error_msg)
|
||||
return {"status": "Failed", "reason": error_msg, "error": str(e)}
|
||||
except TimeoutError as e:
|
||||
error_msg = f"Connection timeout to SMTP server {settings.email_host}:{settings.email_port} - {str(e)}"
|
||||
logger.error(error_msg)
|
||||
return {"status": "Failed", "reason": error_msg, "error": str(e)}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to send {filename} via email: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import ftplib
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.celery_app import celery
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
def upload_to_ftp(file_path: str):
|
||||
"""Uploads a file to an FTP server in the configured folder."""
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
# Extract filename
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
# Check if FTP settings are configured
|
||||
if not settings.ftp_host:
|
||||
error_msg = "FTP host is not configured"
|
||||
logger.error(error_msg)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
try:
|
||||
# Connect to FTP server
|
||||
ftp = ftplib.FTP()
|
||||
ftp.connect(
|
||||
host=settings.ftp_host,
|
||||
port=settings.ftp_port or 21
|
||||
)
|
||||
|
||||
# Login with credentials
|
||||
ftp.login(
|
||||
user=settings.ftp_username,
|
||||
passwd=settings.ftp_password
|
||||
)
|
||||
|
||||
# Change to target directory if specified
|
||||
if settings.ftp_folder:
|
||||
try:
|
||||
# Try to navigate to the directory, create if it doesn't exist
|
||||
ftp_folder = settings.ftp_folder
|
||||
# Remove leading slash if present
|
||||
if ftp_folder.startswith('/'):
|
||||
ftp_folder = ftp_folder[1:]
|
||||
|
||||
# Try to change to the directory
|
||||
try:
|
||||
ftp.cwd(ftp_folder)
|
||||
except ftplib.error_perm:
|
||||
# Create directory structure if it doesn't exist
|
||||
folders = ftp_folder.split('/')
|
||||
current_dir = ''
|
||||
for folder in folders:
|
||||
if folder:
|
||||
current_dir += f"/{folder}"
|
||||
try:
|
||||
ftp.cwd(current_dir)
|
||||
except ftplib.error_perm:
|
||||
ftp.mkd(current_dir)
|
||||
ftp.cwd(current_dir)
|
||||
except ftplib.Error as e:
|
||||
error_msg = f"Failed to change/create directory on FTP server: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
# Upload the file
|
||||
with open(file_path, 'rb') as file_data:
|
||||
ftp.storbinary(f'STOR {filename}', file_data)
|
||||
|
||||
# Close FTP connection
|
||||
ftp.quit()
|
||||
|
||||
logger.info(f"Successfully uploaded {filename} to FTP server at {settings.ftp_host}")
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file": file_path,
|
||||
"ftp_host": settings.ftp_host,
|
||||
"ftp_path": f"{settings.ftp_folder}/{filename}" if settings.ftp_folder else filename
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to upload {filename} to FTP server: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
@@ -0,0 +1,103 @@
|
||||
"""
|
||||
app/tasks/upload_to_google_drive.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
from googleapiclient.discovery import build
|
||||
from googleapiclient.http import MediaFileUpload
|
||||
from google.oauth2.service_account import Credentials
|
||||
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.celery_app import celery
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def get_google_drive_service():
|
||||
"""
|
||||
Authenticate with Google Drive API using service account credentials
|
||||
and return an authorized service object.
|
||||
"""
|
||||
try:
|
||||
# Load service account credentials from settings
|
||||
if not settings.google_drive_credentials_json:
|
||||
logger.error("Google Drive credentials not configured")
|
||||
return None
|
||||
|
||||
credentials_dict = json.loads(settings.google_drive_credentials_json)
|
||||
credentials = Credentials.from_service_account_info(
|
||||
credentials_dict,
|
||||
scopes=['https://www.googleapis.com/auth/drive']
|
||||
)
|
||||
|
||||
# Delegate to user if specified
|
||||
if settings.google_drive_delegate_to:
|
||||
credentials = credentials.with_subject(settings.google_drive_delegate_to)
|
||||
|
||||
# Build and return the service
|
||||
service = build('drive', 'v3', credentials=credentials)
|
||||
return service
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to authenticate with Google Drive: {str(e)}")
|
||||
return None
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
def upload_to_google_drive(file_path: str):
|
||||
"""Uploads a file to Google Drive in the configured folder."""
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
# Extract filename from path
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
try:
|
||||
# Get Google Drive service
|
||||
service = get_google_drive_service()
|
||||
if not service:
|
||||
raise Exception("Failed to initialize Google Drive service")
|
||||
|
||||
# Prepare the file metadata
|
||||
file_metadata = {
|
||||
'name': filename,
|
||||
}
|
||||
|
||||
# If folder ID is specified, set parent folder
|
||||
if settings.google_drive_folder_id:
|
||||
file_metadata['parents'] = [settings.google_drive_folder_id]
|
||||
|
||||
# Upload file with metadata
|
||||
media = MediaFileUpload(
|
||||
file_path,
|
||||
mimetype='application/pdf',
|
||||
resumable=True
|
||||
)
|
||||
|
||||
file = service.files().create(
|
||||
body=file_metadata,
|
||||
media_body=media,
|
||||
fields='id,name,webViewLink'
|
||||
).execute()
|
||||
|
||||
# Log success details
|
||||
file_id = file.get('id')
|
||||
web_view_link = file.get('webViewLink')
|
||||
|
||||
logger.info(f"Successfully uploaded {filename} to Google Drive with ID: {file_id}")
|
||||
logger.info(f"File accessible at: {web_view_link}")
|
||||
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file_path": file_path,
|
||||
"google_drive_file_id": file_id,
|
||||
"google_drive_web_link": web_view_link
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to upload {filename} to Google Drive: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
@@ -1,37 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import logging
|
||||
import requests
|
||||
from requests.auth import HTTPBasicAuth
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.celery_app import celery
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
def upload_to_nextcloud(file_path: str):
|
||||
"""Uploads a file to Nextcloud in the configured folder."""
|
||||
|
||||
"""
|
||||
Upload a file to Nextcloud WebDAV.
|
||||
"""
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
# Extract filename
|
||||
error_msg = f"File not found: {file_path}"
|
||||
logger.error(error_msg)
|
||||
raise FileNotFoundError(error_msg)
|
||||
|
||||
# For Nextcloud, we need to check for 'nextcloud_upload_url' instead of 'nextcloud_url'
|
||||
# This is what's shown in your env view
|
||||
if not (getattr(settings, 'nextcloud_upload_url', None) and
|
||||
getattr(settings, 'nextcloud_username', None) and
|
||||
getattr(settings, 'nextcloud_password', None)):
|
||||
logger.info("Nextcloud upload skipped: Missing configuration")
|
||||
return {"status": "Skipped", "reason": "Nextcloud settings not configured"}
|
||||
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
# Construct the full upload URL
|
||||
nextcloud_url = f"{settings.nextcloud_upload_url}/{settings.nextcloud_folder}/{filename}"
|
||||
|
||||
# Read file content
|
||||
with open(file_path, "rb") as file_data:
|
||||
response = requests.put(
|
||||
nextcloud_url,
|
||||
auth=(settings.nextcloud_username, settings.nextcloud_password),
|
||||
data=file_data
|
||||
)
|
||||
|
||||
# Check if upload was successful
|
||||
if response.status_code in (200, 201):
|
||||
print(f"[INFO] Successfully uploaded {filename} to Nextcloud at {nextcloud_url}.")
|
||||
return {"status": "Completed", "file": file_path}
|
||||
else:
|
||||
error_msg = f"[ERROR] Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
|
||||
print(error_msg)
|
||||
sanitized_filename = sanitize_filename(filename)
|
||||
|
||||
try:
|
||||
# Prepare WebDAV URL - use nextcloud_upload_url instead of nextcloud_url
|
||||
webdav_url = settings.nextcloud_upload_url
|
||||
if not webdav_url.endswith('/'):
|
||||
webdav_url += '/'
|
||||
|
||||
# Calculate remote path based on local file structure
|
||||
remote_base = getattr(settings, 'nextcloud_folder', '') or ""
|
||||
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
|
||||
full_url = f"{webdav_url}/{remote_path}"
|
||||
|
||||
# Remove any double slashes (except in http://)
|
||||
full_url = full_url.replace('://', '$PLACEHOLDER$')
|
||||
while '//' in full_url:
|
||||
full_url = full_url.replace('//', '/')
|
||||
full_url = full_url.replace('$PLACEHOLDER$', '://')
|
||||
|
||||
# Function to check if file exists in Nextcloud
|
||||
def check_exists_in_nextcloud(path):
|
||||
check_url = f"{webdav_url}{os.path.dirname(path)}"
|
||||
try:
|
||||
response = requests.request(
|
||||
'PROPFIND',
|
||||
check_url,
|
||||
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
|
||||
headers={'Depth': '1'},
|
||||
timeout=10
|
||||
)
|
||||
|
||||
return path in response.text
|
||||
except Exception:
|
||||
# If we can't check, assume it doesn't exist
|
||||
return False
|
||||
|
||||
# Check for potential file collision and get a unique name if needed
|
||||
remote_path = get_unique_filename(remote_path, check_exists_in_nextcloud)
|
||||
full_url = f"{webdav_url}/{remote_path}"
|
||||
|
||||
# Fix double slashes again
|
||||
full_url = full_url.replace('://', '$PLACEHOLDER$')
|
||||
while '//' in full_url:
|
||||
full_url = full_url.replace('//', '/')
|
||||
full_url = full_url.replace('$PLACEHOLDER$', '://')
|
||||
|
||||
# Create necessary parent folders
|
||||
parent_dirs = os.path.dirname(remote_path)
|
||||
if parent_dirs:
|
||||
current_path = ""
|
||||
for folder in parent_dirs.split('/'):
|
||||
if not folder:
|
||||
continue
|
||||
current_path += f"{folder}/"
|
||||
mkdir_url = f"{webdav_url}/{current_path}"
|
||||
# Fix double slashes
|
||||
mkdir_url = mkdir_url.replace('://', '$PLACEHOLDER$')
|
||||
while '//' in mkdir_url:
|
||||
mkdir_url = mkdir_url.replace('//', '/')
|
||||
mkdir_url = mkdir_url.replace('$PLACEHOLDER$', '://')
|
||||
|
||||
requests.request(
|
||||
'MKCOL',
|
||||
mkdir_url,
|
||||
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
|
||||
timeout=10
|
||||
)
|
||||
|
||||
# Upload the file
|
||||
logger.info(f"Uploading {filename} to Nextcloud at {full_url}")
|
||||
with open(file_path, 'rb') as file_data:
|
||||
response = requests.put(
|
||||
full_url,
|
||||
data=file_data,
|
||||
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
|
||||
headers={'Content-Type': 'application/octet-stream'},
|
||||
timeout=60 # Longer timeout for larger files
|
||||
)
|
||||
|
||||
if response.status_code in (201, 204): # Created or No Content
|
||||
logger.info(f"Successfully uploaded {filename} to Nextcloud at {remote_path}")
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file_path": file_path,
|
||||
"nextcloud_path": remote_path,
|
||||
"response_code": response.status_code
|
||||
}
|
||||
else:
|
||||
error_msg = f"[ERROR] Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"[ERROR] Failed to upload {filename} to Nextcloud: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
import requests
|
||||
import msal
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.celery_app import celery
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def get_onedrive_token():
|
||||
"""
|
||||
Get an access token for Microsoft Graph API using the appropriate flow.
|
||||
For personal accounts, uses refresh token flow.
|
||||
For organizational accounts, uses client credentials flow if refresh token isn't provided.
|
||||
"""
|
||||
# Check for required settings
|
||||
if not settings.onedrive_client_id or not settings.onedrive_client_secret:
|
||||
raise ValueError("OneDrive client ID and client secret must be configured")
|
||||
|
||||
# Use refresh token flow (works for both personal and org accounts)
|
||||
if settings.onedrive_refresh_token:
|
||||
# Use MSAL to get token from refresh token
|
||||
app = msal.PublicClientApplication(settings.onedrive_client_id)
|
||||
|
||||
# Request new token using refresh token
|
||||
token_response = app.acquire_token_by_refresh_token(
|
||||
refresh_token=settings.onedrive_refresh_token,
|
||||
scopes=["https://graph.microsoft.com/Files.ReadWrite"]
|
||||
)
|
||||
|
||||
if "access_token" not in token_response:
|
||||
error = token_response.get("error", "")
|
||||
error_desc = token_response.get("error_description", "Unknown error")
|
||||
raise ValueError(f"Failed to get access token: {error} - {error_desc}")
|
||||
|
||||
return token_response["access_token"]
|
||||
|
||||
# No refresh token - try client credentials (only works for org accounts)
|
||||
elif settings.onedrive_tenant_id and settings.onedrive_tenant_id != "common":
|
||||
authority = f"https://login.microsoftonline.com/{settings.onedrive_tenant_id}"
|
||||
app = msal.ConfidentialClientApplication(
|
||||
client_id=settings.onedrive_client_id,
|
||||
client_credential=settings.onedrive_client_secret,
|
||||
authority=authority
|
||||
)
|
||||
|
||||
# Acquire token for application
|
||||
token_response = app.acquire_token_for_client(
|
||||
scopes=["https://graph.microsoft.com/.default"]
|
||||
)
|
||||
|
||||
if "access_token" not in token_response:
|
||||
error = token_response.get("error", "")
|
||||
error_desc = token_response.get("error_description", "Unknown error")
|
||||
raise ValueError(f"Failed to get access token: {error} - {error_desc}")
|
||||
|
||||
return token_response["access_token"]
|
||||
|
||||
else:
|
||||
raise ValueError("For personal Microsoft accounts, ONEDRIVE_REFRESH_TOKEN must be configured")
|
||||
|
||||
def create_upload_session(filename, folder_path, access_token):
|
||||
"""Creates an upload session for large files in Microsoft Graph API."""
|
||||
# Construct the API endpoint
|
||||
base_url = "https://graph.microsoft.com/v1.0/me/drive"
|
||||
|
||||
# Format the folder path correctly
|
||||
if folder_path:
|
||||
# Remove leading/trailing slashes
|
||||
folder_path = folder_path.strip('/')
|
||||
# Replace spaces with %20
|
||||
folder_path = folder_path.replace(' ', '%20')
|
||||
item_path = f"/root:/{folder_path}/{filename}:/createUploadSession"
|
||||
else:
|
||||
item_path = f"/root:/{filename}:/createUploadSession"
|
||||
|
||||
url = f"{base_url}{item_path}"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {access_token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
response = requests.post(url, headers=headers)
|
||||
|
||||
if response.status_code == 200:
|
||||
return response.json().get("uploadUrl")
|
||||
else:
|
||||
error_msg = f"Failed to create upload session: {response.status_code} - {response.text}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
def upload_large_file(file_path, upload_url):
|
||||
"""
|
||||
Upload a large file to OneDrive using the upload session URL.
|
||||
Uses chunked upload for reliability.
|
||||
"""
|
||||
# Get file size
|
||||
file_size = os.path.getsize(file_path)
|
||||
|
||||
# Define chunk size (10 MB)
|
||||
chunk_size = 10 * 1024 * 1024
|
||||
|
||||
# Open and read file in chunks
|
||||
with open(file_path, 'rb') as f:
|
||||
# Process file in chunks
|
||||
chunk_number = 0
|
||||
while True:
|
||||
chunk = f.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
|
||||
# Get the position in the file
|
||||
chunk_start = chunk_number * chunk_size
|
||||
chunk_end = chunk_start + len(chunk) - 1
|
||||
|
||||
# Prepare content range header
|
||||
content_range = f"bytes {chunk_start}-{chunk_end}/{file_size}"
|
||||
|
||||
# Upload chunk
|
||||
headers = {
|
||||
"Content-Length": str(len(chunk)),
|
||||
"Content-Range": content_range
|
||||
}
|
||||
|
||||
# Try to upload chunk with retries
|
||||
max_retries = 3
|
||||
retry_delay = 2 # seconds
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = requests.put(
|
||||
upload_url,
|
||||
headers=headers,
|
||||
data=chunk
|
||||
)
|
||||
|
||||
# Check if successful
|
||||
if response.status_code in (201, 202):
|
||||
# 201 = Created (final chunk), 202 = Accepted (more chunks coming)
|
||||
break
|
||||
else:
|
||||
logger.warning(f"Chunk upload failed (attempt {attempt+1}): {response.status_code}")
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(retry_delay * (attempt + 1))
|
||||
except Exception as e:
|
||||
logger.warning(f"Chunk upload error (attempt {attempt+1}): {str(e)}")
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(retry_delay * (attempt + 1))
|
||||
|
||||
if response.status_code not in (201, 202):
|
||||
raise Exception(f"Failed to upload chunk after {max_retries} attempts: {response.status_code} - {response.text}")
|
||||
|
||||
# Move to next chunk
|
||||
chunk_number += 1
|
||||
|
||||
# If we get here, all chunks were uploaded successfully
|
||||
# The last response should contain the file metadata
|
||||
return response.json()
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
def upload_to_onedrive(file_path: str):
|
||||
"""Uploads a file to OneDrive in the configured folder."""
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
# Extract filename
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
# Check if OneDrive settings are configured
|
||||
if not settings.onedrive_client_id:
|
||||
error_msg = "OneDrive client ID is not configured"
|
||||
logger.error(error_msg)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
try:
|
||||
# Get access token
|
||||
access_token = get_onedrive_token()
|
||||
|
||||
# Create upload session
|
||||
upload_url = create_upload_session(filename, settings.onedrive_folder_path, access_token)
|
||||
|
||||
# Upload the file
|
||||
result = upload_large_file(file_path, upload_url)
|
||||
|
||||
# Log success
|
||||
web_url = result.get("webUrl", "Not available")
|
||||
logger.info(f"Successfully uploaded {filename} to OneDrive at path {settings.onedrive_folder_path}")
|
||||
logger.info(f"File accessible at: {web_url}")
|
||||
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file_path": file_path,
|
||||
"onedrive_path": f"{settings.onedrive_folder_path}/{filename}",
|
||||
"web_url": web_url
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to upload {filename} to OneDrive: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
@@ -82,31 +82,31 @@ def poll_task_for_document_id(task_id: str) -> int:
|
||||
)
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
def upload_to_paperless(file_path: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Uploads a PDF to Paperless with minimal metadata (filename and date only).
|
||||
|
||||
1. Extracts the filename and date from the file.
|
||||
2. POSTs the PDF to Paperless => returns a quoted UUID string (task_id).
|
||||
3. Polls /api/tasks/?task_id=<uuid> until SUCCESS or FAILURE => doc_id.
|
||||
def upload_to_paperless(file_path: str):
|
||||
"""Uploads a file to Paperless-ngx."""
|
||||
|
||||
Returns a dict with status, the paperless_task_id, paperless_document_id, and file_path.
|
||||
"""
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
base_name = os.path.basename(file_path)
|
||||
# Extract filename
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
# Check if Paperless settings are configured
|
||||
if not settings.paperless_host or not settings.paperless_ngx_api_token:
|
||||
error_msg = "Paperless-ngx credentials are not fully configured"
|
||||
logger.error(error_msg)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
# Upload the PDF
|
||||
post_url = _paperless_api_url("/api/documents/post_document/")
|
||||
with open(file_path, "rb") as f:
|
||||
files = {
|
||||
"document": (base_name, f, "application/pdf"),
|
||||
"document": (filename, f, "application/pdf"),
|
||||
}
|
||||
data = {"title": base_name} # Title = Filename (no additional metadata)
|
||||
data = {"title": filename} # Title = Filename (no additional metadata)
|
||||
|
||||
try:
|
||||
logger.debug("Posting document to Paperless: file=%s", base_name)
|
||||
logger.debug("Posting document to Paperless: file=%s", filename)
|
||||
resp = requests.post(post_url, headers=_get_headers(), files=files, data=data)
|
||||
resp.raise_for_status()
|
||||
except requests.exceptions.RequestException as exc:
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import logging
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.celery_app import celery
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
def upload_to_s3(file_path: str):
|
||||
"""Uploads a file to Amazon S3 in the configured bucket and folder."""
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
# Extract filename
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
# Check if S3 settings are configured
|
||||
if not settings.s3_bucket_name:
|
||||
error_msg = "S3 bucket name is not configured"
|
||||
logger.error(error_msg)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
if not settings.aws_access_key_id or not settings.aws_secret_access_key:
|
||||
error_msg = "AWS credentials are not configured"
|
||||
logger.error(error_msg)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
try:
|
||||
# Create S3 client
|
||||
s3_client = boto3.client(
|
||||
's3',
|
||||
region_name=settings.aws_region,
|
||||
aws_access_key_id=settings.aws_access_key_id,
|
||||
aws_secret_access_key=settings.aws_secret_access_key
|
||||
)
|
||||
|
||||
# Construct the S3 key (path within the bucket)
|
||||
if settings.s3_folder_prefix:
|
||||
# Ensure folder prefix ends with a slash
|
||||
folder_prefix = settings.s3_folder_prefix
|
||||
if not folder_prefix.endswith('/'):
|
||||
folder_prefix += '/'
|
||||
s3_key = f"{folder_prefix}{filename}"
|
||||
else:
|
||||
s3_key = filename
|
||||
|
||||
# Prepare extra arguments
|
||||
extra_args = {
|
||||
'StorageClass': settings.s3_storage_class
|
||||
}
|
||||
|
||||
# Add ACL if configured
|
||||
if settings.s3_acl:
|
||||
extra_args['ACL'] = settings.s3_acl
|
||||
|
||||
# Upload file
|
||||
s3_client.upload_file(
|
||||
file_path,
|
||||
settings.s3_bucket_name,
|
||||
s3_key,
|
||||
ExtraArgs=extra_args
|
||||
)
|
||||
|
||||
# Generate URL to the file (useful for public files)
|
||||
# For private files, this is just a reference and won't be accessible directly
|
||||
s3_url = f"https://{settings.s3_bucket_name}.s3.{settings.aws_region}.amazonaws.com/{s3_key}"
|
||||
|
||||
logger.info(f"Successfully uploaded {filename} to S3 bucket {settings.s3_bucket_name} at path {s3_key}")
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file": file_path,
|
||||
"s3_bucket": settings.s3_bucket_name,
|
||||
"s3_key": s3_key,
|
||||
"s3_url": s3_url
|
||||
}
|
||||
|
||||
except ClientError as e:
|
||||
error_msg = f"Failed to upload {filename} to S3: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error uploading {filename} to S3: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import logging
|
||||
import paramiko
|
||||
from pathlib import Path
|
||||
from app.config import settings
|
||||
from app.celery_app import celery
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
def upload_to_sftp(file_path: str):
|
||||
"""
|
||||
Upload a file to an SFTP server.
|
||||
"""
|
||||
if not os.path.exists(file_path):
|
||||
error_msg = f"File not found: {file_path}"
|
||||
logger.error(error_msg)
|
||||
raise FileNotFoundError(error_msg)
|
||||
|
||||
if not (settings.sftp_host and settings.sftp_port and settings.sftp_username):
|
||||
logger.info("SFTP upload skipped: Missing configuration")
|
||||
return {"status": "Skipped", "reason": "SFTP settings not configured"}
|
||||
|
||||
filename = os.path.basename(file_path)
|
||||
sanitized_filename = sanitize_filename(filename)
|
||||
|
||||
# SSH client for SFTP connection
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
try:
|
||||
# Setup connection parameters
|
||||
connect_kwargs = {
|
||||
"hostname": settings.sftp_host,
|
||||
"port": settings.sftp_port,
|
||||
"username": settings.sftp_username,
|
||||
}
|
||||
|
||||
# Check for authentication methods - use key if available, otherwise try password
|
||||
sftp_key_path = getattr(settings, 'sftp_private_key', None)
|
||||
sftp_key_passphrase = getattr(settings, 'sftp_private_key_passphrase', None)
|
||||
|
||||
if sftp_key_path and os.path.exists(sftp_key_path):
|
||||
logger.info(f"Using SSH key authentication with key: {sftp_key_path}")
|
||||
connect_kwargs["key_filename"] = sftp_key_path
|
||||
if sftp_key_passphrase:
|
||||
connect_kwargs["passphrase"] = sftp_key_passphrase
|
||||
elif settings.sftp_password:
|
||||
logger.info("Using password authentication for SFTP")
|
||||
connect_kwargs["password"] = settings.sftp_password
|
||||
else:
|
||||
error_msg = "No authentication method available for SFTP (no key or password)"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
# Connect to the server
|
||||
logger.info(f"Connecting to SFTP server at {settings.sftp_host}:{settings.sftp_port}")
|
||||
ssh.connect(**connect_kwargs)
|
||||
|
||||
# Open SFTP session
|
||||
sftp = ssh.open_sftp()
|
||||
|
||||
# Calculate remote path based on local file structure
|
||||
remote_base = settings.sftp_folder or ""
|
||||
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
|
||||
|
||||
# Function to check if file exists in SFTP server
|
||||
def check_exists_in_sftp(path):
|
||||
try:
|
||||
sftp.stat(path)
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
# Check for potential file collision and get a unique name if needed
|
||||
remote_path = get_unique_filename(remote_path, check_exists_in_sftp)
|
||||
|
||||
# Create parent directories if needed
|
||||
remote_dir = os.path.dirname(remote_path)
|
||||
if remote_dir:
|
||||
try:
|
||||
# Try to create the full directory path
|
||||
current_dir = ""
|
||||
for dir_part in remote_dir.split("/"):
|
||||
if not dir_part:
|
||||
continue
|
||||
current_dir += f"/{dir_part}"
|
||||
try:
|
||||
sftp.stat(current_dir)
|
||||
except FileNotFoundError:
|
||||
logger.info(f"Creating directory on SFTP server: {current_dir}")
|
||||
sftp.mkdir(current_dir)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to create directory structure {remote_dir}: {str(e)}")
|
||||
|
||||
# Upload the file
|
||||
logger.info(f"Uploading {filename} to SFTP at {remote_path}")
|
||||
sftp.put(file_path, remote_path)
|
||||
logger.info(f"Successfully uploaded {filename} to SFTP at {remote_path}")
|
||||
|
||||
# Close connections
|
||||
sftp.close()
|
||||
ssh.close()
|
||||
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file_path": file_path,
|
||||
"sftp_path": remote_path
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
# Make sure connections are closed
|
||||
try:
|
||||
if 'sftp' in locals():
|
||||
sftp.close()
|
||||
ssh.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
error_msg = f"Failed to upload {filename} to SFTP server: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import requests
|
||||
from urllib.parse import urljoin
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.celery_app import celery
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
def upload_to_webdav(file_path: str):
|
||||
"""Uploads a file to a WebDAV server in the configured folder."""
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
# Extract filename
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
# Check if WebDAV settings are configured
|
||||
if not settings.webdav_url:
|
||||
error_msg = "WebDAV URL is not configured"
|
||||
logger.error(error_msg)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
# Construct the full upload URL
|
||||
webdav_folder = settings.webdav_folder or ""
|
||||
# Ensure folder doesn't have leading slash if we're joining it to the base URL
|
||||
if webdav_folder and webdav_folder.startswith("/"):
|
||||
webdav_folder = webdav_folder[1:]
|
||||
|
||||
# Join the base URL and folder path
|
||||
target_url = urljoin(settings.webdav_url, webdav_folder)
|
||||
# Ensure URL ends with a slash for proper joining with filename
|
||||
if not target_url.endswith("/"):
|
||||
target_url += "/"
|
||||
|
||||
# Construct final URL with filename
|
||||
webdav_url = urljoin(target_url, filename)
|
||||
|
||||
# Read file content
|
||||
try:
|
||||
with open(file_path, "rb") as file_data:
|
||||
response = requests.put(
|
||||
webdav_url,
|
||||
auth=(settings.webdav_username, settings.webdav_password),
|
||||
data=file_data,
|
||||
verify=settings.webdav_verify_ssl if hasattr(settings, "webdav_verify_ssl") else True
|
||||
)
|
||||
|
||||
# Check if upload was successful
|
||||
if response.status_code in (200, 201, 204):
|
||||
logger.info(f"Successfully uploaded {filename} to WebDAV at {webdav_url}.")
|
||||
return {"status": "Completed", "file": file_path, "url": webdav_url}
|
||||
else:
|
||||
error_msg = f"Failed to upload {filename} to WebDAV: {response.status_code} - {response.text}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error uploading {filename} to WebDAV: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import json
|
||||
import tempfile
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.celery_app import celery
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
def upload_with_rclone(file_path: str, destination: str):
|
||||
"""
|
||||
Uploads a file using rclone to the specified destination.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to upload
|
||||
destination: Rclone destination in format "remote:path/to/folder"
|
||||
e.g. "gdrive:Uploads" or "dropbox:Documents/Uploads"
|
||||
"""
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
# Extract filename
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
# Check if rclone is installed and config exists
|
||||
rclone_config_path = os.path.join(settings.workdir, "rclone.conf")
|
||||
if not os.path.exists(rclone_config_path):
|
||||
error_msg = f"Rclone configuration not found at {rclone_config_path}"
|
||||
logger.error(error_msg)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
try:
|
||||
# Split destination into remote and path
|
||||
if ":" not in destination:
|
||||
raise ValueError(f"Invalid destination format: {destination}. Expected format: remote:path")
|
||||
|
||||
remote, remote_path = destination.split(":", 1)
|
||||
|
||||
# Ensure the remote path exists (create folders if needed)
|
||||
mkdir_cmd = [
|
||||
"rclone",
|
||||
"mkdir",
|
||||
"--config", rclone_config_path,
|
||||
destination
|
||||
]
|
||||
|
||||
subprocess.run(mkdir_cmd, check=True, capture_output=True)
|
||||
|
||||
# Construct the upload command
|
||||
upload_cmd = [
|
||||
"rclone",
|
||||
"copy",
|
||||
"--config", rclone_config_path,
|
||||
file_path,
|
||||
destination,
|
||||
"--progress"
|
||||
]
|
||||
|
||||
# Execute the upload command
|
||||
result = subprocess.run(upload_cmd, check=True, capture_output=True, text=True)
|
||||
|
||||
# Check if upload was successful
|
||||
if result.returncode == 0:
|
||||
# Try to get a public link if possible
|
||||
try:
|
||||
link_cmd = [
|
||||
"rclone",
|
||||
"link",
|
||||
"--config", rclone_config_path,
|
||||
f"{destination}/{filename}"
|
||||
]
|
||||
link_result = subprocess.run(link_cmd, capture_output=True, text=True)
|
||||
public_url = link_result.stdout.strip() if link_result.returncode == 0 else None
|
||||
except Exception:
|
||||
public_url = None
|
||||
|
||||
logger.info(f"Successfully uploaded {filename} to {destination}")
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file": file_path,
|
||||
"destination": destination,
|
||||
"public_url": public_url
|
||||
}
|
||||
else:
|
||||
error_msg = f"Failed to upload {filename} to {destination}: {result.stderr}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
error_msg = f"Rclone error: {e.stderr.decode('utf-8') if hasattr(e.stderr, 'decode') else e.stderr}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error uploading {filename} to {destination}: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
def send_to_all_rclone_destinations(file_path: str):
|
||||
"""
|
||||
Uploads a file to all configured rclone destinations.
|
||||
Destinations are loaded from the rclone configuration file.
|
||||
"""
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
# Extract filename
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
# Path to rclone config
|
||||
rclone_config_path = os.path.join(settings.workdir, "rclone.conf")
|
||||
if not os.path.exists(rclone_config_path):
|
||||
error_msg = f"Rclone configuration not found at {rclone_config_path}"
|
||||
logger.error(error_msg)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
# Get list of configured destinations from rclone
|
||||
try:
|
||||
remotes_cmd = ["rclone", "listremotes", "--config", rclone_config_path]
|
||||
result = subprocess.run(remotes_cmd, check=True, capture_output=True, text=True)
|
||||
|
||||
if result.returncode == 0:
|
||||
# Process the list of remotes
|
||||
remotes = [r.strip() for r in result.stdout.splitlines() if r.strip()]
|
||||
|
||||
# Target directories for each remote (from settings)
|
||||
remote_paths = {}
|
||||
for remote in remotes:
|
||||
remote_name = remote.rstrip(':')
|
||||
path_setting_name = f"rclone_{remote_name}_path"
|
||||
if hasattr(settings, path_setting_name) and getattr(settings, path_setting_name):
|
||||
remote_paths[remote] = getattr(settings, path_setting_name)
|
||||
else:
|
||||
# Default to root of remote if not specified
|
||||
remote_paths[remote] = ""
|
||||
|
||||
# Queue upload tasks for each configured destination
|
||||
results = {}
|
||||
for remote, path in remote_paths.items():
|
||||
full_destination = f"{remote}{path}"
|
||||
if path and not path.endswith('/'):
|
||||
full_destination += '/'
|
||||
|
||||
logger.info(f"Queueing {file_path} for upload to {full_destination}")
|
||||
task = upload_with_rclone.delay(file_path, full_destination)
|
||||
results[f"rclone_{remote.rstrip(':')}_task_id"] = task.id
|
||||
|
||||
return {
|
||||
"status": "Queued",
|
||||
"file_path": file_path,
|
||||
"tasks": results
|
||||
}
|
||||
else:
|
||||
error_msg = f"Failed to list rclone remotes: {result.stderr}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error setting up rclone uploads for {filename}: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
Reference in New Issue
Block a user