fix: merge main branch and renumber migration 027→037
Resolve 3 merge conflicts and renumber the automation_hooks migration to follow main's migration chain (036_add_document_translation_fields). Conflicts resolved: - app/api/__init__.py: add automation_router alongside main's new routers - app/utils/settings_service.py: add automation_hooks_enabled alongside compliance_enabled - tests/conftest.py: add AutomationHook alongside AuditLog/ComplianceTemplate imports Migration renumbered: - 027_add_automation_hooks → 037_add_automation_hooks - down_revision: 026_add_scheduled_jobs → 036_add_document_translation_fields Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -216,6 +216,30 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
|
||||
except Exception as search_exc:
|
||||
logger.warning(f"[{task_id}] Meilisearch indexing failed (non-fatal): {search_exc}")
|
||||
|
||||
# Cache the detected language on the FileRecord and trigger
|
||||
# default-language translation when the document is in a
|
||||
# different language.
|
||||
detected_lang = metadata.get("language") if metadata else None
|
||||
if detected_lang and extracted_text:
|
||||
try:
|
||||
file_record.detected_language = detected_lang
|
||||
db.commit()
|
||||
|
||||
from app.tasks.translate_to_default_language import translate_to_default_language
|
||||
|
||||
translate_to_default_language.delay(
|
||||
file_id,
|
||||
extracted_text,
|
||||
detected_lang,
|
||||
owner_id=file_record.owner_id,
|
||||
)
|
||||
logger.info(
|
||||
f"[{task_id}] Queued default-language translation for file {file_id} "
|
||||
f"(detected: {detected_lang})"
|
||||
)
|
||||
except Exception as trans_exc:
|
||||
logger.warning(f"[{task_id}] Could not queue translation task (non-fatal): {trans_exc}")
|
||||
|
||||
# Persist the metadata into a JSON file with the same base name.
|
||||
# Include file path references for traceability
|
||||
logger.info(f"[{task_id}] Persisting metadata to JSON")
|
||||
|
||||
@@ -1,191 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
# Import the shared Celery instance
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import log_task_progress
|
||||
from app.utils.ai_provider import get_ai_provider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_json_from_text(text):
|
||||
"""
|
||||
Try to extract a JSON object from the text.
|
||||
- First, check for a JSON block inside triple backticks.
|
||||
- If not found, try to extract text from the first '{' to the last '}'.
|
||||
"""
|
||||
pattern = r"```(?:json)?\s*(\{.*?\})\s*```"
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1)
|
||||
else:
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start != -1 and end != -1 and end > start:
|
||||
return text[start : end + 1]
|
||||
return None
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: int = None):
|
||||
"""
|
||||
Uses OpenAI to classify document metadata.
|
||||
|
||||
Args:
|
||||
filename: Can be either a basename (e.g., "file.pdf") or a full path (e.g., "/workdir/processed/file.pdf")
|
||||
cleaned_text: The extracted text from the document
|
||||
file_id: Optional file ID for tracking
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting metadata extraction for: {filename}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_metadata_with_gpt",
|
||||
"in_progress",
|
||||
f"Extracting metadata for {os.path.basename(filename)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Get file_id from database if not provided
|
||||
if file_id is None:
|
||||
tmp_dir = os.path.join(settings.workdir, "tmp")
|
||||
# Handle both basename and full path
|
||||
if os.path.isabs(filename):
|
||||
file_path = filename
|
||||
else:
|
||||
file_path = os.path.join(tmp_dir, filename)
|
||||
if os.path.exists(file_path):
|
||||
with SessionLocal() as db:
|
||||
file_record = db.query(FileRecord).filter_by(local_filename=file_path).first()
|
||||
if file_record:
|
||||
file_id = file_record.id
|
||||
|
||||
prompt = (
|
||||
"You are a specialized document analyzer trained to extract structured metadata from documents.\n"
|
||||
"Your task is to analyze the given text and return a well-structured JSON object.\n\n"
|
||||
"Extract and return the following fields:\n"
|
||||
"1. **filename**: Machine-readable filename "
|
||||
"(YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, periods, and underscores).\n"
|
||||
'2. **empfaenger**: The recipient, or "Unknown" if not found.\n'
|
||||
'3. **absender**: The sender, or "Unknown" if not found.\n'
|
||||
"4. **correspondent**: The entity or company that issued the document "
|
||||
'(shortest possible name, e.g., "Amazon" instead of "Amazon EU SARL, German branch").\n'
|
||||
"5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, "
|
||||
"Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].\n"
|
||||
"6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, "
|
||||
"Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, "
|
||||
"Private_Korrespondenz, Sonstige_Informationen].\n"
|
||||
"7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).\n"
|
||||
"8. **tags**: A list of up to 4 relevant thematic keywords.\n"
|
||||
'9. **language**: Detected document language (ISO 639-1 code, e.g., "de" or "en").\n'
|
||||
"10. **title**: A human-readable title summarizing the document content.\n"
|
||||
"11. **confidence_score**: A numeric value (0-100) indicating the confidence level "
|
||||
"of the extracted metadata.\n"
|
||||
"12. **reference_number**: Extracted invoice/order/reference number if available.\n"
|
||||
"13. **monetary_amounts**: A list of key monetary values detected in the document.\n\n"
|
||||
"### Important Rules:\n"
|
||||
"- **OCR Correction**: Assume the text has been corrected for OCR errors.\n"
|
||||
"- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.\n"
|
||||
"- **Title**: Concise, no addresses, and contains key identifying features.\n"
|
||||
"- **Date Selection**: Use the most relevant date if multiple are found.\n"
|
||||
"- **Output Language**: Maintain the document's original language.\n\n"
|
||||
f"Extracted text:\n{cleaned_text}\n\n"
|
||||
"Return only valid JSON with no additional commentary.\n"
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(f"[{task_id}] Sending classification request for {filename}...")
|
||||
log_task_progress(task_id, "call_ai_provider", "in_progress", "Calling AI provider API", file_id=file_id)
|
||||
provider = get_ai_provider()
|
||||
model = settings.ai_model or settings.openai_model
|
||||
content = provider.chat_completion(
|
||||
messages=[
|
||||
{"role": "system", "content": "You are an intelligent document classifier."},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
model=model,
|
||||
temperature=0,
|
||||
)
|
||||
|
||||
logger.info(f"[{task_id}] Raw classification response for {filename}: {content[:200]}...")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"call_ai_provider",
|
||||
"success",
|
||||
"Received AI provider response",
|
||||
file_id=file_id,
|
||||
detail=f"Raw classification response:\n{content}",
|
||||
)
|
||||
|
||||
json_text = extract_json_from_text(content)
|
||||
if not json_text:
|
||||
logger.error(f"[{task_id}] Could not find valid JSON in GPT response for {filename}.")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_metadata_with_gpt",
|
||||
"failure",
|
||||
"Invalid JSON in response",
|
||||
file_id=file_id,
|
||||
detail=f"Could not parse valid JSON from GPT response.\nRaw response:\n{content}",
|
||||
)
|
||||
return {}
|
||||
|
||||
metadata = json.loads(json_text)
|
||||
|
||||
# SECURITY: Validate filename format from GPT to prevent path traversal
|
||||
# The prompt requests filenames with only letters, numbers, periods, and underscores
|
||||
# Enforce this constraint to prevent malicious filenames
|
||||
suggested_filename = metadata.get("filename", "")
|
||||
if suggested_filename:
|
||||
# Check if filename contains only safe characters AND explicitly check for ".."
|
||||
# Defense in depth: While the regex [\w\-\. ]+ already excludes / and \,
|
||||
# we explicitly reject ".." to guard against:
|
||||
# 1. Potential locale-specific \w behavior
|
||||
# 2. Files literally named ".." which are valid but problematic
|
||||
# 3. Future code changes that might relax the regex
|
||||
if not re.match(r"^[\w\-\. ]+$", suggested_filename) or ".." in suggested_filename:
|
||||
logger.warning(f"[{task_id}] Invalid filename format from GPT: '{suggested_filename}', using fallback")
|
||||
# Reset to empty to trigger fallback to original filename
|
||||
metadata["filename"] = ""
|
||||
|
||||
logger.info(f"[{task_id}] Extracted metadata: {metadata}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"parse_metadata",
|
||||
"success",
|
||||
f"Parsed metadata: {list(metadata.keys())}",
|
||||
file_id=file_id,
|
||||
detail=f"Extracted metadata:\n{json.dumps(metadata, ensure_ascii=False, indent=2)}",
|
||||
)
|
||||
|
||||
# Trigger the next step: embedding metadata into the PDF
|
||||
# Pass the filename (can be basename or full path) so embed_metadata_into_pdf can find the file on disk
|
||||
logger.info(f"[{task_id}] Queueing metadata embedding task")
|
||||
log_task_progress(
|
||||
task_id, "extract_metadata_with_gpt", "success", "Metadata extracted, queuing embed task", file_id=file_id
|
||||
)
|
||||
embed_metadata_into_pdf.delay(filename, cleaned_text, metadata, file_id)
|
||||
|
||||
return {"s3_file": os.path.basename(filename), "metadata": metadata}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"[{task_id}] AI provider classification failed for {filename}: {e}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_metadata_with_gpt",
|
||||
"failure",
|
||||
f"Exception: {str(e)}",
|
||||
file_id=file_id,
|
||||
detail=f"AI provider classification failed for {filename}.\nException: {str(e)}",
|
||||
)
|
||||
return {}
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
# Import the shared Celery instance
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import log_task_progress
|
||||
from app.utils.ai_provider import get_ai_provider
|
||||
from app.utils.filename_utils import VALID_FILENAME_RE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_json_from_text(text):
|
||||
"""
|
||||
Try to extract a JSON object from the text.
|
||||
- First, check for a JSON block inside triple backticks.
|
||||
- If not found, try to extract text from the first '{' to the last '}'.
|
||||
"""
|
||||
pattern = r"```(?:json)?\s*(\{.*?\})\s*```"
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1)
|
||||
else:
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start != -1 and end != -1 and end > start:
|
||||
return text[start : end + 1]
|
||||
return None
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: int = None):
|
||||
"""
|
||||
Uses OpenAI to classify document metadata.
|
||||
|
||||
Args:
|
||||
filename: Can be either a basename (e.g., "file.pdf") or a full path (e.g., "/workdir/processed/file.pdf")
|
||||
cleaned_text: The extracted text from the document
|
||||
file_id: Optional file ID for tracking
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting metadata extraction for: {filename}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_metadata_with_gpt",
|
||||
"in_progress",
|
||||
f"Extracting metadata for {os.path.basename(filename)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Get file_id from database if not provided
|
||||
if file_id is None:
|
||||
tmp_dir = os.path.join(settings.workdir, "tmp")
|
||||
# Handle both basename and full path
|
||||
if os.path.isabs(filename):
|
||||
file_path = filename
|
||||
else:
|
||||
file_path = os.path.join(tmp_dir, filename)
|
||||
if os.path.exists(file_path):
|
||||
with SessionLocal() as db:
|
||||
file_record = db.query(FileRecord).filter_by(local_filename=file_path).first()
|
||||
if file_record:
|
||||
file_id = file_record.id
|
||||
|
||||
prompt = (
|
||||
"You are a specialized document analyzer trained to extract structured metadata from documents.\n"
|
||||
"Your task is to analyze the given text and return a well-structured JSON object.\n\n"
|
||||
"Extract and return the following fields:\n"
|
||||
"1. **filename**: Machine-readable filename "
|
||||
"(YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, spaces, dashes, periods, and underscores).\n"
|
||||
'2. **empfaenger**: The recipient, or "Unknown" if not found.\n'
|
||||
'3. **absender**: The sender, or "Unknown" if not found.\n'
|
||||
"4. **correspondent**: The entity or company that issued the document "
|
||||
'(shortest possible name, e.g., "Amazon" instead of "Amazon EU SARL, German branch").\n'
|
||||
"5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, "
|
||||
"Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].\n"
|
||||
"6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, "
|
||||
"Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, "
|
||||
"Private_Korrespondenz, Sonstige_Informationen].\n"
|
||||
"7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).\n"
|
||||
"8. **tags**: A list of up to 4 relevant thematic keywords.\n"
|
||||
'9. **language**: Detected document language (ISO 639-1 code, e.g., "de" or "en").\n'
|
||||
"10. **title**: A human-readable title summarizing the document content.\n"
|
||||
"11. **confidence_score**: A numeric value (0-100) indicating the confidence level "
|
||||
"of the extracted metadata.\n"
|
||||
"12. **reference_number**: Extracted invoice/order/reference number if available.\n"
|
||||
"13. **monetary_amounts**: A list of key monetary values detected in the document.\n\n"
|
||||
"### Important Rules:\n"
|
||||
"- **OCR Correction**: Assume the text has been corrected for OCR errors.\n"
|
||||
"- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.\n"
|
||||
"- **Title**: Concise, no addresses, and contains key identifying features.\n"
|
||||
"- **Date Selection**: Use the most relevant date if multiple are found.\n"
|
||||
"- **Output Language**: Maintain the document's original language.\n\n"
|
||||
f"Extracted text:\n{cleaned_text}\n\n"
|
||||
"Return only valid JSON with no additional commentary.\n"
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(f"[{task_id}] Sending classification request for {filename}...")
|
||||
log_task_progress(task_id, "call_ai_provider", "in_progress", "Calling AI provider API", file_id=file_id)
|
||||
provider = get_ai_provider()
|
||||
model = settings.ai_model or settings.openai_model
|
||||
content = provider.chat_completion(
|
||||
messages=[
|
||||
{"role": "system", "content": "You are an intelligent document classifier."},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
model=model,
|
||||
temperature=0,
|
||||
)
|
||||
|
||||
logger.info(f"[{task_id}] Raw classification response for {filename}: {content[:200]}...")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"call_ai_provider",
|
||||
"success",
|
||||
"Received AI provider response",
|
||||
file_id=file_id,
|
||||
detail=f"Raw classification response:\n{content}",
|
||||
)
|
||||
|
||||
json_text = extract_json_from_text(content)
|
||||
if not json_text:
|
||||
logger.error(f"[{task_id}] Could not find valid JSON in GPT response for {filename}.")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_metadata_with_gpt",
|
||||
"failure",
|
||||
"Invalid JSON in response",
|
||||
file_id=file_id,
|
||||
detail=f"Could not parse valid JSON from GPT response.\nRaw response:\n{content}",
|
||||
)
|
||||
return {}
|
||||
|
||||
metadata = json.loads(json_text)
|
||||
|
||||
# SECURITY: Validate filename format from GPT to prevent path traversal
|
||||
# The prompt requests filenames with only letters, numbers, periods, and underscores
|
||||
# Enforce this constraint to prevent malicious filenames
|
||||
suggested_filename = metadata.get("filename", "")
|
||||
if suggested_filename:
|
||||
# Check if filename contains only safe characters AND explicitly check for ".."
|
||||
# Defense in depth: While the regex VALID_FILENAME_PATTERN already excludes / and \,
|
||||
# we explicitly reject ".." to guard against:
|
||||
# 1. Potential locale-specific \w behavior
|
||||
# 2. Files literally named ".." which are valid but problematic
|
||||
# 3. Future code changes that might relax the regex
|
||||
if not VALID_FILENAME_RE.match(suggested_filename) or ".." in suggested_filename:
|
||||
logger.warning(f"[{task_id}] Invalid filename format from GPT: '{suggested_filename}', using fallback")
|
||||
# Reset to empty to trigger fallback to original filename
|
||||
metadata["filename"] = ""
|
||||
|
||||
logger.info(f"[{task_id}] Extracted metadata: {metadata}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"parse_metadata",
|
||||
"success",
|
||||
f"Parsed metadata: {list(metadata.keys())}",
|
||||
file_id=file_id,
|
||||
detail=f"Extracted metadata:\n{json.dumps(metadata, ensure_ascii=False, indent=2)}",
|
||||
)
|
||||
|
||||
# Trigger the next step: embedding metadata into the PDF
|
||||
# Pass the filename (can be basename or full path) so embed_metadata_into_pdf can find the file on disk
|
||||
logger.info(f"[{task_id}] Queueing metadata embedding task")
|
||||
log_task_progress(
|
||||
task_id, "extract_metadata_with_gpt", "success", "Metadata extracted, queuing embed task", file_id=file_id
|
||||
)
|
||||
embed_metadata_into_pdf.delay(filename, cleaned_text, metadata, file_id)
|
||||
|
||||
return {"s3_file": os.path.basename(filename), "metadata": metadata}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"[{task_id}] AI provider classification failed for {filename}: {e}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_metadata_with_gpt",
|
||||
"failure",
|
||||
f"Exception: {str(e)}",
|
||||
file_id=file_id,
|
||||
detail=f"AI provider classification failed for {filename}.\nException: {str(e)}",
|
||||
)
|
||||
return {}
|
||||
|
||||
@@ -21,8 +21,9 @@ from app.tasks.send_to_all import (
|
||||
# Import database and logging utils from main
|
||||
from app.utils import log_task_progress
|
||||
|
||||
# Import notification utility
|
||||
# Import notification utilities
|
||||
from app.utils.notification import notify_file_processed
|
||||
from app.utils.user_notification import notify_user_document_processed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -139,4 +140,15 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
|
||||
except Exception as e:
|
||||
logger.warning(f"[WARNING] Failed to send file processed notification: {e}")
|
||||
|
||||
# 6. Send per-user notification
|
||||
if owner_id:
|
||||
try:
|
||||
notify_user_document_processed(
|
||||
owner_id=owner_id,
|
||||
filename=os.path.basename(processed_file),
|
||||
file_id=file_id,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"[WARNING] Failed to send per-user processed notification: {e}")
|
||||
|
||||
return {"status": "Completed", "file": processed_file}
|
||||
|
||||
+92
-25
@@ -13,7 +13,11 @@ from celery import shared_task
|
||||
from app.config import settings
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
|
||||
from app.tasks.process_document import process_document # Updated import
|
||||
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES
|
||||
from app.utils.allowed_types import (
|
||||
ALL_CATEGORIES,
|
||||
DEFAULT_CATEGORIES,
|
||||
get_allowed_types_for_categories,
|
||||
)
|
||||
|
||||
# Database session for per-user IMAP accounts (imported lazily to avoid circular imports)
|
||||
_db_session_factory = None
|
||||
@@ -50,6 +54,38 @@ def _decrypt_imap_password(password: str | None) -> str | None:
|
||||
return decrypt_value(password)
|
||||
|
||||
|
||||
def _resolve_categories_for_profile(profile_id: int | None) -> list[str]:
|
||||
"""Return the list of allowed categories for a profile ID.
|
||||
|
||||
Loads the profile from the database. If ``profile_id`` is ``None`` or the
|
||||
profile is not found, falls back to the global ``settings.imap_attachment_filter``
|
||||
string (``'documents_only'`` → default categories; ``'all'`` → all categories).
|
||||
"""
|
||||
if profile_id is not None:
|
||||
try:
|
||||
from app.models import ImapIngestionProfile
|
||||
|
||||
db = _get_db_session()
|
||||
try:
|
||||
profile = db.query(ImapIngestionProfile).filter(ImapIngestionProfile.id == profile_id).first()
|
||||
if profile:
|
||||
return json.loads(profile.allowed_categories)
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"Could not load IMAP ingestion profile %d (%s: %s) — using global default",
|
||||
profile_id,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
|
||||
# Fall back to global setting
|
||||
if settings.imap_attachment_filter == "all":
|
||||
return ALL_CATEGORIES
|
||||
return DEFAULT_CATEGORIES
|
||||
|
||||
|
||||
LOCK_KEY = "imap_lock" # Unique key for locking
|
||||
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
|
||||
|
||||
@@ -180,6 +216,7 @@ def _pull_user_imap_accounts() -> None:
|
||||
use_ssl=acct.use_ssl,
|
||||
delete_after_process=acct.delete_after_process,
|
||||
owner_id=acct.owner_id,
|
||||
allowed_categories=_resolve_categories_for_profile(acct.profile_id),
|
||||
)
|
||||
# Record successful poll
|
||||
acct.last_checked_at = datetime.now(timezone.utc)
|
||||
@@ -250,6 +287,9 @@ def _pull_user_integration_imap() -> None:
|
||||
use_ssl = cfg.get("use_ssl", True)
|
||||
delete_after = cfg.get("delete_after_process", False)
|
||||
gmail_labels = cfg.get("gmail_apply_labels", True)
|
||||
# Integrations can store a profile_id in config; fall back to global default
|
||||
profile_id = cfg.get("profile_id")
|
||||
allowed_categories = _resolve_categories_for_profile(profile_id)
|
||||
|
||||
if not (host and username and password):
|
||||
logger.warning(
|
||||
@@ -269,6 +309,7 @@ def _pull_user_integration_imap() -> None:
|
||||
delete_after_process=delete_after,
|
||||
owner_id=integ.owner_id,
|
||||
gmail_apply_labels=gmail_labels,
|
||||
allowed_categories=allowed_categories,
|
||||
)
|
||||
integ.last_used_at = datetime.now(timezone.utc)
|
||||
integ.last_error = None
|
||||
@@ -329,6 +370,7 @@ def pull_inbox(
|
||||
delete_after_process,
|
||||
owner_id=None,
|
||||
gmail_apply_labels=True,
|
||||
allowed_categories=None,
|
||||
):
|
||||
"""
|
||||
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
|
||||
@@ -345,8 +387,22 @@ def pull_inbox(
|
||||
attributed to this user via ``process_document`` / ``convert_to_pdf``.
|
||||
gmail_apply_labels: Whether to apply Gmail-specific labels and stars to
|
||||
processed emails. Only relevant for Gmail hosts. Defaults to True.
|
||||
allowed_categories: List of file-type category keys to ingest (e.g.
|
||||
``["pdf", "office", "images"]``). ``None`` falls back to the
|
||||
global ``settings.imap_attachment_filter`` mapping.
|
||||
"""
|
||||
logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
|
||||
if allowed_categories is None:
|
||||
allowed_categories = _resolve_categories_for_profile(None)
|
||||
|
||||
effective_mime_types, effective_extensions = get_allowed_types_for_categories(allowed_categories)
|
||||
logger.info(
|
||||
"Connecting to %s at %s:%s (SSL=%s) — categories: %s",
|
||||
mailbox_key,
|
||||
host,
|
||||
port,
|
||||
use_ssl,
|
||||
allowed_categories,
|
||||
)
|
||||
processed_emails = load_processed_emails()
|
||||
|
||||
try:
|
||||
@@ -405,9 +461,13 @@ def pull_inbox(
|
||||
logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
|
||||
continue
|
||||
|
||||
# Process attachments (and convert non-PDF files).
|
||||
# We call the function without assigning its return value since it is not used.
|
||||
fetch_attachments_and_enqueue(email_message, owner_id=owner_id)
|
||||
# Process attachments using the resolved mime types / extensions.
|
||||
fetch_attachments_and_enqueue(
|
||||
email_message,
|
||||
owner_id=owner_id,
|
||||
effective_mime_types=effective_mime_types,
|
||||
effective_extensions=effective_extensions,
|
||||
)
|
||||
|
||||
if settings.imap_readonly_mode:
|
||||
logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
|
||||
@@ -436,27 +496,23 @@ def pull_inbox(
|
||||
logger.exception("Error pulling mailbox %s: %s", mailbox_key, e)
|
||||
|
||||
|
||||
def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
|
||||
def fetch_attachments_and_enqueue(
|
||||
email_message,
|
||||
owner_id: str | None = None,
|
||||
effective_mime_types: frozenset[str] | None = None,
|
||||
effective_extensions: frozenset[str] | None = None,
|
||||
):
|
||||
"""
|
||||
Extracts attachments from the email and processes only allowed file types.
|
||||
|
||||
Files are accepted if either:
|
||||
1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR
|
||||
2. They have a '.pdf' file extension (regardless of MIME type)
|
||||
The caller is responsible for computing ``effective_mime_types`` and
|
||||
``effective_extensions`` from the relevant :class:`ImapIngestionProfile` (or
|
||||
the global default) via :func:`app.utils.allowed_types.get_allowed_types_for_categories`
|
||||
before calling this function. ``pull_inbox`` does this automatically.
|
||||
|
||||
Allowed file types include:
|
||||
- PDF: application/pdf or *.pdf extension
|
||||
- Microsoft Office files:
|
||||
- Word: application/msword,
|
||||
application/vnd.openxmlformats-officedocument.wordprocessingml.document
|
||||
- Excel: application/vnd.ms-excel,
|
||||
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
|
||||
- PowerPoint: application/vnd.ms-powerpoint,
|
||||
application/vnd.openxmlformats-officedocument.presentationml.presentation
|
||||
- Other meaningful attachments:
|
||||
- Plain text: text/plain
|
||||
- CSV: text/csv
|
||||
- Rich Text Format: application/rtf, text/rtf
|
||||
If either set is ``None`` the function falls back to the default category list
|
||||
so the function still works correctly when called directly in tests or from
|
||||
other contexts.
|
||||
|
||||
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
|
||||
any other allowed file is enqueued for conversion to PDF.
|
||||
@@ -465,9 +521,14 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
|
||||
email_message: The parsed email message to extract attachments from.
|
||||
owner_id: Optional user identifier forwarded to ``process_document`` /
|
||||
``convert_to_pdf`` for multi-tenant attribution.
|
||||
effective_mime_types: Pre-computed frozenset of allowed MIME type strings.
|
||||
effective_extensions: Pre-computed frozenset of allowed file extension strings.
|
||||
|
||||
Returns True if at least one allowed attachment was processed.
|
||||
"""
|
||||
if effective_mime_types is None or effective_extensions is None:
|
||||
effective_mime_types, effective_extensions = get_allowed_types_for_categories(DEFAULT_CATEGORIES)
|
||||
|
||||
has_attachment = False
|
||||
for part in email_message.walk():
|
||||
if part.get_content_maintype() == "multipart":
|
||||
@@ -482,9 +543,15 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
|
||||
|
||||
mime_type = part.get_content_type()
|
||||
file_ext = os.path.splitext(filename)[1].lower()
|
||||
|
||||
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
|
||||
if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension:
|
||||
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
|
||||
if mime_type not in effective_mime_types and file_ext not in effective_extensions and not is_pdf_by_extension:
|
||||
logger.info(
|
||||
"Skipping attachment %s (MIME: %s, ext: %s) — not in effective allowed set",
|
||||
filename,
|
||||
mime_type,
|
||||
file_ext,
|
||||
)
|
||||
continue
|
||||
|
||||
file_path = os.path.join(settings.workdir, filename)
|
||||
@@ -495,7 +562,7 @@ def fetch_attachments_and_enqueue(email_message, owner_id: str | None = None):
|
||||
if mime_type == "application/pdf" or is_pdf_by_extension:
|
||||
process_document.delay(file_path, owner_id=owner_id)
|
||||
logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type)
|
||||
elif mime_type in ALLOWED_MIME_TYPES:
|
||||
elif mime_type in effective_mime_types:
|
||||
# Other allowed files are sent for conversion
|
||||
convert_to_pdf.delay(file_path, owner_id=owner_id)
|
||||
logger.info("Enqueued file for conversion to PDF: %s", filename)
|
||||
|
||||
+61
-12
@@ -12,6 +12,7 @@ from app.tasks.upload_to_dropbox import upload_to_dropbox
|
||||
from app.tasks.upload_to_email import upload_to_email
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive
|
||||
from app.tasks.upload_to_icloud import upload_to_icloud
|
||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||
from app.tasks.upload_to_onedrive import upload_to_onedrive
|
||||
from app.tasks.upload_to_paperless import upload_to_paperless
|
||||
@@ -25,18 +26,32 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _should_upload_to_dropbox():
|
||||
return bool(settings.dropbox_app_key and settings.dropbox_app_secret and settings.dropbox_refresh_token)
|
||||
return bool(
|
||||
getattr(settings, "dropbox_enabled", True)
|
||||
and settings.dropbox_app_key
|
||||
and settings.dropbox_app_secret
|
||||
and settings.dropbox_refresh_token
|
||||
)
|
||||
|
||||
|
||||
def _should_upload_to_nextcloud():
|
||||
return bool(settings.nextcloud_upload_url and settings.nextcloud_username and settings.nextcloud_password)
|
||||
return bool(
|
||||
getattr(settings, "nextcloud_enabled", True)
|
||||
and settings.nextcloud_upload_url
|
||||
and settings.nextcloud_username
|
||||
and settings.nextcloud_password
|
||||
)
|
||||
|
||||
|
||||
def _should_upload_to_paperless():
|
||||
return bool(settings.paperless_ngx_api_token and settings.paperless_host)
|
||||
return bool(
|
||||
getattr(settings, "paperless_enabled", True) and settings.paperless_ngx_api_token and settings.paperless_host
|
||||
)
|
||||
|
||||
|
||||
def _should_upload_to_google_drive():
|
||||
if not getattr(settings, "google_drive_enabled", True):
|
||||
return False
|
||||
# Check for OAuth configuration
|
||||
if getattr(settings, "google_drive_use_oauth", False):
|
||||
return bool(
|
||||
@@ -51,20 +66,33 @@ def _should_upload_to_google_drive():
|
||||
|
||||
|
||||
def _should_upload_to_webdav():
|
||||
return bool(settings.webdav_url and settings.webdav_username and settings.webdav_password)
|
||||
return bool(
|
||||
getattr(settings, "webdav_enabled", True)
|
||||
and settings.webdav_url
|
||||
and settings.webdav_username
|
||||
and settings.webdav_password
|
||||
)
|
||||
|
||||
|
||||
def _should_upload_to_ftp():
|
||||
return bool(settings.ftp_host and settings.ftp_username and settings.ftp_password)
|
||||
return bool(
|
||||
getattr(settings, "ftp_enabled", True) and settings.ftp_host and settings.ftp_username and settings.ftp_password
|
||||
)
|
||||
|
||||
|
||||
def _should_upload_to_sftp():
|
||||
return bool(settings.sftp_host and settings.sftp_username and (settings.sftp_password or settings.sftp_private_key))
|
||||
return bool(
|
||||
getattr(settings, "sftp_enabled", True)
|
||||
and settings.sftp_host
|
||||
and settings.sftp_username
|
||||
and (settings.sftp_password or settings.sftp_private_key)
|
||||
)
|
||||
|
||||
|
||||
def _should_upload_to_email():
|
||||
return bool(
|
||||
settings.dest_email_host
|
||||
getattr(settings, "dest_email_enabled", True)
|
||||
and settings.dest_email_host
|
||||
and settings.dest_email_username
|
||||
and settings.dest_email_password
|
||||
and settings.dest_email_default_recipient
|
||||
@@ -72,18 +100,32 @@ def _should_upload_to_email():
|
||||
|
||||
|
||||
def _should_upload_to_onedrive():
|
||||
return bool(settings.onedrive_client_id and settings.onedrive_client_secret and settings.onedrive_refresh_token)
|
||||
return bool(
|
||||
getattr(settings, "onedrive_enabled", True)
|
||||
and settings.onedrive_client_id
|
||||
and settings.onedrive_client_secret
|
||||
and settings.onedrive_refresh_token
|
||||
)
|
||||
|
||||
|
||||
def _should_upload_to_s3():
|
||||
return bool(settings.s3_bucket_name and settings.aws_access_key_id and settings.aws_secret_access_key)
|
||||
return bool(
|
||||
getattr(settings, "s3_enabled", True)
|
||||
and settings.s3_bucket_name
|
||||
and settings.aws_access_key_id
|
||||
and settings.aws_secret_access_key
|
||||
)
|
||||
|
||||
|
||||
def _should_upload_to_icloud():
|
||||
return bool(getattr(settings, "icloud_enabled", True) and settings.icloud_username and settings.icloud_password)
|
||||
|
||||
|
||||
def get_configured_services_from_validator():
|
||||
"""
|
||||
Use the config validator to determine which services are configured properly.
|
||||
Use the config validator to determine which services are configured and enabled.
|
||||
Returns a dictionary with service names as keys and boolean values indicating
|
||||
whether they're properly configured.
|
||||
whether they're properly configured AND explicitly enabled.
|
||||
"""
|
||||
providers = get_provider_status()
|
||||
|
||||
@@ -98,12 +140,14 @@ def get_configured_services_from_validator():
|
||||
"Email": "email",
|
||||
"OneDrive": "onedrive",
|
||||
"S3 Storage": "s3",
|
||||
"iCloud Drive": "icloud",
|
||||
}
|
||||
|
||||
result = {}
|
||||
for provider_name, internal_name in service_map.items():
|
||||
if provider_name in providers:
|
||||
result[internal_name] = providers[provider_name].get("configured", False)
|
||||
provider = providers[provider_name]
|
||||
result[internal_name] = provider.get("configured", False) and provider.get("enabled", True)
|
||||
|
||||
return result
|
||||
|
||||
@@ -206,6 +250,11 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
|
||||
"should_upload": _should_upload_to_s3,
|
||||
"upload_func": upload_to_s3,
|
||||
},
|
||||
{
|
||||
"name": "icloud",
|
||||
"should_upload": _should_upload_to_icloud,
|
||||
"upload_func": upload_to_icloud,
|
||||
},
|
||||
]
|
||||
|
||||
# Optionally get configuration status from validator
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Celery task to translate extracted document text into the default target language.
|
||||
|
||||
This task is triggered after metadata extraction when the detected document
|
||||
language differs from the user's (or system) default document language. The
|
||||
translated text is persisted in ``FileRecord.default_language_text`` so that
|
||||
users can always read a reference copy in their preferred language.
|
||||
|
||||
Other ad-hoc translations are generated on the fly via the ``/api/files/{id}/translate``
|
||||
endpoint and are NOT persisted.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord, UserProfile
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import log_task_progress
|
||||
from app.utils.ai_provider import get_ai_provider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _resolve_default_language(owner_id: str | None) -> str:
|
||||
"""Return the default document language for the given owner.
|
||||
|
||||
Resolution order:
|
||||
1. ``UserProfile.default_document_language`` (per-user override)
|
||||
2. ``settings.default_document_language`` (global setting)
|
||||
"""
|
||||
if owner_id:
|
||||
with SessionLocal() as db:
|
||||
profile = db.query(UserProfile).filter_by(user_id=owner_id).first()
|
||||
if profile and profile.default_document_language:
|
||||
return profile.default_document_language
|
||||
return settings.default_document_language
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def translate_to_default_language(
|
||||
self,
|
||||
file_id: int,
|
||||
extracted_text: str,
|
||||
detected_language: str,
|
||||
owner_id: str | None = None,
|
||||
) -> dict:
|
||||
"""Translate *extracted_text* into the default document language and persist the result.
|
||||
|
||||
Args:
|
||||
file_id: Primary key of the :class:`FileRecord`.
|
||||
extracted_text: The OCR / refined text in the document's original language.
|
||||
detected_language: ISO 639-1 code of the document's detected language.
|
||||
owner_id: Owner identifier used to resolve per-user language preference.
|
||||
|
||||
Returns:
|
||||
A dict with ``status``, ``target_language``, and the translated text length.
|
||||
"""
|
||||
task_id = self.request.id
|
||||
target_language = _resolve_default_language(owner_id)
|
||||
|
||||
# Nothing to do when the document is already in the target language.
|
||||
if detected_language == target_language:
|
||||
logger.info(
|
||||
f"[{task_id}] Document {file_id} already in target language '{target_language}', skipping translation"
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"translate_to_default_language",
|
||||
"skipped",
|
||||
f"Document already in {target_language}",
|
||||
file_id=file_id,
|
||||
)
|
||||
return {"status": "skipped", "reason": "already_in_target_language"}
|
||||
|
||||
logger.info(f"[{task_id}] Translating document {file_id} from '{detected_language}' to '{target_language}'")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"translate_to_default_language",
|
||||
"in_progress",
|
||||
f"Translating from {detected_language} to {target_language}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
try:
|
||||
provider = get_ai_provider()
|
||||
model = settings.ai_model or settings.openai_model
|
||||
translated_text = provider.chat_completion(
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
f"You are a professional translator. Translate the following text "
|
||||
f"from {detected_language} to {target_language}. "
|
||||
f"Preserve the original formatting, paragraph structure, and meaning. "
|
||||
f"Do not add any commentary or explanation — output ONLY the translated text."
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": extracted_text},
|
||||
],
|
||||
model=model,
|
||||
temperature=0.3,
|
||||
)
|
||||
|
||||
# Persist the translation.
|
||||
with SessionLocal() as db:
|
||||
record = db.query(FileRecord).filter_by(id=file_id).first()
|
||||
if record:
|
||||
record.default_language_text = translated_text
|
||||
record.default_language_code = target_language
|
||||
record.detected_language = detected_language
|
||||
db.commit()
|
||||
logger.info(
|
||||
f"[{task_id}] Stored default-language translation ({len(translated_text)} chars) for file {file_id}"
|
||||
)
|
||||
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"translate_to_default_language",
|
||||
"success",
|
||||
f"Translated {len(extracted_text)} → {len(translated_text)} chars ({detected_language} → {target_language})",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"target_language": target_language,
|
||||
"translated_length": len(translated_text),
|
||||
}
|
||||
|
||||
except Exception as exc:
|
||||
logger.exception(f"[{task_id}] Translation failed for file {file_id}: {exc}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"translate_to_default_language",
|
||||
"failure",
|
||||
f"Exception: {exc}",
|
||||
file_id=file_id,
|
||||
)
|
||||
raise
|
||||
@@ -11,6 +11,7 @@ from email.mime.image import MIMEImage
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
import pypdf
|
||||
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
||||
|
||||
from app.celery_app import celery
|
||||
@@ -23,6 +24,15 @@ logger = logging.getLogger(__name__)
|
||||
# Constants
|
||||
_LOGO_FILENAME = "logo.png"
|
||||
|
||||
# Mapping from PDF metadata keys (with leading slash stripped) to application-specific names.
|
||||
# This mirrors the inverse of the mapping used in app/tasks/embed_metadata_into_pdf.py.
|
||||
_PDF_METADATA_KEY_MAP = {
|
||||
"Title": "filename",
|
||||
"Author": "absender",
|
||||
"Subject": "document_type",
|
||||
"Keywords": "tags",
|
||||
}
|
||||
|
||||
|
||||
def get_email_template(template_name="default.html"):
|
||||
"""
|
||||
@@ -63,9 +73,12 @@ def extract_metadata_from_file(file_path):
|
||||
"""
|
||||
Try to extract metadata from a file using several methods:
|
||||
1. Check for a .json metadata file with the same name
|
||||
2. Extract metadata from PDF if it's embedded
|
||||
2. Extract embedded metadata from PDF using pypdf
|
||||
|
||||
Returns a dictionary of metadata or None if not found
|
||||
JSON metadata takes precedence; embedded PDF metadata fills in any missing
|
||||
fields using the application's standard key mapping (e.g., /Title → filename).
|
||||
|
||||
Returns a dictionary of metadata (may be empty if none found).
|
||||
"""
|
||||
metadata = {}
|
||||
|
||||
@@ -76,12 +89,28 @@ def extract_metadata_from_file(file_path):
|
||||
with open(metadata_path, "r", encoding="utf-8") as f:
|
||||
metadata = json.load(f)
|
||||
logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
|
||||
return metadata
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load metadata from JSON file: {str(e)}")
|
||||
|
||||
# TODO: For PDF files, try to extract embedded metadata using PyPDF2
|
||||
# This would require additional dependencies, so for now we'll just check for external JSON
|
||||
# Try to extract embedded metadata from PDF
|
||||
if file_path.lower().endswith(".pdf") and os.path.exists(file_path):
|
||||
try:
|
||||
with open(file_path, "rb") as f:
|
||||
pdf_reader = pypdf.PdfReader(f)
|
||||
pdf_metadata = pdf_reader.metadata
|
||||
if pdf_metadata:
|
||||
for key, value in pdf_metadata.items():
|
||||
# Remove the leading slash from PDF metadata keys (e.g., '/Title' -> 'Title')
|
||||
clean_key = key[1:] if key.startswith("/") else key
|
||||
# Map to application-specific key names where possible
|
||||
mapped_key = _PDF_METADATA_KEY_MAP.get(clean_key, clean_key)
|
||||
# Only set if not already present (JSON metadata takes precedence)
|
||||
if mapped_key not in metadata:
|
||||
metadata[mapped_key] = str(value)
|
||||
|
||||
logger.info(f"Extracted embedded metadata from PDF: {file_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to extract metadata from PDF {file_path}: {str(e)}")
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Upload files to Apple iCloud Drive via the pyicloud library.
|
||||
|
||||
This module uses the ``pyicloud`` library to authenticate with Apple's iCloud
|
||||
service and upload files to iCloud Drive. Because Apple does not offer a public
|
||||
REST API for iCloud Drive, this integration relies on the *unofficial*
|
||||
reverse-engineered protocol implemented by ``pyicloud``.
|
||||
|
||||
Requirements
|
||||
~~~~~~~~~~~~
|
||||
* An Apple ID with iCloud Drive enabled.
|
||||
* An **app-specific password** generated at https://appleid.apple.com (required
|
||||
when two-factor authentication is active – which is the default for all modern
|
||||
Apple IDs).
|
||||
* The ``pyicloud`` Python package (``pip install pyicloud``).
|
||||
|
||||
Configuration
|
||||
~~~~~~~~~~~~~
|
||||
Set the following environment variables (or ``app/config.py`` fields):
|
||||
|
||||
* ``ICLOUD_USERNAME`` – Apple ID email address.
|
||||
* ``ICLOUD_PASSWORD`` – App-specific password.
|
||||
* ``ICLOUD_FOLDER`` – Target folder path inside iCloud Drive, using ``/`` as
|
||||
the separator (e.g. ``Documents/Uploads``). The folder is created
|
||||
automatically if it does not exist.
|
||||
* ``ICLOUD_COOKIE_DIRECTORY`` – (Optional) Directory for persisting session
|
||||
cookies so that re-authentication is avoided between task runs. Defaults to
|
||||
``~/.pyicloud``.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import UploadTaskWithRetry
|
||||
from app.utils import log_task_progress
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_icloud_api(
|
||||
username: str,
|
||||
password: str,
|
||||
cookie_directory: str | None = None,
|
||||
):
|
||||
"""Return an authenticated ``PyiCloudService`` instance.
|
||||
|
||||
Args:
|
||||
username: Apple ID email address.
|
||||
password: App-specific password.
|
||||
cookie_directory: Optional directory for session cookies.
|
||||
|
||||
Returns:
|
||||
An authenticated ``PyiCloudService`` instance.
|
||||
|
||||
Raises:
|
||||
ImportError: If ``pyicloud`` is not installed.
|
||||
ValueError: If authentication fails or 2FA is required interactively.
|
||||
"""
|
||||
from pyicloud import PyiCloudService # noqa: S404 – unofficial third-party iCloud client
|
||||
|
||||
kwargs: dict = {}
|
||||
if cookie_directory:
|
||||
kwargs["cookie_directory"] = cookie_directory
|
||||
|
||||
api = PyiCloudService(username, password, **kwargs)
|
||||
|
||||
# If 2SA/2FA is required the user must use an app-specific password instead.
|
||||
if api.requires_2sa or api.requires_2fa:
|
||||
raise ValueError(
|
||||
"iCloud account requires two-factor authentication. "
|
||||
"Please generate an app-specific password at https://appleid.apple.com "
|
||||
"and use it as ICLOUD_PASSWORD."
|
||||
)
|
||||
|
||||
return api
|
||||
|
||||
|
||||
def _navigate_to_folder(drive_root, folder_path: str):
|
||||
"""Navigate into (or create) the folder hierarchy described by *folder_path*.
|
||||
|
||||
Args:
|
||||
drive_root: The iCloud Drive root node (``api.drive``).
|
||||
folder_path: ``/``-separated path such as ``Documents/Uploads``.
|
||||
|
||||
Returns:
|
||||
The drive node representing the target folder.
|
||||
"""
|
||||
node = drive_root
|
||||
if not folder_path:
|
||||
return node
|
||||
|
||||
parts = [p for p in folder_path.strip("/").split("/") if p]
|
||||
for part in parts:
|
||||
children = {child.name: child for child in node.dir()}
|
||||
if part in children:
|
||||
node = children[part]
|
||||
else:
|
||||
# Create the missing folder
|
||||
node = node.mkdir(part)
|
||||
return node
|
||||
|
||||
|
||||
@celery.task(base=UploadTaskWithRetry, bind=True)
|
||||
def upload_to_icloud(self, file_path: str, file_id: int = None, folder_override: str = None):
|
||||
"""Upload a file to Apple iCloud Drive.
|
||||
|
||||
Args:
|
||||
file_path: Local path to the file to upload.
|
||||
file_id: Optional ``FileRecord.id`` for progress logging.
|
||||
folder_override: If provided, overrides the default ``ICLOUD_FOLDER``
|
||||
setting for this upload.
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting iCloud Drive upload: {file_path}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"upload_to_icloud",
|
||||
"in_progress",
|
||||
f"Uploading to iCloud Drive: {os.path.basename(file_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Validate inputs
|
||||
# ------------------------------------------------------------------
|
||||
if not os.path.exists(file_path):
|
||||
error_msg = f"File not found: {file_path}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_icloud", "failure", error_msg, file_id=file_id)
|
||||
raise FileNotFoundError(error_msg)
|
||||
|
||||
if not settings.icloud_username or not settings.icloud_password:
|
||||
error_msg = "iCloud credentials are not configured (ICLOUD_USERNAME / ICLOUD_PASSWORD)"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_icloud", "failure", error_msg, file_id=file_id)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
filename = os.path.basename(file_path)
|
||||
target_folder = folder_override if folder_override is not None else (settings.icloud_folder or "")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Authenticate & upload
|
||||
# ------------------------------------------------------------------
|
||||
try:
|
||||
api = _get_icloud_api(
|
||||
settings.icloud_username,
|
||||
settings.icloud_password,
|
||||
settings.icloud_cookie_directory,
|
||||
)
|
||||
|
||||
folder_node = _navigate_to_folder(api.drive, target_folder)
|
||||
|
||||
with open(file_path, "rb") as fh:
|
||||
folder_node.upload(fh)
|
||||
|
||||
logger.info(f"[{task_id}] Successfully uploaded {filename} to iCloud Drive folder '{target_folder}'")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"upload_to_icloud",
|
||||
"success",
|
||||
f"Uploaded to iCloud Drive: {filename}",
|
||||
file_id=file_id,
|
||||
)
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file": file_path,
|
||||
"icloud_folder": target_folder or "/",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error uploading {filename} to iCloud Drive: {e}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_icloud", "failure", error_msg, file_id=file_id)
|
||||
raise RuntimeError(error_msg) from e
|
||||
@@ -571,6 +571,37 @@ def _upload_rclone(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], t
|
||||
return {"status": "Completed", "rclone_dest": dest}
|
||||
|
||||
|
||||
def _upload_icloud(file_path: str, cfg: dict[str, Any], creds: dict[str, Any], task_id: str) -> dict[str, Any]:
|
||||
"""Upload *file_path* to iCloud Drive using per-user credentials.
|
||||
|
||||
Expected *cfg* keys:
|
||||
* ``folder`` – target folder path inside iCloud Drive (e.g. ``Documents/Uploads``).
|
||||
* ``cookie_directory`` – (optional) path for session cookie persistence.
|
||||
|
||||
Expected *creds* keys:
|
||||
* ``username`` – Apple ID email address.
|
||||
* ``password`` – app-specific password.
|
||||
"""
|
||||
from app.tasks.upload_to_icloud import _get_icloud_api, _navigate_to_folder
|
||||
|
||||
username = creds.get("username") or ""
|
||||
password = creds.get("password") or ""
|
||||
folder = cfg.get("folder") or ""
|
||||
cookie_directory = cfg.get("cookie_directory") or None
|
||||
|
||||
if not username or not password:
|
||||
raise ValueError("iCloud integration is missing username or password in credentials")
|
||||
|
||||
api = _get_icloud_api(username, password, cookie_directory)
|
||||
folder_node = _navigate_to_folder(api.drive, folder)
|
||||
|
||||
with open(file_path, "rb") as fh:
|
||||
folder_node.upload(fh)
|
||||
|
||||
logger.info("[%s] iCloud Drive upload complete: folder=%s", task_id, folder or "/")
|
||||
return {"status": "Completed", "icloud_folder": folder or "/"}
|
||||
|
||||
|
||||
# Map IntegrationType → upload helper
|
||||
_UPLOAD_HANDLERS = {
|
||||
IntegrationType.DROPBOX: _upload_dropbox,
|
||||
@@ -584,6 +615,7 @@ _UPLOAD_HANDLERS = {
|
||||
IntegrationType.PAPERLESS: _upload_paperless,
|
||||
IntegrationType.EMAIL: _upload_email,
|
||||
IntegrationType.RCLONE: _upload_rclone,
|
||||
IntegrationType.ICLOUD: _upload_icloud,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -55,12 +55,12 @@ def upload_with_rclone(self, file_path: str, destination: str):
|
||||
|
||||
try:
|
||||
# Ensure the remote path exists (create folders if needed)
|
||||
mkdir_cmd = ["rclone", "mkdir", "--config", rclone_config_path, destination]
|
||||
mkdir_cmd = ["rclone", "mkdir", "--config", rclone_config_path, "--", destination]
|
||||
|
||||
subprocess.run(mkdir_cmd, check=True, capture_output=True) # noqa: S603
|
||||
|
||||
# Construct the upload command
|
||||
upload_cmd = ["rclone", "copy", "--config", rclone_config_path, file_path, destination, "--progress"]
|
||||
upload_cmd = ["rclone", "copy", "--config", rclone_config_path, "--progress", "--", file_path, destination]
|
||||
|
||||
log_task_progress(task_id, "rclone_upload", "in_progress", f"Executing rclone copy to {destination}")
|
||||
|
||||
@@ -71,7 +71,7 @@ def upload_with_rclone(self, file_path: str, destination: str):
|
||||
if result.returncode == 0:
|
||||
# Try to get a public link if possible
|
||||
try:
|
||||
link_cmd = ["rclone", "link", "--config", rclone_config_path, f"{destination}/{filename}"]
|
||||
link_cmd = ["rclone", "link", "--config", rclone_config_path, "--", f"{destination}/{filename}"]
|
||||
link_result = subprocess.run(link_cmd, capture_output=True, text=True, check=False) # noqa: S603
|
||||
public_url = link_result.stdout.strip() if link_result.returncode == 0 else None
|
||||
except (subprocess.SubprocessError, OSError) as e:
|
||||
|
||||
Reference in New Issue
Block a user