gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: fix all flake8 linter errors across app/ directory

Browse Source 
- Run Black formatter and isort on all app/ files
- Remove unused imports (F401) across multiple files
- Add # noqa: F401 for intentional re-exports in celery_worker.py,
  tasks/__init__.py, utils.py, frontend.py, views/base.py
- Fix f-strings without placeholders (F541) in azure.py, notification.py,
  check_credentials.py, upload_to_onedrive.py, settings.py
- Fix bare except (E722) in upload_to_sftp.py
- Fix block comment format (E265) in models.py
- Move imports to top of file to fix E402 in celery_app.py, celery_worker.py
- Fix line-too-long (E501) by wrapping strings in multiple files
- Remove unused variable (F841) in upload_to_nextcloud.py

Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-08 17:42:33 +00:00
parent 7827b97e06
commit d08040ac4a
73 changed files with 2200 additions and 2185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/tasks/__init__.py
+2 -2
View File
@@ -1,3 +1,3 @@
# Import tasks so they can be discovered by Celery
from app.tasks.process_document import process_document
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence
from app.tasks.process_document import process_document # noqa: F401
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence # noqa: F401
app/tasks/check_credentials.py
+80 -68
View File
@@ -1,26 +1,29 @@
import logging
from app.celery_app import celery
from app.config import settings
from app.utils.notification import notify_credential_failure
import time
import os
import json
import asyncio
import inspect
import json
import logging
import os
import time
# Import the test functions from API routes
from app.api.openai import test_openai_connection
from app.api.azure import test_azure_connection
from app.api.dropbox import test_dropbox_token
from app.api.google_drive import test_google_drive_token
from app.api.onedrive import test_onedrive_token
# Import the test functions from API routes
from app.api.openai import test_openai_connection
from app.celery_app import celery
from app.config import settings
# Import config validation utilities
from app.utils.config_validator import validate_storage_configs, get_provider_status
from app.utils.config_validator import get_provider_status, validate_storage_configs
from app.utils.notification import notify_credential_failure
# Create an enhanced mock Request object for API functions that expect it
class MockRequest:
"""Mock request object with session and other attributes needed for API functions"""
def __init__(self):
self.session = {"user": {"id": "credential_checker", "name": "System Credential Checker"}}
self.app = None
@@ -34,31 +37,35 @@ class MockRequest:
async def form(self):
return {}
logger = logging.getLogger(__name__)
# Path to store failure counts
FAILURE_STATE_FILE = os.path.join(settings.workdir, 'credential_failures.json')
FAILURE_STATE_FILE = os.path.join(settings.workdir, "credential_failures.json")
def get_failure_state():
"""Read the failure state from file"""
try:
if os.path.exists(FAILURE_STATE_FILE):
with open(FAILURE_STATE_FILE, 'r') as f:
with open(FAILURE_STATE_FILE, "r") as f:
return json.load(f)
except Exception as e:
logger.error(f"Error reading failure state file: {e}")
# Default empty state
return {}
def save_failure_state(state):
"""Save failure state to file"""
try:
with open(FAILURE_STATE_FILE, 'w') as f:
with open(FAILURE_STATE_FILE, "w") as f:
json.dump(state, f)
except Exception as e:
logger.error(f"Error saving failure state file: {e}")
# Helper function to get the inner function without the decorator
def unwrap_decorated_function(func):
"""Get the original function from a decorated function"""
@@ -66,6 +73,7 @@ def unwrap_decorated_function(func):
return unwrap_decorated_function(func.__wrapped__)
return func
# Create synchronous versions of the test functions that bypass authentication
def sync_test_openai_connection():
"""Synchronous wrapper for the OpenAI test function that bypasses auth"""
@@ -76,6 +84,7 @@ def sync_test_openai_connection():
return asyncio.run(inner_func(request))
return inner_func(request)
def sync_test_azure_connection():
"""Synchronous wrapper for the Azure test function that bypasses auth"""
inner_func = unwrap_decorated_function(test_azure_connection)
@@ -84,6 +93,7 @@ def sync_test_azure_connection():
return asyncio.run(inner_func(request))
return inner_func(request)
def sync_test_dropbox_token():
"""Synchronous wrapper for the Dropbox test function that bypasses auth"""
inner_func = unwrap_decorated_function(test_dropbox_token)
@@ -92,6 +102,7 @@ def sync_test_dropbox_token():
return asyncio.run(inner_func(request))
return inner_func(request)
def sync_test_google_drive_token():
"""Synchronous wrapper for the Google Drive test function that bypasses auth"""
inner_func = unwrap_decorated_function(test_google_drive_token)
@@ -100,6 +111,7 @@ def sync_test_google_drive_token():
return asyncio.run(inner_func(request))
return inner_func(request)
def sync_test_onedrive_token():
"""Synchronous wrapper for the OneDrive test function that bypasses auth"""
inner_func = unwrap_decorated_function(test_onedrive_token)
@@ -108,160 +120,160 @@ def sync_test_onedrive_token():
return asyncio.run(inner_func(request))
return inner_func(request)
@celery.task
def check_credentials():
"""Check all configured credentials and notify if any are invalid"""
logger.info("Starting credential check task")
# Load current failure state
failure_state = get_failure_state()
# Track failures
failures = []
# Get provider configurations from config_validator
provider_status = get_provider_status()
storage_configs = validate_storage_configs()
# Define services with their test functions and configuration status
services = [
{
"name": "OpenAI",
"name": "OpenAI",
"check_func": sync_test_openai_connection,
"configured": provider_status.get("OpenAI", {}).get("configured", False),
"config_issues": [] # OpenAI isn't in storage_configs
"config_issues": [], # OpenAI isn't in storage_configs
},
{
"name": "Azure Document Intelligence",
"name": "Azure Document Intelligence",
"check_func": sync_test_azure_connection,
"configured": provider_status.get("Azure AI", {}).get("configured", False),
"config_issues": [] # Azure isn't in storage_configs
"config_issues": [], # Azure isn't in storage_configs
},
{
"name": "Dropbox",
"name": "Dropbox",
"check_func": sync_test_dropbox_token,
"configured": provider_status.get("Dropbox", {}).get("configured", False),
"config_issues": storage_configs.get("dropbox", [])
"config_issues": storage_configs.get("dropbox", []),
},
{
"name": "Google Drive",
"name": "Google Drive",
"check_func": sync_test_google_drive_token,
"configured": provider_status.get("Google Drive", {}).get("configured", False),
"config_issues": storage_configs.get("google_drive", [])
"config_issues": storage_configs.get("google_drive", []),
},
{
"name": "OneDrive",
"name": "OneDrive",
"check_func": sync_test_onedrive_token,
"configured": provider_status.get("OneDrive", {}).get("configured", False),
"config_issues": storage_configs.get("onedrive", [])
}
"config_issues": storage_configs.get("onedrive", []),
},
]
# Check each service
results = {}
current_time = int(time.time())
for service in services:
service_name = service["name"]
logger.info(f"Checking credentials for {service_name}")
# Skip services that aren't configured
if not service["configured"]:
config_issues = service["config_issues"]
issue_msg = f"Not properly configured" + (f": {', '.join(config_issues)}" if config_issues else "")
issue_msg = "Not properly configured" + (f": {', '.join(config_issues)}" if config_issues else "")
logger.info(f"Skipping {service_name}: {issue_msg}")
results[service_name] = {
"status": "unconfigured",
"message": issue_msg
}
results[service_name] = {"status": "unconfigured", "message": issue_msg}
continue
try:
# Call the synchronized test function and get the result
result = service["check_func"]()
# All test functions return a dict with "status" field
is_valid = result.get("status") == "success"
error_message = result.get("message", "Unknown error")
# Store the result
results[service_name] = {
"status": "valid" if is_valid else "invalid",
"message": error_message
}
results[service_name] = {"status": "valid" if is_valid else "invalid", "message": error_message}
if not is_valid:
failures.append(service_name)
# Get current failure count for this service
service_state = failure_state.get(service_name, {"count": 0, "last_notified": 0})
service_state["count"] = service_state.get("count", 0) + 1
# Only notify if we haven't reached the notification threshold (3 failures)
# or if this is the first failure after a recovery
if service_state["count"] <= 3 or service_state.get("recovered", False):
notify_credential_failure(service_name, error_message)
service_state["last_notified"] = current_time
service_state["recovered"] = False
logger.warning(f"{service_name} credentials check failed ({service_state['count']} times): {error_message}")
logger.warning(
f"{service_name} credentials check failed ({service_state['count']} times): {error_message}"
)
else:
# We're in cooldown mode
logger.warning(f"{service_name} credentials check failed ({service_state['count']} times): {error_message} - notification suppressed")
logger.warning(
f"{service_name} credentials check failed ({service_state['count']} times): "
f"{error_message} - notification suppressed"
)
# Update failure state
failure_state[service_name] = service_state
else:
logger.info(f"{service_name} credentials are valid")
# Check if this was previously failing and now recovered
if service_name in failure_state and failure_state[service_name].get("count", 0) > 0:
logger.info(f"{service_name} has recovered after {failure_state[service_name]['count']} failures")
# Mark it as recovered and reset count
failure_state[service_name] = {"count": 0, "recovered": True, "last_notified": 0}
elif service_name in failure_state:
# Just make sure recovered flag is cleared if it was there
failure_state[service_name]["recovered"] = True
except Exception as e:
logger.error(f"Error checking {service_name} credentials: {e}", exc_info=True)
failures.append(service_name)
error_message = f"Exception during credential check: {str(e)}"
# Get current failure count for this service
service_state = failure_state.get(service_name, {"count": 0, "last_notified": 0})
service_state["count"] = service_state.get("count", 0) + 1
# Only notify if we haven't reached the notification threshold or if we just recovered
if service_state["count"] <= 3 or service_state.get("recovered", False):
notify_credential_failure(service_name, error_message)
service_state["last_notified"] = current_time
service_state["recovered"] = False
# Update failure state
failure_state[service_name] = service_state
# Store the error result
results[service_name] = {
"status": "error",
"message": error_message
}
results[service_name] = {"status": "error", "message": error_message}
# Save updated failure state
save_failure_state(failure_state)
# Count only services that were actually checked (configured services)
configured_services = [s for s in services if s["configured"]]
num_configured = len(configured_services)
# Summarize results
logger.info(f"Credential check completed. Configured services: {num_configured}, Valid: {num_configured - len(failures)}, Invalid: {len(failures)}")
logger.info(
f"Credential check completed. Configured services: {num_configured}, "
f"Valid: {num_configured - len(failures)}, Invalid: {len(failures)}"
)
return {
"checked": num_configured,
"unconfigured": len(services) - num_configured,
"failures": len(failures),
"results": results,
"failure_state": failure_state
"failure_state": failure_state,
}
app/tasks/embed_metadata_into_pdf.py
+49 -25
View File
@@ -1,20 +1,21 @@
#!/usr/bin/env python3
import json
import logging
import os
import shutil
import tempfile
import logging
import PyPDF2 # Replace fitz with PyPDF2
import json
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.finalize_document_storage import finalize_document_storage
# Import the shared Celery instance
from app.celery_app import celery
from app.utils import log_task_progress
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord
from app.tasks.finalize_document_storage import finalize_document_storage
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
@@ -26,6 +27,7 @@ logger = logging.getLogger(__name__)
TMP_SUBDIR = "tmp"
PROCESSED_SUBDIR = "processed"
def unique_filepath(directory, base_filename, extension=".pdf"):
"""
Returns a unique filepath in the specified directory.
@@ -41,6 +43,7 @@ def unique_filepath(directory, base_filename, extension=".pdf"):
return candidate
counter += 1
def persist_metadata(metadata, final_pdf_path):
"""
Saves the metadata dictionary to a JSON file with the same base name as the final PDF.
@@ -53,6 +56,7 @@ def persist_metadata(metadata, final_pdf_path):
json.dump(metadata, f, ensure_ascii=False, indent=2)
return json_path
@celery.task(base=BaseTaskWithRetry, bind=True)
def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, metadata: dict, file_id: int = None):
"""
@@ -70,15 +74,21 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting metadata embedding for: {local_file_path}")
log_task_progress(task_id, "embed_metadata_into_pdf", "in_progress", f"Embedding metadata into {os.path.basename(local_file_path)}", file_id=file_id)
log_task_progress(
task_id,
"embed_metadata_into_pdf",
"in_progress",
f"Embedding metadata into {os.path.basename(local_file_path)}",
file_id=file_id,
)
# Get file_id from database if not provided (fallback only, prefer passing file_id explicitly)
if file_id is None:
with SessionLocal() as db:
file_record = db.query(FileRecord).filter_by(local_filename=local_file_path).first()
if file_record:
file_id = file_record.id
# Check for file existence; if not found, try the known shared tmp directory.
if not os.path.exists(local_file_path):
alt_path = os.path.join(settings.workdir, TMP_SUBDIR, os.path.basename(local_file_path))
@@ -93,7 +103,7 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
original_file = local_file_path
# Create a temporary file with the same extension as the original
_, ext = os.path.splitext(local_file_path)
tmp_file = tempfile.NamedTemporaryFile(mode='wb', suffix=ext, prefix='processed_', delete=False)
tmp_file = tempfile.NamedTemporaryFile(mode="wb", suffix=ext, prefix="processed_", delete=False)
processed_file = tmp_file.name
tmp_file.close()
@@ -105,24 +115,26 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
log_task_progress(task_id, "modify_pdf", "in_progress", "Modifying PDF metadata", file_id=file_id)
# Open the PDF and modify metadata
with open(processed_file, 'rb') as file:
with open(processed_file, "rb") as file:
pdf_reader = PyPDF2.PdfReader(file)
pdf_writer = PyPDF2.PdfWriter()
# Copy all pages from the reader to the writer
for page in pdf_reader.pages:
pdf_writer.add_page(page)
# Set PDF metadata
pdf_writer.add_metadata({
"/Title": metadata.get("filename", "Unknown Document"),
"/Author": metadata.get("absender", "Unknown"),
"/Subject": metadata.get("document_type", "Unknown"),
"/Keywords": ", ".join(metadata.get("tags", []))
})
pdf_writer.add_metadata(
{
"/Title": metadata.get("filename", "Unknown Document"),
"/Author": metadata.get("absender", "Unknown"),
"/Subject": metadata.get("document_type", "Unknown"),
"/Keywords": ", ".join(metadata.get("tags", [])),
}
)
# Write the modified PDF
with open(processed_file, 'wb') as output_file:
with open(processed_file, "wb") as output_file:
pdf_writer.write(output_file)
logger.info(f"[{task_id}] Metadata embedded successfully in {processed_file}")
@@ -139,24 +151,36 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
final_file_path = unique_filepath(final_dir, suggested_filename, extension=".pdf")
logger.info(f"[{task_id}] Moving file to: {final_file_path}")
log_task_progress(task_id, "move_to_processed", "in_progress", f"Moving to processed: {suggested_filename}.pdf", file_id=file_id)
log_task_progress(
task_id,
"move_to_processed",
"in_progress",
f"Moving to processed: {suggested_filename}.pdf",
file_id=file_id,
)
# Move the processed file using shutil.move to handle cross-device moves.
shutil.move(processed_file, final_file_path)
# Ensure the temporary file is deleted if it still exists.
if os.path.exists(processed_file):
os.remove(processed_file)
log_task_progress(task_id, "move_to_processed", "success", f"Moved to: {os.path.basename(final_file_path)}", file_id=file_id)
log_task_progress(
task_id, "move_to_processed", "success", f"Moved to: {os.path.basename(final_file_path)}", file_id=file_id
)
# Persist the metadata into a JSON file with the same base name.
logger.info(f"[{task_id}] Persisting metadata to JSON")
log_task_progress(task_id, "save_metadata_json", "in_progress", "Saving metadata JSON", file_id=file_id)
json_path = persist_metadata(metadata, final_file_path)
logger.info(f"[{task_id}] Metadata persisted to {json_path}")
log_task_progress(task_id, "save_metadata_json", "success", f"Saved: {os.path.basename(json_path)}", file_id=file_id)
log_task_progress(
task_id, "save_metadata_json", "success", f"Saved: {os.path.basename(json_path)}", file_id=file_id
)
# Trigger the next step: final storage.
logger.info(f"[{task_id}] Queueing final storage task")
log_task_progress(task_id, "embed_metadata_into_pdf", "success", "Metadata embedded, queuing finalization", file_id=file_id)
log_task_progress(
task_id, "embed_metadata_into_pdf", "success", "Metadata embedded, queuing finalization", file_id=file_id
)
finalize_document_storage.delay(original_file, final_file_path, metadata, file_id=file_id)
# After triggering final storage, delete the original file if it is in workdir/tmp.
app/tasks/extract_metadata_with_gpt.py
+59 -50
View File
@@ -1,33 +1,32 @@
#!/usr/bin/env python3
import json
import re
import logging
import os
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
import re
import openai
# Import the shared Celery instance
from app.celery_app import celery
import openai
import logging
from app.utils import log_task_progress
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
# Initialize OpenAI client dynamically with better error handling
try:
client = openai.OpenAI(
api_key=settings.openai_api_key,
base_url=settings.openai_base_url
)
client = openai.OpenAI(api_key=settings.openai_api_key, base_url=settings.openai_base_url)
logger.info("OpenAI client initialized successfully")
except Exception as e:
logger.error(f"Failed to initialize OpenAI client: {e}")
client = None
def extract_json_from_text(text):
"""
Try to extract a JSON object from the text.
@@ -42,16 +41,19 @@ def extract_json_from_text(text):
start = text.find("{")
end = text.rfind("}")
if start != -1 and end != -1 and end > start:
return text[start:end+1]
return text[start : end + 1]
return None
@celery.task(base=BaseTaskWithRetry, bind=True)
def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: int = None):
"""Uses OpenAI to classify document metadata."""
task_id = self.request.id
logger.info(f"[{task_id}] Starting metadata extraction for: {filename}")
log_task_progress(task_id, "extract_metadata_with_gpt", "in_progress", f"Extracting metadata for {filename}", file_id=file_id)
log_task_progress(
task_id, "extract_metadata_with_gpt", "in_progress", f"Extracting metadata for {filename}", file_id=file_id
)
# Get file_id from database if not provided
if file_id is None:
tmp_dir = os.path.join(settings.workdir, "tmp")
@@ -61,38 +63,39 @@ def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: i
file_record = db.query(FileRecord).filter_by(local_filename=file_path).first()
if file_record:
file_id = file_record.id
prompt = f"""
You are a specialized document analyzer trained to extract structured metadata from documents.
Your task is to analyze the given text and return a well-structured JSON object.
Extract and return the following fields:
1. **filename**: Machine-readable filename (YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, periods, and underscores).
2. **empfaenger**: The recipient, or "Unknown" if not found.
3. **absender**: The sender, or "Unknown" if not found.
4. **correspondent**: The entity or company that issued the document (shortest possible name, e.g., "Amazon" instead of "Amazon EU SARL, German branch").
5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].
6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, Private_Korrespondenz, Sonstige_Informationen].
7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).
8. **tags**: A list of up to 4 relevant thematic keywords.
9. **language**: Detected document language (ISO 639-1 code, e.g., "de" or "en").
10. **title**: A human-readable title summarizing the document content.
11. **confidence_score**: A numeric value (0-100) indicating the confidence level of the extracted metadata.
12. **reference_number**: Extracted invoice/order/reference number if available.
13. **monetary_amounts**: A list of key monetary values detected in the document.
### Important Rules:
- **OCR Correction**: Assume the text has been corrected for OCR errors.
- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.
- **Title**: Concise, no addresses, and contains key identifying features.
- **Date Selection**: Use the most relevant date if multiple are found.
- **Output Language**: Maintain the document's original language.
Extracted text:
{cleaned_text}
Return only valid JSON with no additional commentary.
"""
prompt = (
"You are a specialized document analyzer trained to extract structured metadata from documents.\n"
"Your task is to analyze the given text and return a well-structured JSON object.\n\n"
"Extract and return the following fields:\n"
"1. **filename**: Machine-readable filename "
"(YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, periods, and underscores).\n"
"2. **empfaenger**: The recipient, or \"Unknown\" if not found.\n"
"3. **absender**: The sender, or \"Unknown\" if not found.\n"
"4. **correspondent**: The entity or company that issued the document "
"(shortest possible name, e.g., \"Amazon\" instead of \"Amazon EU SARL, German branch\").\n"
"5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, "
"Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].\n"
"6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, "
"Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, "
"Private_Korrespondenz, Sonstige_Informationen].\n"
"7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).\n"
"8. **tags**: A list of up to 4 relevant thematic keywords.\n"
"9. **language**: Detected document language (ISO 639-1 code, e.g., \"de\" or \"en\").\n"
"10. **title**: A human-readable title summarizing the document content.\n"
"11. **confidence_score**: A numeric value (0-100) indicating the confidence level "
"of the extracted metadata.\n"
"12. **reference_number**: Extracted invoice/order/reference number if available.\n"
"13. **monetary_amounts**: A list of key monetary values detected in the document.\n\n"
"### Important Rules:\n"
"- **OCR Correction**: Assume the text has been corrected for OCR errors.\n"
"- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.\n"
"- **Title**: Concise, no addresses, and contains key identifying features.\n"
"- **Date Selection**: Use the most relevant date if multiple are found.\n"
"- **Output Language**: Maintain the document's original language.\n\n"
f"Extracted text:\n{cleaned_text}\n\n"
"Return only valid JSON with no additional commentary.\n"
)
try:
logger.info(f"[{task_id}] Sending classification request for {filename}...")
@@ -101,9 +104,9 @@ Return only valid JSON with no additional commentary.
model=settings.openai_model,
messages=[
{"role": "system", "content": "You are an intelligent document classifier."},
{"role": "user", "content": prompt}
{"role": "user", "content": prompt},
],
temperature=0
temperature=0,
)
content = completion.choices[0].message.content
@@ -113,16 +116,22 @@ Return only valid JSON with no additional commentary.
json_text = extract_json_from_text(content)
if not json_text:
logger.error(f"[{task_id}] Could not find valid JSON in GPT response for {filename}.")
log_task_progress(task_id, "extract_metadata_with_gpt", "failure", "Invalid JSON in response", file_id=file_id)
log_task_progress(
task_id, "extract_metadata_with_gpt", "failure", "Invalid JSON in response", file_id=file_id
)
return {}
metadata = json.loads(json_text)
logger.info(f"[{task_id}] Extracted metadata: {metadata}")
log_task_progress(task_id, "parse_metadata", "success", f"Parsed metadata: {list(metadata.keys())}", file_id=file_id)
log_task_progress(
task_id, "parse_metadata", "success", f"Parsed metadata: {list(metadata.keys())}", file_id=file_id
)
# Trigger the next step: embedding metadata into the PDF
logger.info(f"[{task_id}] Queueing metadata embedding task")
log_task_progress(task_id, "extract_metadata_with_gpt", "success", "Metadata extracted, queuing embed task", file_id=file_id)
log_task_progress(
task_id, "extract_metadata_with_gpt", "success", "Metadata extracted, queuing embed task", file_id=file_id
)
embed_metadata_into_pdf.delay(filename, cleaned_text, metadata, file_id)
return {"s3_file": filename, "metadata": metadata}
app/tasks/finalize_document_storage.py
+27 -26
View File
@@ -2,21 +2,22 @@
import logging
import os
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
# Import the shared Celery instance
from app.celery_app import celery
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord
from app.tasks.retry_config import BaseTaskWithRetry
# Import the aggregator task and validator
from app.tasks.send_to_all import send_to_all_destinations, get_configured_services_from_validator
# Import notification utility
from app.utils.notification import notify_file_processed
from app.tasks.send_to_all import get_configured_services_from_validator, send_to_all_destinations
# Import database and logging utils from main
from app.utils import log_task_progress
from app.database import SessionLocal
from app.models import FileRecord
# Import notification utility
from app.utils.notification import notify_file_processed
logger = logging.getLogger(__name__)
@@ -30,18 +31,22 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
"""
task_id = self.request.id
logger.info(f"[{task_id}] Finalizing document storage for {processed_file}")
# 1. Update Database Status (From Main)
log_task_progress(task_id, "finalize_document_storage", "in_progress", f"Finalizing: {os.path.basename(processed_file)}", file_id=file_id)
log_task_progress(
task_id,
"finalize_document_storage",
"in_progress",
f"Finalizing: {os.path.basename(processed_file)}",
file_id=file_id,
)
# Get file_id from database if not provided (fallback logic from Main)
if file_id is None:
with SessionLocal() as db:
# Only as a last resort, try to find by exact match on local_filename
tmp_path = os.path.join(settings.workdir, "tmp", os.path.basename(original_file))
file_record = db.query(FileRecord).filter(
FileRecord.local_filename == tmp_path
).first()
file_record = db.query(FileRecord).filter(FileRecord.local_filename == tmp_path).first()
if file_record:
file_id = file_record.id
@@ -54,7 +59,7 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
for service_name, is_configured in configured_services.items():
if is_configured:
# Format service names for display
display_name = service_name.replace('_', ' ').title()
display_name = service_name.replace("_", " ").title()
configured_destinations.append(display_name)
except Exception as e:
logger.warning(f"[WARNING] Could not determine configured destinations: {e}")
@@ -63,8 +68,10 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
# 3. Queue Uploads (Merged)
# Uses Main branch signature to ensure file_id is passed, but keeps logic structure
logger.info(f"[{task_id}] Queueing uploads to all destinations")
log_task_progress(task_id, "finalize_document_storage", "success", "Queuing uploads to destinations", file_id=file_id)
log_task_progress(
task_id, "finalize_document_storage", "success", "Queuing uploads to destinations", file_id=file_id
)
# Note: send_to_all_destinations is asynchronous and queues upload tasks
# We pass 'True' (delete_after) and 'file_id' as per Main branch requirements
send_to_all_destinations.delay(processed_file, True, file_id)
@@ -76,17 +83,11 @@ def finalize_document_storage(self, original_file: str, processed_file: str, met
# Get file information
file_size = os.path.getsize(processed_file) if os.path.exists(processed_file) else 0
filename = os.path.basename(processed_file)
notify_file_processed(
filename=filename,
file_size=file_size,
metadata=metadata,
destinations=configured_destinations
filename=filename, file_size=file_size, metadata=metadata, destinations=configured_destinations
)
except Exception as e:
logger.warning(f"[WARNING] Failed to send file processed notification: {e}")
return {
"status": "Completed",
"file": processed_file
}
return {"status": "Completed", "file": processed_file}
app/tasks/imap_tasks.py
+29 -34
View File
@@ -1,16 +1,18 @@
#!/usr/bin/env python3
import os
import json
import email
import imaplib
import json
import logging
import redis
import os
import re
from datetime import datetime, timedelta, timezone
import redis
from celery import shared_task
from app.config import settings
from app.tasks.process_document import process_document # Updated import
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
from app.tasks.process_document import process_document # Updated import
logger = logging.getLogger(__name__)
@@ -18,7 +20,7 @@ logger = logging.getLogger(__name__)
redis_client = redis.StrictRedis.from_url(settings.redis_url, decode_responses=True)
LOCK_KEY = "imap_lock" # Unique key for locking
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
# Local cache file for tracking processed emails
CACHE_FILE = os.path.join(settings.workdir, "processed_mails.json")
@@ -141,20 +143,18 @@ def check_and_pull_mailbox(
)
def pull_inbox(mailbox_key, host, port, username, password, use_ssl,
delete_after_process):
def pull_inbox(mailbox_key, host, port, username, password, use_ssl, delete_after_process):
"""
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
and processes attachments while preserving the original unread status.
For Gmail:
- Attempts to select the localized All Mail folder.
- Runs an X-GM-RAW query: "in:anywhere in:unread newer_than:3d has:attachment".
For non-Gmail mailboxes, it falls back to selecting the INBOX with a SINCE/UNSEEN filter.
"""
logger.info("Connecting to %s at %s:%s (SSL=%s)",
mailbox_key, host, port, use_ssl)
logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
processed_emails = load_processed_emails()
try:
@@ -177,13 +177,11 @@ def pull_inbox(mailbox_key, host, port, username, password, use_ssl,
else:
# For non-Gmail, select INBOX and use SINCE/UNSEEN query.
mail.select("INBOX")
since_date = (datetime.now(timezone.utc) - timedelta(days=3)
).strftime("%d-%b-%Y")
status, search_data = mail.search(None, f'(SINCE {since_date} UNSEEN)')
since_date = (datetime.now(timezone.utc) - timedelta(days=3)).strftime("%d-%b-%Y")
status, search_data = mail.search(None, f"(SINCE {since_date} UNSEEN)")
if status != "OK":
logger.warning("Search failed on mailbox %s. Status=%s",
mailbox_key, status)
logger.warning("Search failed on mailbox %s. Status=%s", mailbox_key, status)
mail.close()
mail.logout()
return
@@ -194,8 +192,7 @@ def pull_inbox(mailbox_key, host, port, username, password, use_ssl,
for num in msg_numbers:
status, msg_data = mail.fetch(num, "(RFC822)")
if status != "OK":
logger.warning("Failed to fetch message %s in %s. Status=%s",
num, mailbox_key, status)
logger.warning("Failed to fetch message %s in %s. Status=%s", num, mailbox_key, status)
continue
raw_email = msg_data[0][1]
@@ -213,8 +210,7 @@ def pull_inbox(mailbox_key, host, port, username, password, use_ssl,
# For Gmail, check if the email already has the "Ingested" label.
if is_gmail_host:
if email_already_has_label(mail, num, "Ingested"):
logger.info("Skipping email %s in %s, already labeled 'Ingested'.",
msg_id, mailbox_key)
logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
continue
# Process attachments (and convert non-PDF files).
@@ -248,28 +244,28 @@ def pull_inbox(mailbox_key, host, port, username, password, use_ssl,
def fetch_attachments_and_enqueue(email_message):
"""
Extracts attachments from the email and processes only allowed file types.
Files are accepted if either:
1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR
2. They have a '.pdf' file extension (regardless of MIME type)
Allowed file types include:
- PDF: application/pdf or *.pdf extension
- Microsoft Office files:
- Word: application/msword,
- Word: application/msword,
application/vnd.openxmlformats-officedocument.wordprocessingml.document
- Excel: application/vnd.ms-excel,
- Excel: application/vnd.ms-excel,
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
- PowerPoint: application/vnd.ms-powerpoint,
- PowerPoint: application/vnd.ms-powerpoint,
application/vnd.openxmlformats-officedocument.presentationml.presentation
- Other meaningful attachments:
- Plain text: text/plain
- CSV: text/csv
- Rich Text Format: application/rtf, text/rtf
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
any other allowed file is enqueued for conversion to PDF.
Returns True if at least one allowed attachment was processed.
"""
ALLOWED_MIME_TYPES = {
@@ -285,7 +281,7 @@ def fetch_attachments_and_enqueue(email_message):
"application/rtf",
"text/rtf",
}
has_attachment = False
for part in email_message.walk():
if part.get_content_maintype() == "multipart":
@@ -296,13 +292,12 @@ def fetch_attachments_and_enqueue(email_message):
continue
# Check if it's a PDF file by extension, regardless of MIME type
is_pdf_by_extension = filename.lower().endswith('.pdf')
is_pdf_by_extension = filename.lower().endswith(".pdf")
mime_type = part.get_content_type()
# Accept file if it has an allowed MIME type OR it's a PDF by extension
if mime_type not in ALLOWED_MIME_TYPES and not is_pdf_by_extension:
logger.info("Skipping attachment %s with MIME type %s",
filename, mime_type)
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
continue
file_path = os.path.join(settings.workdir, filename)
@@ -331,7 +326,7 @@ def email_already_has_label(mail, msg_id, label="Ingested"):
# Convert msg_id to bytes if it's an integer
if isinstance(msg_id, int):
msg_id = str(msg_id).encode()
label_status, label_data = mail.fetch(msg_id, "(X-GM-LABELS)")
if label_status == "OK" and label_data and len(label_data) > 0:
raw_labels = label_data[0][1].decode("utf-8", errors="ignore")
@@ -401,7 +396,7 @@ def find_all_mail_xlist(mail):
Returns the folder name if found, otherwise None.
"""
tag = mail._new_tag().decode("ascii")
command_str = f"{tag} XLIST \"\" \"*\""
command_str = f'{tag} XLIST "" "*"'
mail.send((command_str + "\r\n").encode("utf-8"))
all_mail_folder = None
app/tasks/process_with_azure_document_intelligence.py
+24 -21
View File
@@ -1,23 +1,23 @@
import os
import logging
import os
import azure.core.exceptions
import PyPDF2
from azure.core.credentials import AzureKeyCredential
from azure.ai.documentintelligence import DocumentIntelligenceClient
from azure.ai.documentintelligence.models import AnalyzeOutputOption, AnalyzeResult
import azure.core.exceptions
from azure.core.credentials import AzureKeyCredential
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.rotate_pdf_pages import rotate_pdf_pages
from app.celery_app import celery
logger = logging.getLogger(__name__)
# Initialize Azure Document Intelligence client with error handling
try:
document_intelligence_client = DocumentIntelligenceClient(
endpoint=settings.azure_endpoint,
credential=AzureKeyCredential(settings.azure_ai_key)
endpoint=settings.azure_endpoint, credential=AzureKeyCredential(settings.azure_ai_key)
)
logger.info("Azure Document Intelligence client initialized successfully")
except (ValueError, azure.core.exceptions.ClientAuthenticationError) as e:
@@ -33,36 +33,38 @@ AZURE_DOC_INTELLIGENCE_LIMITS = {
"max_pages": 2000,
}
def get_pdf_page_count(file_path):
"""Get the number of pages in a PDF file."""
try:
with open(file_path, 'rb') as file:
with open(file_path, "rb") as file:
pdf_reader = PyPDF2.PdfReader(file)
return len(pdf_reader.pages)
except Exception as e:
logger.error(f"Error getting PDF page count: {e}")
return None
def check_page_rotation(result, filename):
"""
Checks if pages in the document are rotated and logs the rotation information.
Args:
result: The AnalyzeResult from Azure Document Intelligence API
filename: The name of the file being processed
Returns:
dict: Dictionary mapping page indices (integers) to rotation angles
"""
logger.error(f"Checking rotation for document: {filename}")
rotation_data = {}
if not hasattr(result, 'pages') or not result.pages:
if not hasattr(result, "pages") or not result.pages:
logger.error(f"No page information available for rotation check: {filename}")
return rotation_data
for i, page in enumerate(result.pages):
if hasattr(page, 'angle'):
if hasattr(page, "angle"):
rotation_angle = page.angle
if rotation_angle != 0:
logger.error(f"Page {i+1} is rotated by {rotation_angle} degrees")
@@ -72,15 +74,16 @@ def check_page_rotation(result, filename):
logger.error(f"Page {i+1} has no rotation (0 degrees)")
else:
logger.error(f"Page {i+1} rotation information not available")
return rotation_data
@celery.task(base=BaseTaskWithRetry)
def process_with_azure_document_intelligence(filename: str, file_id: int = None):
"""
Processes a PDF document using Azure Document Intelligence and overlays OCR text onto
the local temporary file (stored under <workdir>/tmp).
Steps:
0. Verify the file meets Azure Document Intelligence service limits
1. Uploads the document for OCR using Azure Document Intelligence.
@@ -88,7 +91,7 @@ def process_with_azure_document_intelligence(filename: str, file_id: int = None)
3. Saves the OCR-processed PDF locally in the same location as before.
4. Checks for page rotation and triggers page rotation if needed.
5. Triggers downstream metadata extraction.
Args:
filename: Name of the file to process
file_id: Optional file ID to pass through to subsequent tasks
@@ -101,13 +104,15 @@ def process_with_azure_document_intelligence(filename: str, file_id: int = None)
# Check file size against service limits
file_size = os.path.getsize(tmp_file_path)
if file_size > AZURE_DOC_INTELLIGENCE_LIMITS["max_file_size_bytes"]:
error_msg = f"File size ({file_size / (1024 * 1024):.2f} MB) exceeds Azure Document Intelligence limit of 500 MB"
error_msg = (
f"File size ({file_size / (1024 * 1024):.2f} MB) exceeds Azure Document Intelligence limit of 500 MB"
)
logger.error(error_msg)
return {"error": error_msg, "file": filename, "status": "Failed - Size limit exceeded"}
# For PDF files, check page count against service limits
# "Fail open" approach: only reject if we're sure it exceeds the limit
if filename.lower().endswith('.pdf'):
if filename.lower().endswith(".pdf"):
page_count = get_pdf_page_count(tmp_file_path)
if page_count is not None and page_count > AZURE_DOC_INTELLIGENCE_LIMITS["max_pages"]:
error_msg = f"PDF page count ({page_count}) exceeds Azure Document Intelligence limit of 2000 pages"
@@ -130,9 +135,7 @@ def process_with_azure_document_intelligence(filename: str, file_id: int = None)
rotation_data = check_page_rotation(result, filename)
# Retrieve the processed searchable PDF
response = document_intelligence_client.get_analyze_result_pdf(
model_id=result.model_id, result_id=operation_id
)
response = document_intelligence_client.get_analyze_result_pdf(model_id=result.model_id, result_id=operation_id)
searchable_pdf_path = tmp_file_path # Overwrite the original PDF location
with open(searchable_pdf_path, "wb") as writer:
writer.writelines(response)
app/tasks/refine_text_with_gpt.py
+15 -11
View File
@@ -1,17 +1,15 @@
#!/usr/bin/env python3
from app.config import settings
import openai
from app.tasks.retry_config import BaseTaskWithRetry
# Import the shared Celery instance
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
# Initialize OpenAI client dynamically
client = openai.OpenAI(
api_key=settings.openai_api_key,
base_url=settings.openai_base_url
)
client = openai.OpenAI(api_key=settings.openai_api_key, base_url=settings.openai_base_url)
@celery.task(base=BaseTaskWithRetry)
def refine_text_with_gpt(filename: str, raw_text: str):
@@ -19,16 +17,22 @@ def refine_text_with_gpt(filename: str, raw_text: str):
response = client.chat.completions.create(
model=settings.openai_model,
messages=[
{"role": "system", "content": "Clean and format the following text. The idea is that the text you see comes from an OCR system and your task is to eliminate OCR errors. Keep the original language when doing so."},
{"role": "user", "content": raw_text}
]
{
"role": "system",
"content": (
"Clean and format the following text. The idea is that the text you see comes from an OCR "
"system and your task is to eliminate OCR errors. Keep the original language when doing so."
),
},
{"role": "user", "content": raw_text},
],
)
cleaned_text = response.choices[0].message.content
# Trigger next task (import locally if needed to avoid circular imports)
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
extract_metadata_with_gpt.delay(filename, cleaned_text)
return {"filename": filename, "cleaned_text": cleaned_text}
app/tasks/retry_config.py
+1 -1
View File
@@ -2,8 +2,8 @@
from celery import Task
class BaseTaskWithRetry(Task):
autoretry_for = (Exception,)
retry_kwargs = {"max_retries": 3, "countdown": 10} # 3 retries, 10s delay
retry_backoff = True # Exponential backoff
app/tasks/rotate_pdf_pages.py
+43 -32
View File
@@ -1,23 +1,24 @@
import os
import logging
import PyPDF2
import math
import json
import logging
import os
import PyPDF2
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
from app.celery_app import celery
from app.config import settings
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
from app.tasks.retry_config import BaseTaskWithRetry
logger = logging.getLogger(__name__)
def determine_rotation_angle(detected_angle):
"""
Determine the optimal rotation angle based on detected angle.
Args:
detected_angle: The angle detected by Azure Document Intelligence
Returns:
int: The angle to rotate the page in PyPDF2 (must be multiple of 90 degrees)
"""
@@ -25,11 +26,11 @@ def determine_rotation_angle(detected_angle):
normalized_angle = detected_angle % 360
if normalized_angle < 0:
normalized_angle += 360
# If angle is very small (< 1 degree), don't rotate
if abs(normalized_angle) < 1 or abs(normalized_angle - 360) < 1:
return 0
# For angles close to 90, 180, or 270 degrees (±5°), round to nearest 90° increment
for target in [90, 180, 270]:
if abs(normalized_angle - target) < 5:
@@ -37,7 +38,7 @@ def determine_rotation_angle(detected_angle):
rotation_value = (360 - target) % 360
logger.info(f"Detected angle {detected_angle}° is close to {target}°, will rotate by {rotation_value}°")
return rotation_value
# For other significant angles, round to nearest 90° increment
# (PyPDF2 only supports rotations in 90-degree increments)
closest_90_multiple = round(normalized_angle / 90) * 90
@@ -46,11 +47,12 @@ def determine_rotation_angle(detected_angle):
logger.info(f"Detected angle {detected_angle}° rounded to {closest_90_multiple}°, will rotate by {rotation_value}°")
return rotation_value
@celery.task(base=BaseTaskWithRetry)
def rotate_pdf_pages(filename: str, extracted_text: str, rotation_data=None, file_id: int = None):
"""
Rotates pages in a PDF document based on detected rotation angles.
Args:
filename: The name of the file to rotate
extracted_text: The extracted text from the document
@@ -61,7 +63,7 @@ def rotate_pdf_pages(filename: str, extracted_text: str, rotation_data=None, fil
pdf_path = os.path.join(settings.workdir, "tmp", filename)
if not os.path.exists(pdf_path):
raise FileNotFoundError(f"PDF file not found: {pdf_path}")
# Skip rotation if no rotation data provided
if not rotation_data:
logger.info(f"No rotation data provided for {filename}, proceeding with metadata extraction")
@@ -80,53 +82,62 @@ def rotate_pdf_pages(filename: str, extracted_text: str, rotation_data=None, fil
logger.info(f"No significant rotations detected in {filename}, proceeding with metadata extraction")
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
return {"file": filename, "status": "no_rotation_needed"}
logger.info(f"Rotating {len(normalized_rotation_data)} pages in {filename}")
applied_rotations = {}
# Load the PDF
with open(pdf_path, 'rb') as file:
with open(pdf_path, "rb") as file:
pdf_reader = PyPDF2.PdfReader(file)
pdf_writer = PyPDF2.PdfWriter()
# Process each page
for page_idx in range(len(pdf_reader.pages)):
page = pdf_reader.pages[page_idx]
# Apply rotation if this page has rotation data
if page_idx in normalized_rotation_data and abs(normalized_rotation_data[page_idx]) > 0:
detected_angle = normalized_rotation_data[page_idx]
rotation_angle = determine_rotation_angle(detected_angle)
if rotation_angle > 0:
# PyPDF2 uses clockwise rotation in 90-degree increments
page.rotate(rotation_angle)
logger.info(f"Page {page_idx+1} rotated by {rotation_angle}° (from detected {detected_angle}°)")
logger.info(
f"Page {page_idx+1} rotated by {rotation_angle}° "
f"(from detected {detected_angle}°)"
)
applied_rotations[str(page_idx)] = rotation_angle
else:
logger.info(f"Page {page_idx+1} had detected angle {detected_angle}° but determined it doesn't need rotation")
logger.info(
f"Page {page_idx+1} had detected angle {detected_angle}° "
"but determined it doesn't need rotation"
)
pdf_writer.add_page(page)
# Save the rotated PDF
with open(pdf_path, 'wb') as output_file:
with open(pdf_path, "wb") as output_file:
pdf_writer.write(output_file)
if applied_rotations:
logger.info(f"Successfully rotated PDF: {filename} with rotations: {json.dumps(applied_rotations)}")
else:
logger.info(f"Detected rotations in {filename} but no rotations were actually applied (angles too small or not multiples of 90°)")
logger.info(
f"Detected rotations in {filename} but no rotations were actually applied "
"(angles too small or not multiples of 90°)"
)
# Continue with metadata extraction
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
return {
"file": filename,
"file": filename,
"status": "rotated" if applied_rotations else "no_rotation_needed",
"detected_rotations": rotation_data,
"applied_rotations": applied_rotations
"applied_rotations": applied_rotations,
}
except Exception as e:
logger.error(f"Error rotating PDF {filename}: {e}")
# Continue with metadata extraction despite rotation failure
app/tasks/send_to_all.py
+83 -77
View File
@@ -1,83 +1,80 @@
#!/usr/bin/env python3
import os
import logging
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_webdav import upload_to_webdav
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_email import upload_to_email
from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_s3 import upload_to_s3
from app.utils.config_validator import get_provider_status
import os
from app.celery_app import celery
from app.utils import log_task_progress
from app.config import settings
from app.database import SessionLocal
from app.models import FileRecord
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_email import upload_to_email
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_s3 import upload_to_s3
from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_webdav import upload_to_webdav
from app.utils import log_task_progress
from app.utils.config_validator import get_provider_status
logger = logging.getLogger(__name__)
def _should_upload_to_dropbox():
return (settings.dropbox_app_key and
settings.dropbox_app_secret and
settings.dropbox_refresh_token)
return settings.dropbox_app_key and settings.dropbox_app_secret and settings.dropbox_refresh_token
def _should_upload_to_nextcloud():
return (settings.nextcloud_upload_url and
settings.nextcloud_username and
settings.nextcloud_password)
return settings.nextcloud_upload_url and settings.nextcloud_username and settings.nextcloud_password
def _should_upload_to_paperless():
return (settings.paperless_ngx_api_token and
settings.paperless_host)
return settings.paperless_ngx_api_token and settings.paperless_host
def _should_upload_to_google_drive():
# Check for OAuth configuration
if getattr(settings, 'google_drive_use_oauth', False):
return (settings.google_drive_client_id and
settings.google_drive_client_secret and
settings.google_drive_refresh_token and
settings.google_drive_folder_id)
if getattr(settings, "google_drive_use_oauth", False):
return (
settings.google_drive_client_id
and settings.google_drive_client_secret
and settings.google_drive_refresh_token
and settings.google_drive_folder_id
)
# Or check for service account configuration
else:
return (settings.google_drive_credentials_json and
settings.google_drive_folder_id)
return settings.google_drive_credentials_json and settings.google_drive_folder_id
def _should_upload_to_webdav():
return (settings.webdav_url and
settings.webdav_username and
settings.webdav_password)
return settings.webdav_url and settings.webdav_username and settings.webdav_password
def _should_upload_to_ftp():
return (settings.ftp_host and
settings.ftp_username and
settings.ftp_password)
return settings.ftp_host and settings.ftp_username and settings.ftp_password
def _should_upload_to_sftp():
return (settings.sftp_host and
settings.sftp_username and
(settings.sftp_password or settings.sftp_private_key))
return settings.sftp_host and settings.sftp_username and (settings.sftp_password or settings.sftp_private_key)
def _should_upload_to_email():
return (settings.email_host and
settings.email_username and
settings.email_password and
settings.email_default_recipient)
return (
settings.email_host and settings.email_username and settings.email_password and settings.email_default_recipient
)
def _should_upload_to_onedrive():
return (settings.onedrive_client_id and
settings.onedrive_client_secret and
settings.onedrive_refresh_token)
return settings.onedrive_client_id and settings.onedrive_client_secret and settings.onedrive_refresh_token
def _should_upload_to_s3():
return (settings.s3_bucket_name and
settings.aws_access_key_id and
settings.aws_secret_access_key)
return settings.s3_bucket_name and settings.aws_access_key_id and settings.aws_secret_access_key
def get_configured_services_from_validator():
"""
@@ -86,7 +83,7 @@ def get_configured_services_from_validator():
whether they're properly configured.
"""
providers = get_provider_status()
service_map = {
"Dropbox": "dropbox",
"NextCloud": "nextcloud",
@@ -97,21 +94,22 @@ def get_configured_services_from_validator():
"SFTP Storage": "sftp",
"Email": "email",
"OneDrive": "onedrive",
"S3 Storage": "s3"
"S3 Storage": "s3",
}
result = {}
for provider_name, internal_name in service_map.items():
if provider_name in providers:
result[internal_name] = providers[provider_name].get('configured', False)
result[internal_name] = providers[provider_name].get("configured", False)
return result
@celery.task(base=BaseTaskWithRetry, bind=True)
def send_to_all_destinations(self, file_path: str, use_validator=True, file_id: int = None):
"""
Distribute a file to all configured storage destinations.
Args:
file_path: Path to the file to distribute
use_validator: Whether to use the config validator to determine enabled services
@@ -119,28 +117,36 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
file_id: Optional file ID to associate with logs
"""
task_id = self.request.id
if not os.path.exists(file_path):
logger.error(f"[{task_id}] File not found: {file_path}")
log_task_progress(task_id, "send_to_all_destinations", "failure", "File not found", file_id=file_id)
raise FileNotFoundError(f"File not found: {file_path}")
logger.info(f"[{task_id}] Sending {file_path} to all configured destinations")
log_task_progress(task_id, "send_to_all_destinations", "in_progress", f"Distributing: {os.path.basename(file_path)}", file_id=file_id)
log_task_progress(
task_id,
"send_to_all_destinations",
"in_progress",
f"Distributing: {os.path.basename(file_path)}",
file_id=file_id,
)
# Get file_id from database if not provided (fallback only, prefer passing file_id explicitly)
if file_id is None:
with SessionLocal() as db:
# Only as a last resort, try to find by basename match
# This should not be needed if file_id is passed correctly through the chain
file_record = db.query(FileRecord).filter(
FileRecord.local_filename == os.path.join(settings.workdir, "tmp", os.path.basename(file_path))
).first()
file_record = (
db.query(FileRecord)
.filter(FileRecord.local_filename == os.path.join(settings.workdir, "tmp", os.path.basename(file_path)))
.first()
)
if file_record:
file_id = file_record.id
results = {}
# Define service configurations
services = [
{
@@ -194,7 +200,7 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
"upload_func": upload_to_s3,
},
]
# Optionally get configuration status from validator
configured_services = {}
if use_validator:
@@ -204,12 +210,12 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
except Exception as e:
logger.warning(f"[{task_id}] Failed to get configuration from validator: {str(e)}")
use_validator = False
# Process each service
queued_count = 0
for service in services:
service_name = service["name"]
# Determine if service is configured
is_configured = False
if use_validator and service_name in configured_services:
@@ -222,26 +228,26 @@ def send_to_all_destinations(self, file_path: str, use_validator=True, file_id:
except Exception as e:
logger.error(f"[{task_id}] Error checking configuration for {service_name}: {str(e)}")
is_configured = False
# Queue the upload task if service is configured
if is_configured:
logger.info(f"[{task_id}] Queueing {file_path} for {service_name} upload")
log_task_progress(task_id, f"queue_{service_name}", "in_progress", f"Queueing upload to {service_name}", file_id=file_id)
log_task_progress(
task_id, f"queue_{service_name}", "in_progress", f"Queueing upload to {service_name}", file_id=file_id
)
try:
task = service["upload_func"].delay(file_path, file_id=file_id)
results[f"{service_name}_task_id"] = task.id
queued_count += 1
log_task_progress(task_id, f"queue_{service_name}", "success", f"Queued for {service_name}", file_id=file_id)
log_task_progress(
task_id, f"queue_{service_name}", "success", f"Queued for {service_name}", file_id=file_id
)
except Exception as e:
logger.error(f"[{task_id}] Failed to queue {service_name} task: {str(e)}")
results[f"{service_name}_error"] = str(e)
log_task_progress(task_id, f"queue_{service_name}", "failure", f"Failed: {str(e)}", file_id=file_id)
logger.info(f"[{task_id}] Queued {queued_count} upload tasks")
log_task_progress(task_id, "send_to_all_destinations", "success", f"Queued {queued_count} uploads", file_id=file_id)
return {
"status": "Queued",
"file_path": file_path,
"tasks": results
}
return {"status": "Queued", "file_path": file_path, "tasks": results}
app/tasks/upload_to_dropbox.py
+66 -61
View File
@@ -1,42 +1,44 @@
#!/usr/bin/env python3
import os
import logging
import requests
import os
import dropbox
import requests
from dropbox.exceptions import ApiError, AuthError
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
from app.utils import log_task_progress
from app.database import SessionLocal
from app.models import FileRecord
from app.utils.filename_utils import extract_remote_path, get_unique_filename
logger = logging.getLogger(__name__)
def _validate_dropbox_settings():
"""Validate that all required Dropbox settings are available."""
missing = []
if not hasattr(settings, 'dropbox_refresh_token') or not settings.dropbox_refresh_token:
if not hasattr(settings, "dropbox_refresh_token") or not settings.dropbox_refresh_token:
missing.append("refresh token")
if not hasattr(settings, 'dropbox_app_key') or not settings.dropbox_app_key:
if not hasattr(settings, "dropbox_app_key") or not settings.dropbox_app_key:
missing.append("app key")
if not hasattr(settings, 'dropbox_app_secret') or not settings.dropbox_app_secret:
if not hasattr(settings, "dropbox_app_secret") or not settings.dropbox_app_secret:
missing.append("app secret")
if missing:
logger.error(f"Cannot refresh Dropbox token: Missing {', '.join(missing)}")
return False
return True
def get_dropbox_access_token():
"""Refresh the Dropbox access token using the stored refresh token from ENV."""
# Check if needed settings are available
if not _validate_dropbox_settings():
return None
@@ -49,7 +51,7 @@ def get_dropbox_access_token():
"client_id": settings.dropbox_app_key,
"client_secret": settings.dropbox_app_secret,
}
response = requests.post(token_url, headers=headers, data=data, timeout=settings.http_request_timeout)
if response.status_code == 200:
@@ -59,13 +61,14 @@ def get_dropbox_access_token():
logger.error(error_msg)
raise Exception(error_msg)
def get_dropbox_client():
"""
Create and return an authenticated Dropbox client using the configured refresh token.
Create and return an authenticated Dropbox client using the configured refresh token.
Returns:
dropbox.Dropbox: Authenticated Dropbox client instance
dropbox.Dropbox: Authenticated Dropbox client instance
Raises:
ValueError: If required Dropbox configuration is missing
AuthError: If authentication with Dropbox fails
@@ -73,72 +76,80 @@ def get_dropbox_client():
app_key = settings.dropbox_app_key
app_secret = settings.dropbox_app_secret
refresh_token = settings.dropbox_refresh_token
refresh_token = settings.dropbox_refresh_token
# Validate configuration
if not app_key or not app_secret:
raise ValueError("Dropbox app key or app secret is not configured")
raise ValueError("Dropbox app key or app secret is not configured")
if not refresh_token:
raise ValueError("Dropbox refresh token is not configured")
raise ValueError("Dropbox refresh token is not configured")
# Create a Dropbox client with refresh token
try:
try:
dbx = dropbox.Dropbox(
app_key=app_key,
app_secret=app_secret,
oauth2_refresh_token=refresh_token
)
dbx = dropbox.Dropbox(app_key=app_key, app_secret=app_secret, oauth2_refresh_token=refresh_token)
# Test the connection
dbx.users_get_current_account()
logger.info("Successfully authenticated with Dropbox")
return dbx
return dbx
except AuthError as auth_error:
logger.error(f"Dropbox authentication failed: {str(auth_error)}")
raise
raise
except Exception as e:
logger.error(f"Error creating Dropbox client: {str(e)}")
raise
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_dropbox(self, file_path: str, file_id: int = None):
"""
Upload a file to Dropbox.
Upload a file to Dropbox.
Args:
file_path: Path to the file to upload
file_id: Optional file ID to associate with logs
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting Dropbox upload: {file_path}")
logger.info(f"[{task_id}] Starting Dropbox upload: {file_path}")
log_task_progress(task_id, "upload_to_dropbox", "in_progress", f"Uploading to Dropbox: {os.path.basename(file_path)}", file_id=file_id)
log_task_progress(
task_id,
"upload_to_dropbox",
"in_progress",
f"Uploading to Dropbox: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_dropbox", "failure", error_msg, file_id=file_id)
raise FileNotFoundError(error_msg)
raise FileNotFoundError(error_msg)
# Check if Dropbox is properly configured
# Check if Dropbox is properly configured
if not (hasattr(settings, 'dropbox_app_key') and settings.dropbox_app_key and
hasattr(settings, 'dropbox_app_secret') and settings.dropbox_app_secret and
if not (
hasattr(settings, "dropbox_app_key")
and settings.dropbox_app_key
and hasattr(settings, "dropbox_app_secret")
and settings.dropbox_app_secret
and hasattr(settings, "dropbox_refresh_token")
and settings.dropbox_refresh_token
):
logger.info(f"[{task_id}] Dropbox upload skipped: Missing configuration")
log_task_progress(task_id, "upload_to_dropbox", "success", "Skipped: Not configured", file_id=file_id)
return {"status": "Skipped", "reason": "Dropbox settings not configured"}
return {"status": "Skipped", "reason": "Dropbox settings not configured"}
filename = os.path.basename(file_path)
filename = os.path.basename(file_path)
try:
# Get the Dropbox client
dbx = get_dropbox_client()
dbx = get_dropbox_client()
# Calculate remote path based on local file structure
remote_base = settings.dropbox_folder or ""
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
# Function to check if file exists in Dropbox
def check_exists_in_dropbox(path):
try:
@@ -148,29 +159,29 @@ def upload_to_dropbox(self, file_path: str, file_id: int = None):
if e.error.is_path() and e.error.get_path().is_not_found():
return False
raise
raise
# Get a unique path in case of collision
remote_full_path = f"/{remote_path}" # Dropbox paths should start with /
remote_full_path = f"/{remote_path}" # Dropbox paths should start with /
remote_full_path = remote_full_path.replace('//', '/') # Clean double slashes
remote_full_path = remote_full_path.replace("//", "/") # Clean double slashes
# Check for potential file collision and get a unique name if needed
dropbox_path = get_unique_filename(remote_full_path, check_exists_in_dropbox)
dropbox_path = get_unique_filename(remote_full_path, check_exists_in_dropbox)
# Upload the file
logger.info(f"[{task_id}] Uploading {filename} to Dropbox at {dropbox_path}")
log_task_progress(task_id, "upload_file", "in_progress", f"Uploading to {dropbox_path}", file_id=file_id)
log_task_progress(task_id, "upload_file", "in_progress", f"Uploading to {dropbox_path}", file_id=file_id)
with open(file_path, "rb") as file_data:
# Use files_upload_session for large files to avoid timeouts
file_size = os.path.getsize(file_path)
if file_size > 10 * 1024 * 1024: # 10 MB threshold for chunked upload
cursor = None
chunk_size = 4 * 1024 * 1024 # 4 MB chunks
file_data.seek(0)
file_data.seek(0)
# Start upload session
session_start = dbx.files_upload_session_start(file_data.read(chunk_size))
cursor = dropbox.files.UploadSessionCursor(session_start.session_id, file_data.tell())
cursor = dropbox.files.UploadSessionCursor(session_start.session_id, file_data.tell())
# Upload chunks until we reach the end
while file_data.tell() < file_size:
if (file_size - file_data.tell()) <= chunk_size:
@@ -178,7 +189,7 @@ def upload_to_dropbox(self, file_path: str, file_id: int = None):
dbx.files_upload_session_finish(
file_data.read(chunk_size),
cursor,
cursor,
dropbox.files.CommitInfo(path=dropbox_path, mode=dropbox.files.WriteMode.overwrite),
)
else:
# More chunks to upload
@@ -187,20 +198,14 @@ def upload_to_dropbox(self, file_path: str, file_id: int = None):
else:
# Small file, direct upload
file_data.seek(0)
file_data.seek(0)
dbx.files_upload(
file_data.read(),
dropbox_path,
mode=dropbox.files.WriteMode.overwrite
)
dbx.files_upload(file_data.read(), dropbox_path, mode=dropbox.files.WriteMode.overwrite)
logger.info(f"[{task_id}] Successfully uploaded {filename} to Dropbox at {dropbox_path}")
logger.info(f"[{task_id}] Successfully uploaded {filename} to Dropbox at {dropbox_path}")
log_task_progress(task_id, "upload_to_dropbox", "success", f"Uploaded to Dropbox: {dropbox_path}", file_id=file_id)
return {
"status": "Completed",
"file_path": file_path,
"dropbox_path": dropbox_path
}
log_task_progress(
task_id, "upload_to_dropbox", "success", f"Uploaded to Dropbox: {dropbox_path}", file_id=file_id
)
return {"status": "Completed", "file_path": file_path, "dropbox_path": dropbox_path}
except AuthError:
error_msg = f"Authentication failed while uploading {filename} to Dropbox. Check token."
logger.error(f"[{task_id}] {error_msg}")
app/tasks/upload_to_email.py
+69 -55
View File
@@ -1,24 +1,26 @@
#!/usr/bin/env python3
import os
import json
import logging
import os
import smtplib
import socket
import logging
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from datetime import datetime
from email.mime.application import MIMEApplication
from email.mime.image import MIMEImage
from datetime import datetime
from pathlib import Path
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from jinja2 import Environment, FileSystemLoader, select_autoescape
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
def get_email_template(template_name="default.html"):
"""
Load email template from one of these locations in order of precedence:
@@ -30,26 +32,22 @@ def get_email_template(template_name="default.html"):
workdir_template_path = os.path.join(settings.workdir, "templates", "email")
if os.path.exists(workdir_template_path):
env = Environment(
loader=FileSystemLoader(workdir_template_path),
autoescape=select_autoescape(['html', 'xml'])
loader=FileSystemLoader(workdir_template_path), autoescape=select_autoescape(["html", "xml"])
)
env.globals['now'] = datetime.now # Add the now function to the Jinja environment
env.globals["now"] = datetime.now # Add the now function to the Jinja environment
template = env.get_template(template_name)
logger.info(f"Using custom email template from workdir: {template_name}")
return template
except Exception as e:
logger.warning(f"Failed to load custom email template: {str(e)}")
# Fallback to built-in template
try:
# Get the app directory path (where this file is)
current_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
app_template_path = os.path.join(current_dir, "templates", "email")
env = Environment(
loader=FileSystemLoader(app_template_path),
autoescape=select_autoescape(['html', 'xml'])
)
env.globals['now'] = datetime.now # Add the now function to the Jinja environment
env = Environment(loader=FileSystemLoader(app_template_path), autoescape=select_autoescape(["html", "xml"]))
env.globals["now"] = datetime.now # Add the now function to the Jinja environment
template = env.get_template(template_name)
logger.info(f"Using built-in email template: {template_name}")
return template
@@ -57,32 +55,34 @@ def get_email_template(template_name="default.html"):
logger.error(f"Failed to load built-in email template: {str(e)}")
raise ValueError(f"Could not find any valid email template: {str(e)}")
def extract_metadata_from_file(file_path):
"""
Try to extract metadata from a file using several methods:
1. Check for a .json metadata file with the same name
2. Extract metadata from PDF if it's embedded
Returns a dictionary of metadata or None if not found
"""
metadata = {}
# Check for separate metadata JSON file
metadata_path = os.path.splitext(file_path)[0] + '.json'
metadata_path = os.path.splitext(file_path)[0] + ".json"
if os.path.exists(metadata_path):
try:
with open(metadata_path, 'r', encoding='utf-8') as f:
with open(metadata_path, "r", encoding="utf-8") as f:
metadata = json.load(f)
logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
return metadata
except Exception as e:
logger.warning(f"Failed to load metadata from JSON file: {str(e)}")
# TODO: For PDF files, try to extract embedded metadata using PyPDF2
# This would require additional dependencies, so for now we'll just check for external JSON
return metadata
def attach_logo(msg):
"""Attach the DocuElevate logo to the email with proper Content-ID."""
try:
@@ -97,27 +97,28 @@ def attach_logo(msg):
# Fallback to logo in frontend/static if app/static doesn't exist
if not os.path.exists(logo_path):
logo_path = os.path.join(app_dir, "..", "frontend", "static", "logo.png")
if os.path.exists(logo_path):
with open(logo_path, 'rb') as img:
with open(logo_path, "rb") as img:
logo_data = img.read()
# Determine image MIME type based on extension
mimetype = 'image/svg+xml' if logo_path.endswith('.svg') else 'image/png'
mimetype = "image/svg+xml" if logo_path.endswith(".svg") else "image/png"
logo_attach = MIMEImage(logo_data, mimetype)
logo_attach.add_header('Content-ID', '<logo>')
logo_attach.add_header('Content-Disposition', 'inline', filename='logo.png')
logo_attach.add_header("Content-ID", "<logo>")
logo_attach.add_header("Content-Disposition", "inline", filename="logo.png")
msg.attach(logo_attach)
logger.info(f"Logo attached from {logo_path}")
return True
else:
logger.warning("Could not find logo file")
return False
except Exception as e:
logger.warning(f"Error attaching logo: {str(e)}")
return False
def _prepare_recipients(recipients):
"""Helper function to prepare email recipients list."""
if not recipients:
@@ -130,22 +131,23 @@ def _prepare_recipients(recipients):
return [recipients], None # Convert single email to list
return recipients, None
def _send_email_with_smtp(msg, filename, recipients):
"""Helper function to handle SMTP connection and sending."""
try:
# First try to resolve the hostname
socket.gethostbyname(settings.email_host)
# Connect to the SMTP server
with smtplib.SMTP(settings.email_host, settings.email_port, timeout=30) as server:
# Use TLS if specified
if settings.email_use_tls:
server.starttls()
# Login if credentials are provided
if settings.email_username and settings.email_password:
server.login(settings.email_username, settings.email_password)
# Send the email
server.send_message(msg)
@@ -160,12 +162,22 @@ def _send_email_with_smtp(msg, filename, recipients):
logger.error(error_msg)
return {"status": "Failed", "reason": error_msg, "error": str(e)}
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_email(self, file_path: str, recipients=None, subject=None, message=None, template_name="default.html", include_metadata=True, file_id: int = None):
def upload_to_email(
self,
file_path: str,
recipients=None,
subject=None,
message=None,
template_name="default.html",
include_metadata=True,
file_id: int = None,
):
"""
Sends a file via email to the specified recipients.
If recipients is None, uses the configured default email recipient.
Args:
file_path: Path to the file to send
recipients: Optional list of recipient email addresses
@@ -180,7 +192,7 @@ def upload_to_email(self, file_path: str, recipients=None, subject=None, message
log_task_progress(
task_id, "upload_to_email", "in_progress", f"Sending via email: {os.path.basename(file_path)}", file_id=file_id
)
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
logger.error(f"[{task_id}] {error_msg}")
@@ -189,17 +201,19 @@ def upload_to_email(self, file_path: str, recipients=None, subject=None, message
# Extract filename
filename = os.path.basename(file_path)
# Check if email settings are configured
if not settings.email_host:
error_msg = "Email host is not configured"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_email", "skipped", error_msg, file_id=file_id)
return {"status": "Skipped", "reason": error_msg}
# Log email configuration for debugging
logger.debug(f"[{task_id}] Email config - Host: {settings.email_host}, Port: {settings.email_port}, "
f"Username: {settings.email_username}, TLS: {settings.email_use_tls}")
logger.debug(
f"[{task_id}] Email config - Host: {settings.email_host}, Port: {settings.email_port}, "
f"Username: {settings.email_username}, TLS: {settings.email_use_tls}"
)
# Process recipients
recipients, error = _prepare_recipients(recipients)
@@ -218,21 +232,21 @@ def upload_to_email(self, file_path: str, recipients=None, subject=None, message
try:
# Create the email
msg = MIMEMultipart('related')
msg['From'] = settings.email_sender or settings.email_username
msg['To'] = ", ".join(recipients)
msg['Subject'] = subject
msg = MIMEMultipart("related")
msg["From"] = settings.email_sender or settings.email_username
msg["To"] = ", ".join(recipients)
msg["Subject"] = subject
# Create alternative part for HTML content
alt_part = MIMEMultipart('alternative')
alt_part = MIMEMultipart("alternative")
msg.attach(alt_part)
# Attach logo to the email
has_logo = attach_logo(msg)
# Load and render template
template = get_email_template(template_name)
# Context data for the template
context = {
"filename": filename,
@@ -243,38 +257,38 @@ def upload_to_email(self, file_path: str, recipients=None, subject=None, message
"metadata": metadata,
"has_metadata": bool(metadata),
"has_logo": has_logo,
"current_year": datetime.now().year
"current_year": datetime.now().year,
}
# Render HTML body
html_content = template.render(**context)
alt_part.attach(MIMEText(html_content, 'html'))
alt_part.attach(MIMEText(html_content, "html"))
# Attach the file
with open(file_path, "rb") as file:
attachment = MIMEApplication(file.read(), _subtype="pdf")
attachment.add_header('Content-Disposition', f'attachment; filename="{filename}"')
attachment.add_header("Content-Disposition", f'attachment; filename="{filename}"')
msg.attach(attachment)
# Send the email through SMTP
error_result = _send_email_with_smtp(msg, filename, recipients)
if error_result:
logger.error(f"[{task_id}] Failed to send email: {error_result.get('reason')}")
log_task_progress(task_id, "upload_to_email", "failure", error_result.get('reason'), file_id=file_id)
log_task_progress(task_id, "upload_to_email", "failure", error_result.get("reason"), file_id=file_id)
return error_result
logger.info(f"[{task_id}] Successfully sent {filename} via email to {len(recipients)} recipients")
log_task_progress(task_id, "upload_to_email", "success", f"Sent via email: {filename}", file_id=file_id)
return {
"status": "Completed",
"file": file_path,
"recipients": recipients,
"subject": subject,
"metadata_included": bool(metadata),
"logo_included": has_logo
"logo_included": has_logo,
}
except Exception as e:
error_msg = f"Failed to send {filename} via email: {str(e)}"
logger.error(f"[{task_id}] {error_msg}")
app/tasks/upload_to_ftp.py
+35 -51
View File
@@ -1,26 +1,28 @@
#!/usr/bin/env python3
import os
# Security Warning: FTP is an insecure protocol. FTPS (FTP_TLS) is strongly recommended.
# This module attempts to use FTPS by default and falls back to plaintext FTP only if configured.
import ftplib # nosec B402 - FTP usage is intentional for legacy server support
import logging
import os
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
import logging
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_ftp(self, file_path: str, file_id: int = None):
"""
Uploads a file to an FTP server in the configured folder.
Security Note: This function prefers FTPS (FTP with TLS) for secure connections.
Plaintext FTP is only used if FTPS fails and ftp_allow_plaintext=True (default).
For security-critical environments, set ftp_allow_plaintext=False and ftp_use_tls=True.
Args:
file_path: Path to the file to upload
file_id: Optional file ID to associate with logs
@@ -49,24 +51,18 @@ def upload_to_ftp(self, file_path: str, file_id: int = None):
try:
# First attempt FTPS (FTP with TLS)
use_tls = getattr(settings, 'ftp_use_tls', True) # Default to try TLS
allow_plaintext = getattr(settings, 'ftp_allow_plaintext', True) # Default to allow plaintext fallback
use_tls = getattr(settings, "ftp_use_tls", True) # Default to try TLS
allow_plaintext = getattr(settings, "ftp_allow_plaintext", True) # Default to allow plaintext fallback
if use_tls:
try:
logger.info(f"Attempting FTPS connection to {settings.ftp_host}")
ftp = ftplib.FTP_TLS()
ftp.connect(
host=settings.ftp_host,
port=settings.ftp_port or 21
)
ftp.connect(host=settings.ftp_host, port=settings.ftp_port or 21)
# Login with credentials
ftp.login(
user=settings.ftp_username,
passwd=settings.ftp_password
)
ftp.login(user=settings.ftp_username, passwd=settings.ftp_password)
# Enable data protection - encrypt the data channel
ftp.prot_p()
logger.info("Successfully established FTPS connection with TLS")
@@ -79,53 +75,41 @@ def upload_to_ftp(self, file_path: str, file_id: int = None):
logger.warning(f"FTPS connection failed, falling back to regular FTP: {str(e)}")
# Fall back to regular FTP - only if explicitly allowed by configuration
ftp = ftplib.FTP() # nosec B321 - Fallback to FTP intentional when configured
ftp.connect(
host=settings.ftp_host,
port=settings.ftp_port or 21
)
ftp.connect(host=settings.ftp_host, port=settings.ftp_port or 21)
# Login with credentials
ftp.login(
user=settings.ftp_username,
passwd=settings.ftp_password
)
ftp.login(user=settings.ftp_username, passwd=settings.ftp_password)
else:
# Check if plaintext is allowed when TLS is explicitly disabled
if not allow_plaintext:
error_msg = "Plaintext FTP is forbidden by configuration"
logger.error(error_msg)
raise Exception(error_msg)
# Directly use regular FTP if TLS is explicitly disabled
logger.warning("Using plaintext FTP - connection is NOT encrypted!")
ftp = ftplib.FTP() # nosec B321 - Plaintext FTP intentional when explicitly configured
ftp.connect(
host=settings.ftp_host,
port=settings.ftp_port or 21
)
ftp.connect(host=settings.ftp_host, port=settings.ftp_port or 21)
# Login with credentials
ftp.login(
user=settings.ftp_username,
passwd=settings.ftp_password
)
ftp.login(user=settings.ftp_username, passwd=settings.ftp_password)
# Change to target directory if specified
if settings.ftp_folder:
try:
# Try to navigate to the directory, create if it doesn't exist
ftp_folder = settings.ftp_folder
# Remove leading slash if present
if ftp_folder.startswith('/'):
if ftp_folder.startswith("/"):
ftp_folder = ftp_folder[1:]
# Try to change to the directory
try:
ftp.cwd(ftp_folder)
except ftplib.error_perm:
# Create directory structure if it doesn't exist
folders = ftp_folder.split('/')
current_dir = ''
folders = ftp_folder.split("/")
current_dir = ""
for folder in folders:
if folder:
current_dir += f"/{folder}"
@@ -138,24 +122,24 @@ def upload_to_ftp(self, file_path: str, file_id: int = None):
error_msg = f"Failed to change/create directory on FTP server: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
# Upload the file
with open(file_path, 'rb') as file_data:
ftp.storbinary(f'STOR {filename}', file_data)
with open(file_path, "rb") as file_data:
ftp.storbinary(f"STOR {filename}", file_data)
# Close FTP connection
ftp.quit()
logger.info(f"[{task_id}] Successfully uploaded {filename} to FTP server at {settings.ftp_host}")
log_task_progress(task_id, "upload_to_ftp", "success", f"Uploaded to FTP: {filename}", file_id=file_id)
return {
"status": "Completed",
"file": file_path,
"status": "Completed",
"file": file_path,
"ftp_host": settings.ftp_host,
"ftp_path": f"{settings.ftp_folder}/{filename}" if settings.ftp_folder else filename,
"used_tls": isinstance(ftp, ftplib.FTP_TLS)
"used_tls": isinstance(ftp, ftplib.FTP_TLS),
}
except Exception as e:
error_msg = f"Failed to upload {filename} to FTP server: {str(e)}"
logger.error(f"[{task_id}] {error_msg}")
app/tasks/upload_to_google_drive.py
+85 -77
View File
@@ -2,24 +2,25 @@
app/tasks/upload_to_google_drive.py
"""
import os
import json
import logging
import os
from google.auth.exceptions import RefreshError
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials as OAuthCredentials
from google.oauth2.service_account import Credentials
from googleapiclient.discovery import build
from googleapiclient.http import MediaFileUpload
from google.oauth2.service_account import Credentials
from google.oauth2.credentials import Credentials as OAuthCredentials
from google_auth_oauthlib.flow import Flow
from google.auth.transport.requests import Request
from google.auth.exceptions import RefreshError
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
def get_drive_service_oauth():
"""
Get Google Drive service using OAuth credentials.
@@ -27,12 +28,14 @@ def get_drive_service_oauth():
"""
try:
# Check for required OAuth settings
if not (settings.google_drive_client_id and
settings.google_drive_client_secret and
settings.google_drive_refresh_token):
if not (
settings.google_drive_client_id
and settings.google_drive_client_secret
and settings.google_drive_refresh_token
):
logger.error("Google Drive OAuth credentials not fully configured")
return None
# Create credentials object from refresh token
credentials = OAuthCredentials(
None, # No access token initially, will be refreshed
@@ -41,16 +44,16 @@ def get_drive_service_oauth():
client_id=settings.google_drive_client_id,
client_secret=settings.google_drive_client_secret,
# Use only drive.file scope
scopes=['https://www.googleapis.com/auth/drive.file']
scopes=["https://www.googleapis.com/auth/drive.file"],
)
# Refresh the access token
credentials.refresh(Request())
# Build and return the service
service = build('drive', 'v3', credentials=credentials)
service = build("drive", "v3", credentials=credentials)
return service
except RefreshError as e:
logger.error(f"Failed to refresh Google Drive token: {str(e)}")
raise
@@ -58,6 +61,7 @@ def get_drive_service_oauth():
logger.error(f"Failed to authenticate with Google Drive OAuth: {str(e)}")
return None
def get_google_drive_service():
"""
Authenticate with Google Drive API using service account credentials
@@ -65,52 +69,53 @@ def get_google_drive_service():
"""
try:
# Check if we should use OAuth instead of service account
if getattr(settings, 'google_drive_use_oauth', False):
if getattr(settings, "google_drive_use_oauth", False):
return get_drive_service_oauth()
# Load service account credentials from settings
if not settings.google_drive_credentials_json:
logger.error("Google Drive credentials not configured")
return None
credentials_dict = json.loads(settings.google_drive_credentials_json)
credentials = Credentials.from_service_account_info(
credentials_dict,
scopes=['https://www.googleapis.com/auth/drive']
credentials_dict, scopes=["https://www.googleapis.com/auth/drive"]
)
# Delegate to user if specified
if settings.google_drive_delegate_to:
credentials = credentials.with_subject(settings.google_drive_delegate_to)
# Build and return the service
service = build('drive', 'v3', credentials=credentials)
service = build("drive", "v3", credentials=credentials)
return service
except Exception as e:
logger.error(f"Failed to authenticate with Google Drive: {str(e)}")
return None
def extract_metadata_from_file(file_path):
"""
Try to extract metadata from a file using several methods:
1. Check for a .json metadata file with the same name
Returns a dictionary of metadata or empty dict if not found
"""
# Check for separate metadata JSON file
metadata_path = os.path.splitext(file_path)[0] + '.json'
metadata_path = os.path.splitext(file_path)[0] + ".json"
if os.path.exists(metadata_path):
try:
with open(metadata_path, 'r', encoding='utf-8') as f:
with open(metadata_path, "r", encoding="utf-8") as f:
metadata = json.load(f)
logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
return metadata
except Exception as e:
logger.warning(f"Failed to load metadata from JSON file: {str(e)}")
return {}
def truncate_property_value(key, value, max_bytes=100):
"""
Truncate a property value to ensure the key+value stays under the byte limit.
@@ -119,35 +124,36 @@ def truncate_property_value(key, value, max_bytes=100):
"""
# Convert to string if not already
str_value = str(value)
# Calculate current size of key and value in bytes
key_bytes = len(key.encode('utf-8'))
value_bytes = len(str_value.encode('utf-8'))
key_bytes = len(key.encode("utf-8"))
value_bytes = len(str_value.encode("utf-8"))
total_bytes = key_bytes + value_bytes
# If under limit, return original value
if total_bytes <= max_bytes:
return str_value
# Calculate how many bytes we need to trim from value
# Leave a small buffer to be safe
bytes_to_trim = total_bytes - max_bytes + 4
# Iteratively truncate the string until it's under the byte limit
while len(str_value.encode('utf-8')) > value_bytes - bytes_to_trim:
while len(str_value.encode("utf-8")) > value_bytes - bytes_to_trim:
str_value = str_value[:-1]
# Add ellipsis to indicate truncation
if str_value != str(value):
str_value = str_value[:-3] + "..."
return str_value
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id: int = None):
"""
Uploads a file to Google Drive in the configured folder with optional metadata.
Args:
file_path: Path to the file to upload
include_metadata: Whether to include metadata in the upload
@@ -156,7 +162,11 @@ def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id:
task_id = self.request.id
logger.info(f"[{task_id}] Starting Google Drive upload: {file_path}")
log_task_progress(
task_id, "upload_to_google_drive", "in_progress", f"Uploading to Google Drive: {os.path.basename(file_path)}", file_id=file_id
task_id,
"upload_to_google_drive",
"in_progress",
f"Uploading to Google Drive: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
@@ -184,18 +194,18 @@ def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id:
# Prepare the file metadata
file_metadata = {
'name': filename,
"name": filename,
}
# If folder ID is specified, set parent folder
if settings.google_drive_folder_id:
file_metadata['parents'] = [settings.google_drive_folder_id]
file_metadata["parents"] = [settings.google_drive_folder_id]
# Add custom properties if metadata exists
if metadata:
# Google Drive properties must be strings and can't be nested objects
file_metadata['properties'] = {}
file_metadata["properties"] = {}
# Only add a few important top-level metadata fields as properties
# Skip nested objects and long values to avoid the 124-byte limit
safe_properties = {}
@@ -203,61 +213,59 @@ def upload_to_google_drive(self, file_path: str, include_metadata=True, file_id:
# Skip nested structures completely - they'll be in the description
if isinstance(value, (dict, list)):
continue
# Try to add simple values with truncation if needed
try:
truncated_value = truncate_property_value(key, value)
safe_properties[key] = truncated_value
except Exception as e:
logger.warning(f"Skipping metadata property {key}: {str(e)}")
# Only use the safe properties
file_metadata['properties'] = safe_properties
file_metadata["properties"] = safe_properties
# Add minimal appProperties
file_metadata['appProperties'] = {
'docuelevate': 'true'
}
file_metadata["appProperties"] = {"docuelevate": "true"}
# Add metadata to file description for better visibility in Google Drive UI
# Description has much higher size limits than properties
formatted_json = json.dumps(metadata, indent=2)
file_metadata['description'] = f"Document Metadata:\n\n```json\n{formatted_json}\n```"
file_metadata["description"] = f"Document Metadata:\n\n```json\n{formatted_json}\n```"
logger.debug(f"Adding metadata to Google Drive file: {json.dumps(file_metadata['properties'])}")
# Upload file with metadata
media = MediaFileUpload(
file_path,
mimetype='application/pdf',
resumable=True
media = MediaFileUpload(file_path, mimetype="application/pdf", resumable=True)
file = (
service.files()
.create(
body=file_metadata, media_body=media, fields="id,name,webViewLink,properties,appProperties,description"
)
.execute()
)
file = service.files().create(
body=file_metadata,
media_body=media,
fields='id,name,webViewLink,properties,appProperties,description'
).execute()
# Log success details
google_drive_file_id = file.get('id')
web_view_link = file.get('webViewLink')
google_drive_file_id = file.get("id")
web_view_link = file.get("webViewLink")
logger.info(f"[{task_id}] Successfully uploaded {filename} to Google Drive with ID: {google_drive_file_id}")
logger.info(f"[{task_id}] File accessible at: {web_view_link}")
log_task_progress(task_id, "upload_to_google_drive", "success", f"Uploaded to Google Drive: {filename}", file_id=file_id)
log_task_progress(
task_id, "upload_to_google_drive", "success", f"Uploaded to Google Drive: {filename}", file_id=file_id
)
result = {
"status": "Completed",
"status": "Completed",
"file_path": file_path,
"google_drive_file_id": google_drive_file_id,
"google_drive_web_link": web_view_link
"google_drive_web_link": web_view_link,
}
# Add metadata info to result if included
if metadata:
result["metadata_included"] = True
return result
except Exception as e:
app/tasks/upload_to_nextcloud.py
+62 -52
View File
@@ -1,143 +1,153 @@
#!/usr/bin/env python3
import os
import logging
import os
import requests
from requests.auth import HTTPBasicAuth
from app.config import settings
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
from app.utils import log_task_progress
from app.database import SessionLocal
from app.models import FileRecord
from app.utils.filename_utils import extract_remote_path, get_unique_filename
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_nextcloud(self, file_path: str, file_id: int = None):
"""
Upload a file to Nextcloud WebDAV.
Args:
file_path: Path to the file to upload
file_id: Optional file ID to associate with logs
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting Nextcloud upload: {file_path}")
log_task_progress(task_id, "upload_to_nextcloud", "in_progress", f"Uploading to Nextcloud: {os.path.basename(file_path)}", file_id=file_id)
log_task_progress(
task_id,
"upload_to_nextcloud",
"in_progress",
f"Uploading to Nextcloud: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
raise FileNotFoundError(error_msg)
# For Nextcloud, we need to check for 'nextcloud_upload_url' instead of 'nextcloud_url'
# This is what's shown in your env view
if not (getattr(settings, 'nextcloud_upload_url', None) and
getattr(settings, 'nextcloud_username', None) and
getattr(settings, 'nextcloud_password', None)):
if not (
getattr(settings, "nextcloud_upload_url", None)
and getattr(settings, "nextcloud_username", None)
and getattr(settings, "nextcloud_password", None)
):
logger.info(f"[{task_id}] Nextcloud upload skipped: Missing configuration")
log_task_progress(task_id, "upload_to_nextcloud", "success", "Skipped: Not configured", file_id=file_id)
return {"status": "Skipped", "reason": "Nextcloud settings not configured"}
filename = os.path.basename(file_path)
sanitized_filename = sanitize_filename(filename)
try:
# Prepare WebDAV URL - use nextcloud_upload_url instead of nextcloud_url
webdav_url = settings.nextcloud_upload_url
if not webdav_url.endswith('/'):
webdav_url += '/'
if not webdav_url.endswith("/"):
webdav_url += "/"
# Calculate remote path based on local file structure
remote_base = getattr(settings, 'nextcloud_folder', '') or ""
remote_base = getattr(settings, "nextcloud_folder", "") or ""
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
full_url = f"{webdav_url}/{remote_path}"
# Remove any double slashes (except in http://)
full_url = full_url.replace('://', '$PLACEHOLDER$')
while '//' in full_url:
full_url = full_url.replace('//', '/')
full_url = full_url.replace('$PLACEHOLDER$', '://')
full_url = full_url.replace("://", "$PLACEHOLDER$")
while "//" in full_url:
full_url = full_url.replace("//", "/")
full_url = full_url.replace("$PLACEHOLDER$", "://")
# Function to check if file exists in Nextcloud
def check_exists_in_nextcloud(path):
check_url = f"{webdav_url}{os.path.dirname(path)}"
try:
response = requests.request(
'PROPFIND',
"PROPFIND",
check_url,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
headers={'Depth': '1'},
timeout=10
headers={"Depth": "1"},
timeout=10,
)
return path in response.text
except Exception:
# If we can't check, assume it doesn't exist
return False
# Check for potential file collision and get a unique name if needed
remote_path = get_unique_filename(remote_path, check_exists_in_nextcloud)
full_url = f"{webdav_url}/{remote_path}"
# Fix double slashes again
full_url = full_url.replace('://', '$PLACEHOLDER$')
while '//' in full_url:
full_url = full_url.replace('//', '/')
full_url = full_url.replace('$PLACEHOLDER$', '://')
full_url = full_url.replace("://", "$PLACEHOLDER$")
while "//" in full_url:
full_url = full_url.replace("//", "/")
full_url = full_url.replace("$PLACEHOLDER$", "://")
# Create necessary parent folders
parent_dirs = os.path.dirname(remote_path)
if parent_dirs:
current_path = ""
for folder in parent_dirs.split('/'):
for folder in parent_dirs.split("/"):
if not folder:
continue
current_path += f"{folder}/"
mkdir_url = f"{webdav_url}/{current_path}"
# Fix double slashes
mkdir_url = mkdir_url.replace('://', '$PLACEHOLDER$')
while '//' in mkdir_url:
mkdir_url = mkdir_url.replace('//', '/')
mkdir_url = mkdir_url.replace('$PLACEHOLDER$', '://')
mkdir_url = mkdir_url.replace("://", "$PLACEHOLDER$")
while "//" in mkdir_url:
mkdir_url = mkdir_url.replace("//", "/")
mkdir_url = mkdir_url.replace("$PLACEHOLDER$", "://")
requests.request(
'MKCOL',
"MKCOL",
mkdir_url,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
timeout=10
timeout=10,
)
# Upload the file
logger.info(f"[{task_id}] Uploading {filename} to Nextcloud at {full_url}")
log_task_progress(task_id, "upload_file", "in_progress", f"Uploading to {remote_path}", file_id=file_id)
with open(file_path, 'rb') as file_data:
with open(file_path, "rb") as file_data:
response = requests.put(
full_url,
data=file_data,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
headers={'Content-Type': 'application/octet-stream'},
timeout=settings.http_request_timeout # Use configured timeout for large files
headers={"Content-Type": "application/octet-stream"},
timeout=settings.http_request_timeout, # Use configured timeout for large files
)
if response.status_code in (201, 204): # Created or No Content
logger.info(f"[{task_id}] Successfully uploaded {filename} to Nextcloud at {remote_path}")
log_task_progress(task_id, "upload_to_nextcloud", "success", f"Uploaded to Nextcloud: {remote_path}", file_id=file_id)
log_task_progress(
task_id, "upload_to_nextcloud", "success", f"Uploaded to Nextcloud: {remote_path}", file_id=file_id
)
return {
"status": "Completed",
"file_path": file_path,
"nextcloud_path": remote_path,
"response_code": response.status_code
"response_code": response.status_code,
}
else:
error_msg = f"Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_nextcloud", "failure", error_msg, file_id=file_id)
raise Exception(error_msg)
except Exception as e:
error_msg = f"Failed to upload {filename} to Nextcloud: {str(e)}"
logger.error(f"[{task_id}] {error_msg}")
app/tasks/upload_to_onedrive.py
+70 -78
View File
@@ -1,18 +1,21 @@
#!/usr/bin/env python3
import logging
import os
import time
import logging
import requests
import msal
import urllib.parse
import msal
import requests
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
def get_onedrive_token():
"""
Get an access token for Microsoft Graph API using the appropriate flow.
@@ -22,94 +25,92 @@ def get_onedrive_token():
# Check for required settings
if not settings.onedrive_client_id or not settings.onedrive_client_secret:
raise ValueError("OneDrive client ID and client secret must be configured")
# Log more details about the configuration
tenant = settings.onedrive_tenant_id or "common"
logger.info(f"Using OneDrive tenant: {tenant}")
# Define scopes consistently
scopes = ["https://graph.microsoft.com/.default"]
# Use refresh token flow (works for both personal and org accounts)
if settings.onedrive_refresh_token:
# Use MSAL's ConfidentialClientApplication instead of PublicClientApplication
app = msal.ConfidentialClientApplication(
client_id=settings.onedrive_client_id,
client_credential=settings.onedrive_client_secret,
authority=f"https://login.microsoftonline.com/{tenant}"
authority=f"https://login.microsoftonline.com/{tenant}",
)
# Request new token using refresh token
logger.info("Attempting to acquire token using refresh token")
token_response = app.acquire_token_by_refresh_token(
refresh_token=settings.onedrive_refresh_token,
scopes=scopes
refresh_token=settings.onedrive_refresh_token, scopes=scopes
)
if "access_token" not in token_response:
error = token_response.get("error", "")
error_desc = token_response.get("error_description", "Unknown error")
# Log more details about the error
logger.error(f"Failed to get access token using refresh token")
logger.error("Failed to get access token using refresh token")
logger.error(f"Error code: {error}")
logger.error(f"Error description: {error_desc}")
if error == "invalid_grant":
logger.error("The refresh token appears to be expired or revoked")
logger.error("A new authorization flow is required to obtain a fresh token")
raise ValueError(f"Failed to get access token: {error} - {error_desc}")
# Check if we received a new refresh token and update it
if "refresh_token" in token_response:
new_refresh_token = token_response["refresh_token"]
logger.info("Received new refresh token from Microsoft")
# Update the refresh token in memory
settings.onedrive_refresh_token = new_refresh_token
logger.info("Updated refresh token in memory")
return token_response["access_token"]
# No refresh token - try client credentials (only works for org accounts)
elif settings.onedrive_tenant_id and settings.onedrive_tenant_id != "common":
authority = f"https://login.microsoftonline.com/{settings.onedrive_tenant_id}"
app = msal.ConfidentialClientApplication(
client_id=settings.onedrive_client_id,
client_credential=settings.onedrive_client_secret,
authority=authority
authority=authority,
)
# Acquire token for application
token_response = app.acquire_token_for_client(
scopes=scopes
)
token_response = app.acquire_token_for_client(scopes=scopes)
if "access_token" not in token_response:
error = token_response.get("error", "")
error_desc = token_response.get("error_description", "Unknown error")
raise ValueError(f"Failed to get access token: {error} - {error_desc}")
return token_response["access_token"]
else:
raise ValueError("For personal Microsoft accounts, ONEDRIVE_REFRESH_TOKEN must be configured")
def create_upload_session(filename, folder_path, access_token):
"""Creates an upload session for large files in Microsoft Graph API."""
# Construct the API endpoint
base_url = "https://graph.microsoft.com/v1.0/me/drive"
# Format the folder path correctly and properly encode for URL
if folder_path:
# Remove leading/trailing slashes
folder_path = folder_path.strip('/')
folder_path = folder_path.strip("/")
# URL encode the path components separately
path_components = folder_path.split('/')
encoded_path = '/'.join(urllib.parse.quote(component) for component in path_components)
path_components = folder_path.split("/")
encoded_path = "/".join(urllib.parse.quote(component) for component in path_components)
# Also encode the filename
encoded_filename = urllib.parse.quote(filename)
item_path = f"/root:/{encoded_path}/{encoded_filename}:/createUploadSession"
@@ -117,25 +118,18 @@ def create_upload_session(filename, folder_path, access_token):
# Just encode the filename
encoded_filename = urllib.parse.quote(filename)
item_path = f"/root:/{encoded_filename}:/createUploadSession"
url = f"{base_url}{item_path}"
# Add required request body (can be empty JSON object)
request_body = {
"item": {
"@microsoft.graph.conflictBehavior": "replace"
}
}
headers = {
"Authorization": f"Bearer {access_token}",
"Content-Type": "application/json"
}
request_body = {"item": {"@microsoft.graph.conflictBehavior": "replace"}}
headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"}
logger.info(f"Creating upload session for {filename} at path {folder_path}")
response = requests.post(url, headers=headers, json=request_body, timeout=settings.http_request_timeout)
if response.status_code == 200:
upload_url = response.json().get("uploadUrl")
logger.info(f"Upload session created successfully for {filename}")
@@ -148,6 +142,7 @@ def create_upload_session(filename, folder_path, access_token):
logger.error(f"Request body: {request_body}")
raise Exception(error_msg)
def upload_large_file(file_path, upload_url):
"""
Upload a large file to OneDrive using the upload session URL.
@@ -155,45 +150,39 @@ def upload_large_file(file_path, upload_url):
"""
# Get file size
file_size = os.path.getsize(file_path)
# Define chunk size (10 MB)
chunk_size = 10 * 1024 * 1024
# Open and read file in chunks
with open(file_path, 'rb') as f:
with open(file_path, "rb") as f:
# Process file in chunks
chunk_number = 0
while True:
chunk = f.read(chunk_size)
if not chunk:
break
# Get the position in the file
chunk_start = chunk_number * chunk_size
chunk_end = chunk_start + len(chunk) - 1
# Prepare content range header
content_range = f"bytes {chunk_start}-{chunk_end}/{file_size}"
# Upload chunk
headers = {
"Content-Length": str(len(chunk)),
"Content-Range": content_range
}
headers = {"Content-Length": str(len(chunk)), "Content-Range": content_range}
# Try to upload chunk with retries
max_retries = 3
retry_delay = 2 # seconds
for attempt in range(max_retries):
try:
response = requests.put(
upload_url,
headers=headers,
data=chunk,
timeout=settings.http_request_timeout
upload_url, headers=headers, data=chunk, timeout=settings.http_request_timeout
)
# Check if successful
if response.status_code in (201, 202):
# 201 = Created (final chunk), 202 = Accepted (more chunks coming)
@@ -206,22 +195,25 @@ def upload_large_file(file_path, upload_url):
logger.warning(f"Chunk upload error (attempt {attempt+1}): {str(e)}")
if attempt < max_retries - 1:
time.sleep(retry_delay * (attempt + 1))
if response.status_code not in (201, 202):
raise Exception(f"Failed to upload chunk after {max_retries} attempts: {response.status_code} - {response.text}")
raise Exception(
f"Failed to upload chunk after {max_retries} attempts: {response.status_code} - {response.text}"
)
# Move to next chunk
chunk_number += 1
# If we get here, all chunks were uploaded successfully
# The last response should contain the file metadata
return response.json()
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_onedrive(self, file_path: str, file_id: int = None):
"""
Uploads a file to OneDrive in the configured folder.
Args:
file_path: Path to the file to upload
file_id: Optional file ID to associate with logs
@@ -235,7 +227,7 @@ def upload_to_onedrive(self, file_path: str, file_id: int = None):
f"Uploading to OneDrive: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
logger.error(f"[{task_id}] {error_msg}")
@@ -255,13 +247,13 @@ def upload_to_onedrive(self, file_path: str, file_id: int = None):
try:
# Get access token
access_token = get_onedrive_token()
# Create upload session
upload_url = create_upload_session(filename, settings.onedrive_folder_path, access_token)
# Upload the file
result = upload_large_file(file_path, upload_url)
# Log success
web_url = result.get("webUrl", "Not available")
logger.info(f"[{task_id}] Successfully uploaded {filename} to OneDrive at path {settings.onedrive_folder_path}")
@@ -269,14 +261,14 @@ def upload_to_onedrive(self, file_path: str, file_id: int = None):
log_task_progress(
task_id, "upload_to_onedrive", "success", f"Uploaded to OneDrive: {filename}", file_id=file_id
)
return {
"status": "Completed",
"file_path": file_path,
"onedrive_path": f"{settings.onedrive_folder_path}/{filename}",
"web_url": web_url
"web_url": web_url,
}
except Exception as e:
error_msg = f"Failed to upload {filename} to OneDrive: {str(e)}"
logger.error(f"[{task_id}] {error_msg}")
app/tasks/upload_to_paperless.py
+34 -28
View File
@@ -1,29 +1,26 @@
#!/usr/bin/env python3
import os
import json
import time
import requests
import logging
from typing import Dict, Any
import os
import time
import requests
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
from app.database import SessionLocal
from app.models import FileRecord
logger = logging.getLogger(__name__)
POLL_MAX_ATTEMPTS = 10
POLL_INTERVAL_SEC = 3
def _get_headers():
"""Returns HTTP headers for Paperless-ngx API calls."""
return {
"Authorization": f"Token {settings.paperless_ngx_api_token}"
}
return {"Authorization": f"Token {settings.paperless_ngx_api_token}"}
def _paperless_api_url(path: str) -> str:
"""
@@ -35,6 +32,7 @@ def _paperless_api_url(path: str) -> str:
path = "/" + path
return f"{host}{path}"
def poll_task_for_document_id(task_id: str) -> int:
"""
Polls /api/tasks/?task_id=<uuid> until we get status=SUCCESS or FAILURE,
@@ -49,14 +47,13 @@ def poll_task_for_document_id(task_id: str) -> int:
while attempts < POLL_MAX_ATTEMPTS:
try:
resp = requests.get(url, headers=_get_headers(), params={"task_id": task_id}, timeout=settings.http_request_timeout)
resp = requests.get(
url, headers=_get_headers(), params={"task_id": task_id}, timeout=settings.http_request_timeout
)
resp.raise_for_status()
tasks_data = resp.json()
except requests.exceptions.RequestException as exc:
logger.warning(
"Failed to poll for task_id='%s'. Attempt=%d Error=%s",
task_id, attempts + 1, exc
)
logger.warning("Failed to poll for task_id='%s'. Attempt=%d Error=%s", task_id, attempts + 1, exc)
time.sleep(POLL_INTERVAL_SEC)
attempts += 1
continue
@@ -71,31 +68,34 @@ def poll_task_for_document_id(task_id: str) -> int:
doc_str = task_info.get("related_document")
if doc_str:
return int(doc_str)
raise RuntimeError(
f"Task {task_id} completed but no doc ID found. Task info: {task_info}"
)
raise RuntimeError(f"Task {task_id} completed but no doc ID found. Task info: {task_info}")
elif status == "FAILURE":
raise RuntimeError(f"Task {task_id} failed: {task_info.get('result')}")
attempts += 1
time.sleep(POLL_INTERVAL_SEC)
raise TimeoutError(
f"Task {task_id} didn't reach SUCCESS within {POLL_MAX_ATTEMPTS} attempts."
)
raise TimeoutError(f"Task {task_id} didn't reach SUCCESS within {POLL_MAX_ATTEMPTS} attempts.")
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_paperless(self, file_path: str, file_id: int = None):
"""
Uploads a file to Paperless-ngx.
Args:
file_path: Path to the file to upload
file_id: Optional file ID to associate with logs
"""
task_id = self.request.id
logger.info(f"[{task_id}] Starting Paperless upload: {file_path}")
log_task_progress(task_id, "upload_to_paperless", "in_progress", f"Uploading to Paperless: {os.path.basename(file_path)}", file_id=file_id)
log_task_progress(
task_id,
"upload_to_paperless",
"in_progress",
f"Uploading to Paperless: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
@@ -125,13 +125,17 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
try:
logger.debug("Posting document to Paperless: file=%s", filename)
resp = requests.post(post_url, headers=_get_headers(), files=files, data=data, timeout=settings.http_request_timeout)
resp = requests.post(
post_url, headers=_get_headers(), files=files, data=data, timeout=settings.http_request_timeout
)
resp.raise_for_status()
except requests.exceptions.RequestException as exc:
error_msg = f"Failed to upload to Paperless: {exc}"
logger.error(
f"[{task_id}] Failed to upload document '%s' to Paperless. Error: %s. Response=%s",
file_path, exc, getattr(exc.response, "text", "<no response>")
file_path,
exc,
getattr(exc.response, "text", "<no response>"),
)
log_task_progress(task_id, "upload_to_paperless", "failure", error_msg, file_id=file_id)
raise
@@ -145,11 +149,13 @@ def upload_to_paperless(self, file_path: str, file_id: int = None):
log_task_progress(task_id, "poll_task", "in_progress", "Waiting for Paperless processing", file_id=file_id)
doc_id = poll_task_for_document_id(raw_task_id)
logger.info(f"[{task_id}] Document {file_path} successfully ingested => ID={doc_id}")
log_task_progress(task_id, "upload_to_paperless", "success", f"Uploaded to Paperless: Doc ID {doc_id}", file_id=file_id)
log_task_progress(
task_id, "upload_to_paperless", "success", f"Uploaded to Paperless: Doc ID {doc_id}", file_id=file_id
)
return {
"status": "Completed",
"paperless_task_id": raw_task_id,
"paperless_document_id": doc_id,
"file_path": file_path
"file_path": file_path,
}
app/tasks/upload_to_s3.py
+5 -8
View File
@@ -1,12 +1,14 @@
#!/usr/bin/env python3
import os
import logging
import os
import boto3
from botocore.exceptions import ClientError
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
logger = logging.getLogger(__name__)
@@ -76,12 +78,7 @@ def upload_to_s3(self, file_path: str, file_id: int = None):
extra_args["ACL"] = settings.s3_acl
# Upload file
s3_client.upload_file(
file_path,
settings.s3_bucket_name,
s3_key,
ExtraArgs=extra_args
)
s3_client.upload_file(file_path, settings.s3_bucket_name, s3_key, ExtraArgs=extra_args)
# Generate URL to the file (useful for public files)
# For private files, this is just a reference and won't be accessible directly
app/tasks/upload_to_sftp.py
+35 -37
View File
@@ -1,22 +1,24 @@
#!/usr/bin/env python3
import os
import logging
import os
import paramiko
from pathlib import Path
from app.config import settings
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
from app.utils import log_task_progress
from app.utils.filename_utils import extract_remote_path, get_unique_filename, sanitize_filename
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_sftp(self, file_path: str, file_id: int = None):
"""
Upload a file to an SFTP server.
Args:
file_path: Path to the file to upload
file_id: Optional file ID to associate with logs
@@ -26,29 +28,29 @@ def upload_to_sftp(self, file_path: str, file_id: int = None):
log_task_progress(
task_id, "upload_to_sftp", "in_progress", f"Uploading to SFTP: {os.path.basename(file_path)}", file_id=file_id
)
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_sftp", "failure", error_msg, file_id=file_id)
raise FileNotFoundError(error_msg)
if not (settings.sftp_host and settings.sftp_port and settings.sftp_username):
error_msg = "SFTP upload skipped: Missing configuration"
logger.info(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_sftp", "skipped", error_msg, file_id=file_id)
return {"status": "Skipped", "reason": "SFTP settings not configured"}
filename = os.path.basename(file_path)
sanitized_filename = sanitize_filename(filename)
# SSH client for SFTP connection
ssh = paramiko.SSHClient()
# Security: Host key verification
# WARNING: AutoAddPolicy automatically trusts unknown host keys (vulnerable to MITM attacks)
# For production, use RejectPolicy and configure known_hosts, or WarningPolicy at minimum
if getattr(settings, 'sftp_disable_host_key_verification', True):
if getattr(settings, "sftp_disable_host_key_verification", True):
logger.warning(
"SFTP host key verification is DISABLED - connections are vulnerable to MITM attacks. "
"For production, set SFTP_DISABLE_HOST_KEY_VERIFICATION=false and configure known_hosts."
@@ -58,7 +60,7 @@ def upload_to_sftp(self, file_path: str, file_id: int = None):
# Use system known_hosts for host key verification (more secure)
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.RejectPolicy())
try:
# Setup connection parameters
connect_kwargs = {
@@ -66,11 +68,11 @@ def upload_to_sftp(self, file_path: str, file_id: int = None):
"port": settings.sftp_port,
"username": settings.sftp_username,
}
# Check for authentication methods - use key if available, otherwise try password
sftp_key_path = getattr(settings, 'sftp_private_key', None)
sftp_key_passphrase = getattr(settings, 'sftp_private_key_passphrase', None)
sftp_key_path = getattr(settings, "sftp_private_key", None)
sftp_key_passphrase = getattr(settings, "sftp_private_key_passphrase", None)
if sftp_key_path and os.path.exists(sftp_key_path):
logger.info(f"Using SSH key authentication with key: {sftp_key_path}")
connect_kwargs["key_filename"] = sftp_key_path
@@ -83,22 +85,22 @@ def upload_to_sftp(self, file_path: str, file_id: int = None):
error_msg = "No authentication method available for SFTP (no key or password)"
logger.error(error_msg)
raise Exception(error_msg)
# Connect to the server
logger.info(f"Connecting to SFTP server at {settings.sftp_host}:{settings.sftp_port}")
ssh.connect(**connect_kwargs)
# Open SFTP session
sftp = ssh.open_sftp()
# Calculate remote path based on local file structure
remote_base = settings.sftp_folder or ""
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
# Ensure the remote path starts with a slash if the base folder does
if remote_base.startswith('/') and not remote_path.startswith('/'):
remote_path = '/' + remote_path
if remote_base.startswith("/") and not remote_path.startswith("/"):
remote_path = "/" + remote_path
# Function to check if file exists in SFTP server
def check_exists_in_sftp(path):
try:
@@ -106,10 +108,10 @@ def upload_to_sftp(self, file_path: str, file_id: int = None):
return True
except FileNotFoundError:
return False
# Check for potential file collision and get a unique name if needed
remote_path = get_unique_filename(remote_path, check_exists_in_sftp)
# Create parent directories if needed
remote_dir = os.path.dirname(remote_path)
if remote_dir:
@@ -127,32 +129,28 @@ def upload_to_sftp(self, file_path: str, file_id: int = None):
sftp.mkdir(current_dir)
except Exception as e:
logger.warning(f"Failed to create directory structure {remote_dir}: {str(e)}")
# Upload the file
logger.info(f"[{task_id}] Uploading {filename} to SFTP at {remote_path}")
sftp.put(file_path, remote_path)
logger.info(f"[{task_id}] Successfully uploaded {filename} to SFTP at {remote_path}")
log_task_progress(task_id, "upload_to_sftp", "success", f"Uploaded to SFTP: {filename}", file_id=file_id)
# Close connections
sftp.close()
ssh.close()
return {
"status": "Completed",
"file_path": file_path,
"sftp_path": remote_path
}
return {"status": "Completed", "file_path": file_path, "sftp_path": remote_path}
except Exception as e:
# Make sure connections are closed
try:
if 'sftp' in locals():
if "sftp" in locals():
sftp.close()
ssh.close()
except:
except Exception:
pass
error_msg = f"Failed to upload {filename} to SFTP server: {str(e)}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_sftp", "failure", error_msg, file_id=file_id)
app/tasks/upload_to_webdav.py
+19 -10
View File
@@ -1,21 +1,24 @@
#!/usr/bin/env python3
import logging
import os
import requests
from urllib.parse import urljoin
import requests
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils import log_task_progress
import logging
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry, bind=True)
def upload_to_webdav(self, file_path: str, file_id: int = None):
"""
Uploads a file to a WebDAV server in the configured folder.
Args:
file_path: Path to the file to upload
file_id: Optional file ID to associate with logs
@@ -23,7 +26,11 @@ def upload_to_webdav(self, file_path: str, file_id: int = None):
task_id = self.request.id
logger.info(f"[{task_id}] Starting WebDAV upload: {file_path}")
log_task_progress(
task_id, "upload_to_webdav", "in_progress", f"Uploading to WebDAV: {os.path.basename(file_path)}", file_id=file_id
task_id,
"upload_to_webdav",
"in_progress",
f"Uploading to WebDAV: {os.path.basename(file_path)}",
file_id=file_id,
)
if not os.path.exists(file_path):
@@ -47,13 +54,13 @@ def upload_to_webdav(self, file_path: str, file_id: int = None):
# Ensure folder doesn't have leading slash if we're joining it to the base URL
if webdav_folder and webdav_folder.startswith("/"):
webdav_folder = webdav_folder[1:]
# Join the base URL and folder path
target_url = urljoin(settings.webdav_url, webdav_folder)
# Ensure URL ends with a slash for proper joining with filename
if not target_url.endswith("/"):
target_url += "/"
# Construct final URL with filename
webdav_url = urljoin(target_url, filename)
@@ -65,20 +72,22 @@ def upload_to_webdav(self, file_path: str, file_id: int = None):
auth=(settings.webdav_username, settings.webdav_password),
data=file_data,
verify=settings.webdav_verify_ssl if hasattr(settings, "webdav_verify_ssl") else True,
timeout=settings.http_request_timeout
timeout=settings.http_request_timeout,
)
# Check if upload was successful
if response.status_code in (200, 201, 204):
logger.info(f"[{task_id}] Successfully uploaded {filename} to WebDAV at {webdav_url}.")
log_task_progress(task_id, "upload_to_webdav", "success", f"Uploaded to WebDAV: {filename}", file_id=file_id)
log_task_progress(
task_id, "upload_to_webdav", "success", f"Uploaded to WebDAV: {filename}", file_id=file_id
)
return {"status": "Completed", "file": file_path, "url": webdav_url}
else:
error_msg = f"Failed to upload {filename} to WebDAV: {response.status_code} - {response.text}"
logger.error(f"[{task_id}] {error_msg}")
log_task_progress(task_id, "upload_to_webdav", "failure", error_msg, file_id=file_id)
raise Exception(error_msg)
except Exception as e:
error_msg = f"Error uploading {filename} to WebDAV: {str(e)}"
logger.error(f"[{task_id}] {error_msg}")
app/tasks/upload_with_rclone.py
+3 -2
View File
@@ -1,11 +1,12 @@
#!/usr/bin/env python3
import logging
import os
import subprocess
import logging
from app.celery_app import celery
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
logger = logging.getLogger(__name__)
app/tasks/uptime_kuma_tasks.py
+3 -1
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import logging
import requests
from celery import shared_task
@@ -8,6 +9,7 @@ from app.config import settings
logger = logging.getLogger(__name__)
@shared_task
def ping_uptime_kuma():
"""
@@ -18,7 +20,7 @@ def ping_uptime_kuma():
if not settings.uptime_kuma_url:
logger.debug("Uptime Kuma URL not configured, skipping ping")
return
try:
logger.info(f"Pinging Uptime Kuma at {settings.uptime_kuma_url}")
response = requests.get(settings.uptime_kuma_url, timeout=10)