gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply Black and isort formatting to modified files

Browse Source 
Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-07 15:00:18 +00:00
parent 8eea420bc5
commit a12ccdabac
2 changed files with 77 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/process.py
+39 -50
View File
@@ -1,26 +1,29 @@
"""
Document processing API endpoints
"""
from fastapi import APIRouter, HTTPException
import logging
import os
from fastapi import APIRouter, HTTPException
from app.api.common import resolve_file_path
from app.auth import require_login
from app.config import settings
from app.api.common import resolve_file_path
from app.tasks.process_document import process_document
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.send_to_all import send_to_all_destinations
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_paperless import upload_to_paperless
# Set up logging
logger = logging.getLogger(__name__)
router = APIRouter()
@router.post("/process/")
@require_login
def process(file_path: str):
@@ -28,101 +31,92 @@ def process(file_path: str):
file_path = resolve_file_path(file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
task = process_document.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_dropbox/")
@require_login
def send_to_dropbox_endpoint(file_path: str):
"""Send a document to Dropbox."""
file_path = resolve_file_path(file_path, 'processed')
file_path = resolve_file_path(file_path, "processed")
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
task = upload_to_dropbox.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_paperless/")
@require_login
def send_to_paperless_endpoint(file_path: str):
"""Send a document to Paperless-ngx."""
file_path = resolve_file_path(file_path, 'processed')
file_path = resolve_file_path(file_path, "processed")
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
task = upload_to_paperless.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_nextcloud/")
@require_login
def send_to_nextcloud_endpoint(file_path: str):
"""Send a document to NextCloud."""
file_path = resolve_file_path(file_path, 'processed')
file_path = resolve_file_path(file_path, "processed")
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
task = upload_to_nextcloud.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_google_drive/")
@require_login
def send_to_google_drive_endpoint(file_path: str):
"""Send a document to Google Drive."""
file_path = resolve_file_path(file_path, 'processed')
file_path = resolve_file_path(file_path, "processed")
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
task = upload_to_google_drive.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_onedrive/")
@require_login
def send_to_onedrive_endpoint(file_path: str):
"""Send a document to OneDrive."""
file_path = resolve_file_path(file_path, 'processed')
file_path = resolve_file_path(file_path, "processed")
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
task = upload_to_onedrive.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_all_destinations/")
@require_login
def send_to_all_destinations_endpoint(file_path: str):
"""Call the aggregator task that sends this file to all configured destinations."""
file_path = resolve_file_path(file_path, 'processed')
file_path = resolve_file_path(file_path, "processed")
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
task = send_to_all_destinations.delay(file_path)
return {"task_id": task.id, "status": "queued", "file_path": file_path}
@router.post("/processall")
@require_login
def process_all_pdfs_in_workdir():
"""
Finds all .pdf files in <workdir> and enqueues them for processing.
For large batches (>processall_throttle_threshold files), tasks are staggered
to avoid overwhelming downstream APIs.
"""
target_dir = settings.workdir
if not os.path.exists(target_dir):
raise HTTPException(
status_code=400, detail=f"Directory {target_dir} does not exist."
)
raise HTTPException(status_code=400, detail=f"Directory {target_dir} does not exist.")
pdf_files = []
for filename in os.listdir(target_dir):
@@ -134,20 +128,20 @@ def process_all_pdfs_in_workdir():
task_ids = []
num_files = len(pdf_files)
# Apply throttling if we have more files than the threshold
apply_throttle = num_files > settings.processall_throttle_threshold
if apply_throttle:
logger.info(
f"Processing {num_files} files with throttling "
f"(threshold: {settings.processall_throttle_threshold}, "
f"delay: {settings.processall_throttle_delay}s per file)"
)
for index, pdf in enumerate(pdf_files):
file_path = os.path.join(target_dir, pdf)
if apply_throttle:
# Stagger task submission with countdown
# First file starts immediately (countdown=0)
@@ -158,17 +152,12 @@ def process_all_pdfs_in_workdir():
else:
# No throttling - enqueue immediately
task = process_document.delay(file_path)
task_ids.append(task.id)
message = f"Enqueued {num_files} PDFs for processing"
if apply_throttle:
total_time = (num_files - 1) * settings.processall_throttle_delay
message += f" (throttled over {total_time} seconds)"
return {
"message": message,
"pdf_files": pdf_files,
"task_ids": task_ids,
"throttled": apply_throttle
}
return {"message": message, "pdf_files": pdf_files, "task_ids": task_ids, "throttled": apply_throttle}
app/config.py
+38 -47
View File
@@ -1,10 +1,12 @@
#!/usr/bin/env python3
from pydantic_settings import BaseSettings
from typing import Optional, List, Dict, Any, Union
from pydantic import Field, validator
import os
from datetime import datetime
from typing import Any, Dict, List, Optional, Union
from pydantic import Field, validator
from pydantic_settings import BaseSettings
class Settings(BaseSettings):
database_url: str
@@ -14,23 +16,23 @@ class Settings(BaseSettings):
openai_model: str = "gpt-4o-mini" # Default model
workdir: str
debug: bool = False # Default to False
# Making Dropbox optional
dropbox_app_key: Optional[str] = None
dropbox_app_secret: Optional[str] = None
dropbox_folder: Optional[str] = None
dropbox_refresh_token: Optional[str] = None
# Making Nextcloud optional
nextcloud_upload_url: Optional[str] = None
nextcloud_username: Optional[str] = None
nextcloud_password: Optional[str] = None
nextcloud_folder: Optional[str] = None
# Making Paperless optional
paperless_ngx_api_token: Optional[str] = None
paperless_host: Optional[str] = None
azure_ai_key: str
azure_region: str
azure_endpoint: str
@@ -71,7 +73,7 @@ class Settings(BaseSettings):
google_drive_credentials_json: Optional[str] = ""
google_drive_folder_id: Optional[str] = ""
google_drive_delegate_to: Optional[str] = "" # Optional delegated user email
# Google Drive OAuth settings
google_drive_use_oauth: bool = False # Default to service account method
google_drive_client_id: Optional[str] = ""
@@ -137,57 +139,43 @@ class Settings(BaseSettings):
# Batch processing settings
processall_throttle_threshold: int = Field(
default=20,
description="Number of files above which throttling is applied in /processall endpoint"
default=20, description="Number of files above which throttling is applied in /processall endpoint"
)
processall_throttle_delay: int = Field(
default=3,
description="Delay in seconds between each task submission when throttling in /processall"
default=3, description="Delay in seconds between each task submission when throttling in /processall"
)
# Notification settings
notification_urls: Union[List[str], str] = Field(
default_factory=list,
description="List of Apprise notification URLs (e.g., discord://, telegram://, etc.)"
)
notify_on_task_failure: bool = Field(
default=True,
description="Send notifications when Celery tasks fail"
default_factory=list, description="List of Apprise notification URLs (e.g., discord://, telegram://, etc.)"
)
notify_on_task_failure: bool = Field(default=True, description="Send notifications when Celery tasks fail")
notify_on_credential_failure: bool = Field(
default=True,
description="Send notifications when credential checks fail"
)
notify_on_startup: bool = Field(
default=True,
description="Send notifications when application starts"
)
notify_on_shutdown: bool = Field(
default=False,
description="Send notifications when application shuts down"
default=True, description="Send notifications when credential checks fail"
)
notify_on_startup: bool = Field(default=True, description="Send notifications when application starts")
notify_on_shutdown: bool = Field(default=False, description="Send notifications when application shuts down")
notify_on_file_processed: bool = Field(
default=True,
description="Send notifications when files are successfully processed"
default=True, description="Send notifications when files are successfully processed"
)
@validator('notification_urls', pre=True)
@validator("notification_urls", pre=True)
def parse_notification_urls(cls, v):
"""Parse notification URLs from string or list"""
if isinstance(v, str):
if ',' in v:
return [url.strip() for url in v.split(',') if url.strip()]
if "," in v:
return [url.strip() for url in v.split(",") if url.strip()]
elif v.strip():
return [v.strip()]
return []
return v
@validator('session_secret')
@validator("session_secret")
def validate_session_secret(cls, v, values):
"""Validate that session_secret is set and has sufficient length when auth is enabled"""
if values.get('auth_enabled') and not v:
if values.get("auth_enabled") and not v:
raise ValueError("SESSION_SECRET must be set when AUTH_ENABLED=True")
if values.get('auth_enabled') and v and len(v) < 32:
if values.get("auth_enabled") and v and len(v) < 32:
raise ValueError("SESSION_SECRET must be at least 32 characters long")
return v
@@ -198,13 +186,13 @@ class Settings(BaseSettings):
env_build_date = os.environ.get("BUILD_DATE")
if env_build_date:
return env_build_date
# Then try to get build date from BUILD_DATE file
build_date_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "BUILD_DATE")
if os.path.exists(build_date_file):
with open(build_date_file, "r") as f:
return f.read().strip()
# Default to unknown if not found
return "Unknown build date"
@@ -215,35 +203,38 @@ class Settings(BaseSettings):
env_version = os.environ.get("APP_VERSION")
if env_version:
return env_version
# Then try to get version from VERSION file
version_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "VERSION")
if os.path.exists(version_file):
with open(version_file, "r") as f:
return f.read().strip()
# Default version if not found
return "0.3.2-dev"
class Config:
env_file = ".env"
# Convert string representations of booleans to actual booleans
# and strip quotes from string values
@classmethod
def parse_env_var(cls, field_name: str, raw_val: str) -> Any:
# First, strip quotes from the value if it's a string
if isinstance(raw_val, str):
if (raw_val.startswith('"') and raw_val.endswith('"')) or \
(raw_val.startswith("'") and raw_val.endswith("'")):
if (raw_val.startswith('"') and raw_val.endswith('"')) or (
raw_val.startswith("'") and raw_val.endswith("'")
):
raw_val = raw_val[1:-1]
raw_val = raw_val.strip()
# Convert string representations of booleans to actual booleans
if field_name.endswith('_enabled') or field_name == 'debug':
if raw_val.lower() in ('false', '0', 'no', 'n', 'f'):
if field_name.endswith("_enabled") or field_name == "debug":
if raw_val.lower() in ("false", "0", "no", "n", "f"):
return False
if raw_val.lower() in ('true', '1', 'yes', 'y', 't'):
if raw_val.lower() in ("true", "1", "yes", "y", "t"):
return True
return raw_val
settings = Settings()