gitea-admin/gh-christianlouis-docuelevate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement Google Drive integration and enhance documentation for new features

Browse Source
This commit is contained in:
Christian Krakau-Louis
2025-04-01 04:44:00 +02:00
parent 71937202a8
commit 6972fb7505
42 changed files with 3590 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.demo
+91 -33
View File
@@ -1,57 +1,115 @@
# **Config Variables**
# **Core Settings**
WORKDIR=/workdir
DATABASE_URL=sqlite:///./app/database.db
REDIS_URL=redis://redis:6379/0
WORKDIR=/workdir
AWS_REGION="eu-central-1"
AZURE_REGION="eastus"
AZURE_ENDPOINT="https://<yourendpoint>.cognitiveservices.azure.com/"
S3_BUCKET_NAME=<your_bucket_name>
NEXTCLOUD_UPLOAD_URL=https://nextcloud.example.com/remote.php/dav/files/<USERNAME>
NEXTCLOUD_FOLDER="<NEXTCLOUD_FOLDER_PATH>"
PAPERLESS_NGX_URL=https://paperless.example.com/api/documents/post_document/
PAPERLESS_HOST=https://paperless.example.com
EXTERNAL_HOSTNAME=docunova.example.com
GOTENBERG_URL=http://gotenberg:3000
# **Tokens/API Credentials**
AWS_ACCESS_KEY_ID="<AWS_ACCESS_KEY>"
AWS_SECRET_ACCESS_KEY="<AWS_SECRET_ACCESS_KEY>"
# **Authentication**
AUTH_ENABLED=true
SESSION_SECRET=<atLeast32Characters>
ADMIN_USERNAME=admin
ADMIN_PASSWORD=your_secure_password
# **OpenID Connect/Authentik Settings**
AUTHENTIK_CLIENT_ID=<yourAuthentikAppClientID>
AUTHENTIK_CLIENT_SECRET=<yourAuthentikAppClientSecret>
AUTHENTIK_CONFIG_URL=<ConfigUrlOfYourApp, e.g. https://authentik.example.com/application/o/docunova/.well-known/openid-configuration>
# **AI/ML Services**
# OpenAI
OPENAI_API_KEY="<OPENAI_API_KEY>"
OPENAI_BASE_URL=https://api.openai.com/v1
OPENAI_MODEL=gpt-4o-mini
PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
DROPBOX_APP_KEY=<DROPBOX_APP_KEY>
DROPBOX_APP_SECRET=<DROPBOX_APP_SECRET>
DROPBOX_REFRESH_TOKEN=<DROPBOX_REFRESH_TOKEN>
# Azure AI
AZURE_REGION="eastus"
AZURE_ENDPOINT="https://<yourendpoint>.cognitiveservices.azure.com/"
AZURE_AI_KEY=<AZURE_AI_KEY>
# **User Credentials**
ADMIN_USERNAME=admin
ADMIN_PASSWORD=your_secure_password
NEXTCLOUD_USERNAME=<NEXTCLOUD_USERNAME>
NEXTCLOUD_PASSWORD=<NEXTCLOUD_PASSWORD>
IMAP1_USERNAME=<IMAP1_USERNAME>
IMAP1_PASSWORD=<IMAP1_PASSWORD>
IMAP2_USERNAME=<IMAP2_USERNAME>
IMAP2_PASSWORD=<IMAP2_PASSWORD>
# **Email Settings**
EMAIL_HOST=smtp.example.com
EMAIL_PORT=587
EMAIL_USERNAME=docunova@example.com
EMAIL_PASSWORD=your_secure_email_password
EMAIL_USE_TLS=True
EMAIL_SENDER=DocuNova System <docunova@example.com>
EMAIL_DEFAULT_RECIPIENT=recipient@example.com
# **IMAP Settings**
IMAP1_HOST=mail.example.com
IMAP1_PORT=993
IMAP1_USERNAME=<IMAP1_USERNAME>
IMAP1_PASSWORD=<IMAP1_PASSWORD>
IMAP1_SSL=true
IMAP1_POLL_INTERVAL_MINUTES=5
IMAP1_DELETE_AFTER_PROCESS=false
IMAP2_HOST=imap.gmail.com
IMAP2_PORT=993
IMAP2_USERNAME=<IMAP2_USERNAME>
IMAP2_PASSWORD=<IMAP2_PASSWORD>
IMAP2_SSL=true
IMAP2_POLL_INTERVAL_MINUTES=10
IMAP2_DELETE_AFTER_PROCESS=false
GOTENBERG_URL=http://gotenberg:3000
# **Storage/Document Services**
# Amazon S3
AWS_REGION=us-east-1
AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
S3_BUCKET_NAME=my-document-bucket
S3_FOLDER_PREFIX=documents/uploads/2023/ # Organizes files in this subfolder
S3_STORAGE_CLASS=STANDARD
S3_ACL=private
# ** needed for Authentik **
AUTH_ENABLED=true
SESSION_SECRET=<atLeast32Characters>
AUTHENTIK_CLIENT_ID=<yourAuthentikAppClientID>
AUTHENTIK_CLIENT_SECRET=<yourAuthentikAppClientSecret>
AUTHENTIK_CONFIG_URL=<ConfigUrlOfYourApp, e.g. https://authentik.example.com/application/o/docunova/.well-known/openid-configuration>
# NextCloud
NEXTCLOUD_UPLOAD_URL=https://nextcloud.example.com/remote.php/dav/files/<USERNAME>
NEXTCLOUD_FOLDER="<NEXTCLOUD_FOLDER_PATH>"
NEXTCLOUD_USERNAME=<NEXTCLOUD_USERNAME>
NEXTCLOUD_PASSWORD=<NEXTCLOUD_PASSWORD>
# Paperless-ngx
PAPERLESS_HOST=https://paperless.example.com
PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
# Dropbox
DROPBOX_APP_KEY=<DROPBOX_APP_KEY>
DROPBOX_APP_SECRET=<DROPBOX_APP_SECRET>
DROPBOX_REFRESH_TOKEN=<DROPBOX_REFRESH_TOKEN>
DROPBOX_FOLDER="/Documents/Uploads"
# Google Drive
GOOGLE_DRIVE_CREDENTIALS_JSON={"type":"service_account","project_id":"your-project","private_key_id":"key-id","private_key":"-----BEGIN PRIVATE KEY-----\nYOUR_PRIVATE_KEY\n-----END PRIVATE KEY-----\n","client_email":"service-account@project.iam.gserviceaccount.com","client_id":"client-id","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/service-account%40project.iam.gserviceaccount.com"}
GOOGLE_DRIVE_FOLDER_ID=<YOUR_FOLDER_ID>
GOOGLE_DRIVE_DELEGATE_TO=<OPTIONAL_USER_EMAIL>
# OneDrive
ONEDRIVE_CLIENT_ID=your-client-id
ONEDRIVE_CLIENT_SECRET=your-client-secret
ONEDRIVE_TENANT_ID=common
ONEDRIVE_REFRESH_TOKEN=your-refresh-token
ONEDRIVE_FOLDER_PATH=Documents/Uploads
# WebDAV
WEBDAV_URL=https://webdav.example.com/path
WEBDAV_USERNAME=webdav_user
WEBDAV_PASSWORD=your_secure_webdav_password
WEBDAV_FOLDER=/Documents/Uploads
WEBDAV_VERIFY_SSL=True
# FTP
FTP_HOST=ftp.example.com
FTP_PORT=21
FTP_USERNAME=ftp_user
FTP_PASSWORD=your_secure_ftp_password
FTP_FOLDER=/Documents/Uploads
# SFTP
SFTP_HOST=sftp.example.com
SFTP_PORT=22
SFTP_USERNAME=sftp_user
SFTP_PASSWORD=your_secure_sftp_password
# SFTP_PRIVATE_KEY=/path/to/private_key.pem
# SFTP_PRIVATE_KEY_PASSPHRASE=optional_passphrase
SFTP_FOLDER=/Documents/Uploads
README.md
+3 -2
View File
@@ -10,7 +10,7 @@
DocuNova automates the handling, extraction, and processing of documents using a variety of services, including:
- **OpenAI** for metadata extraction and text refinement.
- **Dropbox** and **Nextcloud** for file storage and uploads.
- **Dropbox**, **Nextcloud**, and **Google Drive** for file storage and uploads.
- **Paperless NGX** for document indexing and management.
- **Azure Document Intelligence** for OCR on PDFs.
- **Gotenberg** for file-to-PDF conversions.
@@ -67,6 +67,7 @@ Every document goes through the following steps:
Processed documents with their metadata can be automatically sent to:
- **Dropbox**: For cloud storage and sharing
- **Nextcloud**: For self-hosted file storage
- **Google Drive**: For Google Workspace integration
- **Paperless-NGX**: For advanced document management with search capabilities
Users can choose to send documents to any combination of these destinations through configuration settings or manual selection.
@@ -74,7 +75,7 @@ Users can choose to send documents to any combination of these destinations thro
## Features
- **Document Upload & Storage**:
- Manual uploads (via API or UI) to Dropbox, Nextcloud, or Paperless.
- Manual uploads (via API or UI) to Dropbox, Nextcloud, Google Drive, or Paperless.
- **OCR Processing (Azure)**:
- Extract text from scanned PDFs using Azure Document Intelligence.
- **Metadata Extraction (OpenAI)**:
app/api.py
+178 -2
View File
@@ -1,12 +1,20 @@
# app/api.py
from fastapi import APIRouter, Request, HTTPException, status, Depends
from fastapi import APIRouter, Request, HTTPException, status, Depends, UploadFile, File
from hashlib import md5
from sqlalchemy.orm import Session
from typing import List
import os
from app.auth import require_login
from app.auth import require_login, get_current_user
from app.database import SessionLocal
from app.models import FileRecord
from app.config import settings
from app.tasks.process_document import process_document
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.send_to_all import send_to_all_destinations
router = APIRouter()
@@ -74,3 +82,171 @@ def list_files_api(request: Request, db: Session = Depends(get_db)):
"created_at": f.created_at.isoformat() if f.created_at else None
})
return result
# API endpoints
@router.get("/diagnostic/settings")
@require_login
async def diagnostic_settings(current_user: dict = Depends(get_current_user)):
"""
API endpoint to dump settings to the log and view basic config information
This endpoint doesn't expose sensitive information like passwords or tokens
"""
from app.utils.config_validator import dump_all_settings
# Dump full settings to log for admin to see
dump_all_settings()
# Return safe subset of settings for API response
safe_settings = {
"workdir": settings.workdir,
"external_hostname": settings.external_hostname,
"configured_services": {
"email": bool(getattr(settings, 'email_host', None)),
"s3": bool(getattr(settings, 's3_bucket_name', None)),
"dropbox": bool(getattr(settings, 'dropbox_refresh_token', None)),
"nextcloud": bool(getattr(settings, 'nextcloud_upload_url', None)),
"sftp": bool(getattr(settings, 'sftp_host', None)),
"paperless": bool(getattr(settings, 'paperless_host', None)),
"google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)),
},
"imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)),
}
return {
"status": "success",
"settings": safe_settings,
"message": "Full settings have been dumped to application logs"
}
@router.post("/process/")
@require_login
def process(file_path: str):
"""API Endpoint to start document processing."""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = process_document.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_dropbox/")
@require_login
def send_to_dropbox(file_path: str):
"""Send a document to Dropbox."""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = upload_to_dropbox.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_paperless/")
@require_login
def send_to_paperless(file_path: str):
"""Send a document to Paperless-ngx."""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = upload_to_paperless.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_nextcloud/")
@require_login
def send_to_nextcloud(file_path: str):
"""Send a document to NextCloud."""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = upload_to_nextcloud.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_google_drive/")
@require_login
def send_to_google_drive(file_path: str):
"""Send a document to Google Drive."""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = upload_to_google_drive.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_all_destinations/")
@require_login
def send_to_all_destinations_endpoint(file_path: str):
"""Call the aggregator task that sends this file to all configured destinations."""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = send_to_all_destinations.delay(file_path)
return {"task_id": task.id, "status": "queued", "file_path": file_path}
@router.post("/processall")
@require_login
def process_all_pdfs_in_workdir():
"""Finds all .pdf files in <workdir> and enqueues them for processing."""
target_dir = settings.workdir
if not os.path.exists(target_dir):
raise HTTPException(
status_code=400, detail=f"Directory {target_dir} does not exist."
)
pdf_files = []
for filename in os.listdir(target_dir):
if filename.lower().endswith(".pdf"):
pdf_files.append(filename)
if not pdf_files:
return {"message": "No PDF files found in that directory."}
task_ids = []
for pdf in pdf_files:
file_path = os.path.join(target_dir, pdf)
task = process_document.delay(file_path)
task_ids.append(task.id)
return {
"message": f"Enqueued {len(pdf_files)} PDFs to upload_to_s3",
"pdf_files": pdf_files,
"task_ids": task_ids
}
@router.post("/ui-upload")
@require_login
async def ui_upload(file: UploadFile = File(...)):
"""Endpoint to accept a user-uploaded file and enqueue it for processing."""
workdir = "/workdir"
target_path = os.path.join(workdir, file.filename)
try:
with open(target_path, "wb") as f:
content = await file.read()
f.write(content)
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to save file: {e}"
)
task = process_document.delay(target_path)
return {"task_id": task.id, "status": "queued"}
# Note: The api/router.py is now a submodule organization,
# but we're keeping this file for compatibility until we've fully migrated
app/api/__init__.py
View File
app/api/router.py
+179
View File
@@ -0,0 +1,179 @@
from fastapi import APIRouter, HTTPException, UploadFile, File, Depends
import os
from app.config import settings
from app.auth import get_current_user, require_login
from app.tasks.process_document import process_document
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.send_to_all import send_to_all_destinations
router = APIRouter()
# Diagnostic endpoints
@router.get("/diagnostic/settings")
@require_login
async def diagnostic_settings(current_user: dict = Depends(get_current_user)):
"""
API endpoint to dump settings to the log and view basic config information
This endpoint doesn't expose sensitive information like passwords or tokens
"""
from app.utils.config_validator import dump_all_settings
# Dump full settings to log for admin to see
dump_all_settings()
# Return safe subset of settings for API response
safe_settings = {
"workdir": settings.workdir,
"external_hostname": settings.external_hostname,
"configured_services": {
"email": bool(getattr(settings, 'email_host', None)),
"s3": bool(getattr(settings, 's3_bucket_name', None)),
"dropbox": bool(getattr(settings, 'dropbox_refresh_token', None)),
"nextcloud": bool(getattr(settings, 'nextcloud_upload_url', None)),
"sftp": bool(getattr(settings, 'sftp_host', None)),
"paperless": bool(getattr(settings, 'paperless_host', None)),
"google_drive": bool(getattr(settings, 'google_drive_credentials_json', None)),
},
"imap_enabled": bool(getattr(settings, 'imap1_host', None) or getattr(settings, 'imap2_host', None)),
}
return {
"status": "success",
"settings": safe_settings,
"message": "Full settings have been dumped to application logs"
}
# File processing endpoints
@router.post("/process/")
@require_login
def process(file_path: str):
"""API Endpoint to start document processing."""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = process_document.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_dropbox/")
@require_login
def send_to_dropbox(file_path: str):
"""Send a document to Dropbox."""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = upload_to_dropbox.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_paperless/")
@require_login
def send_to_paperless(file_path: str):
"""Send a document to Paperless-ngx."""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = upload_to_paperless.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_nextcloud/")
@require_login
def send_to_nextcloud(file_path: str):
"""Send a document to NextCloud."""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = upload_to_nextcloud.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_google_drive/")
@require_login
def send_to_google_drive(file_path: str):
"""Send a document to Google Drive."""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = upload_to_google_drive.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@router.post("/send_to_all_destinations/")
@require_login
def send_to_all_destinations_endpoint(file_path: str):
"""Call the aggregator task that sends this file to all configured destinations."""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = send_to_all_destinations.delay(file_path)
return {"task_id": task.id, "status": "queued", "file_path": file_path}
@router.post("/processall")
@require_login
def process_all_pdfs_in_workdir():
"""Finds all .pdf files in <workdir> and enqueues them for processing."""
target_dir = settings.workdir
if not os.path.exists(target_dir):
raise HTTPException(
status_code=400, detail=f"Directory {target_dir} does not exist."
)
pdf_files = []
for filename in os.listdir(target_dir):
if filename.lower().endswith(".pdf"):
pdf_files.append(filename)
if not pdf_files:
return {"message": "No PDF files found in that directory."}
task_ids = []
for pdf in pdf_files:
file_path = os.path.join(target_dir, pdf)
task = process_document.delay(file_path)
task_ids.append(task.id)
return {
"message": f"Enqueued {len(pdf_files)} PDFs to upload_to_s3",
"pdf_files": pdf_files,
"task_ids": task_ids
}
@router.post("/ui-upload")
@require_login
async def ui_upload(file: UploadFile = File(...)):
"""Endpoint to accept a user-uploaded file and enqueue it for processing."""
workdir = "/workdir"
target_path = os.path.join(workdir, file.filename)
try:
with open(target_path, "wb") as f:
content = await file.read()
f.write(content)
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to save file: {e}"
)
task = process_document.delay(target_path)
return {"task_id": task.id, "status": "queued"}
app/celery_worker.py
+8
View File
@@ -20,6 +20,14 @@ from app.tasks.convert_to_pdf import convert_to_pdf
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_webdav import upload_to_webdav
from app.tasks.upload_to_s3 import upload_to_s3
from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_email import upload_to_email
from app.tasks.imap_tasks import pull_all_inboxes
from app.tasks.send_to_all import send_to_all_destinations
app/config.py
+71 -17
View File
@@ -4,34 +4,35 @@ from pydantic_settings import BaseSettings
from typing import Optional
class Settings(BaseSettings):
admin_username: str
admin_password: str
aws_access_key_id: str
aws_secret_access_key: str
aws_region: str
database_url: str
redis_url: str
s3_bucket_name: str
openai_api_key: str
openai_base_url: str = "https://api.openai.com/v1" # Default to OpenAI's endpoint
openai_model: str = "gpt-4o-mini" # Default model
workdir: str
dropbox_app_key: str
dropbox_app_secret: str
dropbox_folder: str
dropbox_refresh_token: str
nextcloud_upload_url: str
nextcloud_username: str
nextcloud_password: str
nextcloud_folder: str
paperless_ngx_api_token: str
paperless_host: str
# Making Dropbox optional
dropbox_app_key: Optional[str] = None
dropbox_app_secret: Optional[str] = None
dropbox_folder: Optional[str] = None
dropbox_refresh_token: Optional[str] = None
# Making Nextcloud optional
nextcloud_upload_url: Optional[str] = None
nextcloud_username: Optional[str] = None
nextcloud_password: Optional[str] = None
nextcloud_folder: Optional[str] = None
# Making Paperless optional
paperless_ngx_api_token: Optional[str] = None
paperless_host: Optional[str] = None
azure_ai_key: str
azure_region: str
azure_endpoint: str
gotenberg_url: str
external_hostname: str = "localhost" # Default to localhost
# Authentik
authentik_client_id: Optional[str] = None
authentik_client_secret: Optional[str] = None
@@ -56,6 +57,59 @@ class Settings(BaseSettings):
imap2_poll_interval_minutes: int = 10
imap2_delete_after_process: bool = False
# Google Drive settings
google_drive_credentials_json: str = ""
google_drive_folder_id: str = ""
google_drive_delegate_to: str = "" # Optional delegated user email
# WebDAV settings
webdav_url: Optional[str] = None
webdav_username: Optional[str] = None
webdav_password: Optional[str] = None
webdav_folder: Optional[str] = None
webdav_verify_ssl: bool = True
# FTP settings
ftp_host: Optional[str] = None
ftp_port: Optional[int] = 21
ftp_username: Optional[str] = None
ftp_password: Optional[str] = None
ftp_folder: Optional[str] = None
# SFTP settings
sftp_host: Optional[str] = None
sftp_port: Optional[int] = 22
sftp_username: Optional[str] = None
sftp_password: Optional[str] = None
sftp_folder: Optional[str] = None
sftp_private_key: Optional[str] = None
sftp_private_key_passphrase: Optional[str] = None
# Email settings
email_host: Optional[str] = None
email_port: Optional[int] = 587
email_username: Optional[str] = None
email_password: Optional[str] = None
email_use_tls: bool = True
email_sender: Optional[str] = None # From address, defaults to email_username if not set
email_default_recipient: Optional[str] = None
# OneDrive settings
onedrive_client_id: Optional[str] = None
onedrive_client_secret: Optional[str] = None
onedrive_tenant_id: Optional[str] = "common" # Default to "common" for personal accounts
onedrive_refresh_token: Optional[str] = None # Required for personal accounts
onedrive_folder_path: Optional[str] = None
# AWS S3 settings
aws_access_key_id: Optional[str] = None
aws_secret_access_key: Optional[str] = None
aws_region: Optional[str] = "us-east-1" # Default region
s3_bucket_name: Optional[str] = None
s3_folder_prefix: Optional[str] = "" # Optional folder prefix (e.g. "uploads/")
s3_storage_class: Optional[str] = "STANDARD" # Default storage class
s3_acl: Optional[str] = "private" # Default ACL
class Config:
env_file = ".env"
app/frontend.py
+59 -12
View File
@@ -1,15 +1,17 @@
# app/frontend.py
import os
from pathlib import Path
from fastapi import APIRouter, Request, Depends
from fastapi import APIRouter, Request, Depends, HTTPException
from fastapi.templating import Jinja2Templates
from pathlib import Path
from sqlalchemy.orm import Session
from datetime import datetime
from app.auth import require_login
from app.database import SessionLocal
from app.config import settings
router = APIRouter()
# Set up Jinja2 templates
templates_dir = Path(__file__).parent.parent / "frontend" / "templates"
templates = Jinja2Templates(directory=str(templates_dir))
@@ -29,24 +31,69 @@ def files_page(request: Request):
"""
return templates.TemplateResponse("files.html", {"request": request})
# ... existing routes for /, /upload, /about, etc. ...
@router.get("/", include_in_schema=False)
async def serve_index(request: Request):
return templates.TemplateResponse("index.html", {"request": request})
return templates.TemplateResponse("index.html", {"request": request})
@router.get("/about", include_in_schema=False)
async def serve_about(request: Request):
return templates.TemplateResponse("about.html", {"request": request})
return templates.TemplateResponse("about.html", {"request": request})
@router.get("/upload", include_in_schema=False)
@require_login
async def serve_upload(request: Request):
return templates.TemplateResponse("upload.html", {"request": request})
return templates.TemplateResponse("upload.html", {"request": request})
@router.get("/favicon.ico", include_in_schema=False)
def favicon():
# If you have a real favicon in `frontend/static/favicon.ico`:
favicon_path = Path(__file__).parent.parent / "frontend" / "static" / "favicon.ico"
return str(favicon_path)
@router.get("/status")
@require_login
async def status_dashboard(request: Request):
"""
Status dashboard showing all configured integration targets
"""
from app.utils.config_validator import get_provider_status
# Get provider status
providers = get_provider_status()
return templates.TemplateResponse(
"status_dashboard.html",
{
"request": request,
"providers": providers,
"debug_enabled": getattr(settings, 'debug', False),
"last_check": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
}
)
@router.get("/env")
@require_login
async def env_debug(request: Request):
"""
Debug endpoint to view environment variables and settings
Only shows values when DEBUG is True
"""
# Default DEBUG to True for this route
debug_enabled = True
# Get settings data
from app.utils.config_validator import get_settings_for_display
settings_data = get_settings_for_display(show_values=debug_enabled)
return templates.TemplateResponse(
"env_debug.html",
{
"request": request,
"settings": settings_data,
"debug_enabled": debug_enabled,
"app_version": getattr(settings, 'version', 'Unknown')
}
)
# Note: The frontend/router.py is now a submodule organization,
# but we're keeping this file for compatibility until we've fully migrated
app/frontend/__init__.py
View File
app/frontend/router.py
+90
View File
@@ -0,0 +1,90 @@
from fastapi import APIRouter, Request, Depends, HTTPException
from fastapi.templating import Jinja2Templates
from pathlib import Path
from datetime import datetime
from sqlalchemy.orm import Session
from app.config import settings
from app.auth import require_login
from app.database import SessionLocal
router = APIRouter()
# Set up Jinja2 templates
templates_dir = Path(__file__).parent.parent.parent / "frontend" / "templates"
templates = Jinja2Templates(directory=str(templates_dir))
def get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()
@router.get("/", include_in_schema=False)
async def serve_index(request: Request):
return templates.TemplateResponse("index.html", {"request": request})
@router.get("/about", include_in_schema=False)
async def serve_about(request: Request):
return templates.TemplateResponse("about.html", {"request": request})
@router.get("/upload", include_in_schema=False)
@require_login
async def serve_upload(request: Request):
return templates.TemplateResponse("upload.html", {"request": request})
@router.get("/files")
@require_login
def files_page(request: Request, db: Session = Depends(get_db)):
return templates.TemplateResponse("files.html", {"request": request})
@router.get("/favicon.ico", include_in_schema=False)
def favicon():
favicon_path = Path(__file__).parent.parent.parent / "frontend" / "static" / "favicon.ico"
return str(favicon_path)
@router.get("/status")
@require_login
async def status_dashboard(request: Request):
"""
Status dashboard showing all configured integration targets
"""
from app.utils.config_validator import get_provider_status
# Get provider status
providers = get_provider_status()
return templates.TemplateResponse(
"status_dashboard.html",
{
"request": request,
"providers": providers,
"debug_enabled": getattr(settings, 'debug', False),
"last_check": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
}
)
@router.get("/env")
@require_login
async def env_debug(request: Request):
"""
Debug endpoint to view environment variables and settings
Only shows values when DEBUG is True
"""
# Default DEBUG to True for this route
debug_enabled = True
# Get settings data
from app.utils.config_validator import get_settings_for_display
settings_data = get_settings_for_display(show_values=debug_enabled)
return templates.TemplateResponse(
"env_debug.html",
{
"request": request,
"settings": settings_data,
"debug_enabled": debug_enabled,
"app_version": getattr(settings, 'version', 'Unknown')
}
)
app/main.py
+38 -127
View File
@@ -1,8 +1,8 @@
#!/usr/bin/env python3
import os
import logging
from fastapi import FastAPI, HTTPException, UploadFile, File, status, Request
from fastapi.responses import FileResponse
from fastapi import FastAPI, HTTPException, Request, status
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from starlette.middleware.sessions import SessionMiddleware
@@ -13,16 +13,17 @@ from pathlib import Path
from app.database import init_db
from app.config import settings
from app.tasks.process_document import process_document # Updated import
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.send_to_all import send_to_all_destinations
from app.utils.config_validator import check_all_configs
from app.api import router as api_router
from app.frontend import router as frontend_router
# Import both the traditional and new routers - we'll keep both available for compatibility
from app.frontend.router import router as frontend_router_original
from app.api.router import router as api_router_original
from app.auth import router as auth_router
# Import also the organized router modules
from app.frontend.router import router as frontend_router_organized
from app.api.router import router as api_router_organized
# Load configuration from .env for the session key
config = Config(".env")
SESSION_SECRET = config(
@@ -53,123 +54,27 @@ app.mount("/static", StaticFiles(directory=frontend_static_dir), name="static")
def on_startup():
init_db() # Create tables if they don't exist
@app.post("/process/")
def process(file_path: str):
"""
API Endpoint to start document processing.
This enqueues document processing which handles the full pipeline.
"""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = process_document.delay(file_path) # Updated function call
return {"task_id": task.id, "status": "queued"}
@app.post("/send_to_dropbox/")
def send_to_dropbox(file_path: str):
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = upload_to_dropbox.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@app.post("/send_to_paperless/")
def send_to_paperless(file_path: str):
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = upload_to_paperless.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@app.post("/send_to_nextcloud/")
def send_to_nextcloud(file_path: str):
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = upload_to_nextcloud.delay(file_path)
return {"task_id": task.id, "status": "queued"}
@app.post("/send_to_all_destinations/")
def send_to_all_destinations_endpoint(file_path: str):
"""
Call the aggregator task that sends this file to dropbox, nextcloud, and paperless.
"""
if not os.path.isabs(file_path):
file_path = os.path.join(settings.workdir, 'processed', file_path)
if not os.path.exists(file_path):
raise HTTPException(
status_code=400, detail=f"File {file_path} not found."
)
task = send_to_all_destinations.delay(file_path)
return {"task_id": task.id, "status": "queued", "file_path": file_path}
@app.post("/processall")
def process_all_pdfs_in_workdir():
"""
Finds all .pdf files in <workdir> and enqueues them for processing.
"""
target_dir = settings.workdir
if not os.path.exists(target_dir):
raise HTTPException(
status_code=400, detail=f"Directory {target_dir} does not exist."
)
pdf_files = []
for filename in os.listdir(target_dir):
if filename.lower().endswith(".pdf"):
pdf_files.append(filename)
if not pdf_files:
return {"message": "No PDF files found in that directory."}
task_ids = []
for pdf in pdf_files:
file_path = os.path.join(target_dir, pdf)
task = process_document.delay(file_path) # Updated function call
task_ids.append(task.id)
return {
"message": f"Enqueued {len(pdf_files)} PDFs to upload_to_s3",
"pdf_files": pdf_files,
"task_ids": task_ids
}
@app.post("/ui-upload")
async def ui_upload(file: UploadFile = File(...)):
"""Endpoint to accept a user-uploaded file and enqueue it for processing."""
workdir = "/workdir"
target_path = os.path.join(workdir, file.filename)
try:
with open(target_path, "wb") as f:
content = await file.read()
f.write(content)
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to save file: {e}"
)
task = process_document.delay(target_path) # Updated function call
return {"task_id": task.id, "status": "queued"}
@app.on_event("startup")
async def startup_event():
"""Run startup tasks for the application"""
# Force settings dump to log for troubleshooting
from app.utils.config_validator import dump_all_settings
dump_all_settings()
# Validate configuration
config_issues = check_all_configs()
# Log overall status
has_issues = any(config_issues['email']) or any(len(issues) > 0 for provider, issues in config_issues['storage'].items())
if has_issues:
logging.warning("Application started with configuration issues - some features may be unavailable")
else:
logging.info("Application started with valid configuration")
logging.info("Router organization note: Using route handlers from main app directory for now")
logging.info("In the future, we'll transition fully to the frontend/ and api/ submodules")
# Custom 404 - we can still return the Jinja2 template, or the old static file:
# For a dynamic 404 using the base layout, see "frontend/404.html" usage below:
@app.exception_handler(404)
async def custom_404_handler(request: Request, exc: HTTPException):
# Serve the 404 template directly
@@ -194,7 +99,13 @@ async def custom_500_handler(request: Request, exc: Exception):
def test_500():
raise RuntimeError("Testing forced 500 error!")
# Include the frontend and auth routers
app.include_router(frontend_router)
# Include the routers - for now we're using the original routers
# Later we can switch to the organized router structure
app.include_router(frontend_router_original)
app.include_router(auth_router)
app.include_router(api_router, prefix="/api")
app.include_router(api_router_original, prefix="/api")
# Note: These are the future, organized router modules that we'll eventually switch to
# For now they are disabled to avoid duplicate route conflicts
# app.include_router(frontend_router_organized)
# app.include_router(api_router_organized, prefix="/api")
app/tasks/send_to_all.py
+89 -12
View File
@@ -1,21 +1,98 @@
# app/tasks/send_to_all.py
#!/usr/bin/env python3
from app.celery_app import celery
import os
import logging
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.tasks.upload_to_dropbox import upload_to_dropbox
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
from app.tasks.upload_to_paperless import upload_to_paperless
from app.tasks.upload_to_google_drive import upload_to_google_drive
from app.tasks.upload_to_webdav import upload_to_webdav
from app.tasks.upload_to_ftp import upload_to_ftp
from app.tasks.upload_to_sftp import upload_to_sftp
from app.tasks.upload_to_email import upload_to_email
from app.tasks.upload_to_onedrive import upload_to_onedrive
from app.tasks.upload_to_s3 import upload_to_s3
from app.celery_app import celery
@celery.task
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry)
def send_to_all_destinations(file_path: str):
"""
Fires off tasks to upload a single file to Dropbox, Nextcloud, and Paperless.
These tasks run in parallel (Celery returns immediately from each .delay()).
"""
upload_to_dropbox.delay(file_path)
upload_to_nextcloud.delay(file_path)
upload_to_paperless.delay(file_path)
"""Distribute a file to all configured storage destinations."""
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
logger.info(f"Sending {file_path} to all configured destinations")
results = {}
# Send to Dropbox if configured
if settings.dropbox_app_key and settings.dropbox_app_secret and settings.dropbox_refresh_token:
logger.info(f"Queueing {file_path} for Dropbox upload")
task = upload_to_dropbox.delay(file_path)
results["dropbox_task_id"] = task.id
# Send to Nextcloud if configured
if settings.nextcloud_upload_url and settings.nextcloud_username and settings.nextcloud_password:
logger.info(f"Queueing {file_path} for Nextcloud upload")
task = upload_to_nextcloud.delay(file_path)
results["nextcloud_task_id"] = task.id
# Send to Paperless if configured
if settings.paperless_ngx_api_token and settings.paperless_host:
logger.info(f"Queueing {file_path} for Paperless upload")
task = upload_to_paperless.delay(file_path)
results["paperless_task_id"] = task.id
# Send to Google Drive if configured
if settings.google_drive_credentials_json:
logger.info(f"Queueing {file_path} for Google Drive upload")
task = upload_to_google_drive.delay(file_path)
results["google_drive_task_id"] = task.id
# Send to WebDAV if configured
if settings.webdav_url and settings.webdav_username and settings.webdav_password:
logger.info(f"Queueing {file_path} for WebDAV upload")
task = upload_to_webdav.delay(file_path)
results["webdav_task_id"] = task.id
# Send to FTP if configured
if settings.ftp_host and settings.ftp_username and settings.ftp_password:
logger.info(f"Queueing {file_path} for FTP upload")
task = upload_to_ftp.delay(file_path)
results["ftp_task_id"] = task.id
# Send to SFTP if configured
if settings.sftp_host and settings.sftp_username and (settings.sftp_password or settings.sftp_private_key):
logger.info(f"Queueing {file_path} for SFTP upload")
task = upload_to_sftp.delay(file_path)
results["sftp_task_id"] = task.id
# Send via email if configured
if settings.email_host and settings.email_username and settings.email_password and settings.email_default_recipient:
logger.info(f"Queueing {file_path} for email delivery")
task = upload_to_email.delay(file_path)
results["email_task_id"] = task.id
# Send to OneDrive if configured
if settings.onedrive_client_id and settings.onedrive_client_secret and (
settings.onedrive_refresh_token or
(settings.onedrive_tenant_id and settings.onedrive_tenant_id != "common")
):
logger.info(f"Queueing {file_path} for OneDrive upload")
task = upload_to_onedrive.delay(file_path)
results["onedrive_task_id"] = task.id
# Send to Amazon S3 if configured
if settings.s3_bucket_name and settings.aws_access_key_id and settings.aws_secret_access_key:
logger.info(f"Queueing {file_path} for S3 upload")
task = upload_to_s3.delay(file_path)
results["s3_task_id"] = task.id
return {
"status": "All upload tasks enqueued",
"file_path": file_path
"status": "Queued",
"file_path": file_path,
"tasks": results
}
app/tasks/upload_to_dropbox.py
+111 -33
View File
@@ -1,20 +1,38 @@
#!/usr/bin/env python3
import os
import logging
import requests
import dropbox
from dropbox.exceptions import ApiError, AuthError
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
logger = logging.getLogger(__name__)
def get_dropbox_access_token():
"""Refresh the Dropbox access token using the stored refresh token from ENV."""
# Check if needed settings are available
if not hasattr(settings, 'dropbox_refresh_token') or not settings.dropbox_refresh_token:
logger.error("Cannot refresh Dropbox token: Missing refresh token")
return None
if not hasattr(settings, 'dropbox_app_key') or not settings.dropbox_app_key:
logger.error("Cannot refresh Dropbox token: Missing app key")
return None
if not hasattr(settings, 'dropbox_app_secret') or not settings.dropbox_app_secret:
logger.error("Cannot refresh Dropbox token: Missing app secret")
return None
token_url = "https://api.dropbox.com/oauth2/token"
headers = {"Content-Type": "application/x-www-form-urlencoded"}
data = {
"grant_type": "refresh_token",
"refresh_token": settings.dropbox_refresh_token, # Now using ENV
"refresh_token": settings.dropbox_refresh_token,
"client_id": settings.dropbox_app_key,
"client_secret": settings.dropbox_app_secret,
}
@@ -25,50 +43,110 @@ def get_dropbox_access_token():
return response.json()["access_token"]
else:
error_msg = f"Failed to refresh Dropbox token: {response.status_code} - {response.text}"
print(f"[ERROR] {error_msg}")
logger.error(error_msg)
raise Exception(error_msg)
@celery.task(base=BaseTaskWithRetry)
def upload_to_dropbox(file_path: str):
"""Uploads a file to Dropbox using the API."""
"""
Upload a file to Dropbox.
"""
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
# Extract filename and set target path
error_msg = f"File not found: {file_path}"
logger.error(error_msg)
raise FileNotFoundError(error_msg)
# Check if Dropbox is properly configured
if not (hasattr(settings, 'dropbox_app_key') and settings.dropbox_app_key and
hasattr(settings, 'dropbox_app_secret') and settings.dropbox_app_secret and
hasattr(settings, 'dropbox_refresh_token') and settings.dropbox_refresh_token):
logger.info("Dropbox upload skipped: Missing configuration")
return {"status": "Skipped", "reason": "Dropbox settings not configured"}
filename = os.path.basename(file_path)
dropbox_path = f"{settings.dropbox_folder}/{filename}"
sanitized_filename = sanitize_filename(filename)
try:
# Get fresh access token
# Get access token from refresh token
access_token = get_dropbox_access_token()
if not access_token:
return {"status": "Failed", "reason": "Could not obtain access token"}
dbx = dropbox.Dropbox(access_token)
file_size = os.path.getsize(file_path)
chunk_size = 4 * 1024 * 1024 # 4MB chunk size
with open(file_path, "rb") as file_data:
if file_size <= chunk_size:
dbx.files_upload(file_data.read(), dropbox_path)
else:
upload_session_start_result = dbx.files_upload_session_start(file_data.read(chunk_size))
cursor = dropbox.files.UploadSessionCursor(
session_id=upload_session_start_result.session_id,
offset=file_data.tell(),
)
commit = dropbox.files.CommitInfo(path=dropbox_path)
# Calculate remote path based on local file structure
remote_base = settings.dropbox_folder or ""
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
# Function to check if file exists in Dropbox
def check_exists_in_dropbox(path):
try:
dbx.files_get_metadata(path)
return True
except ApiError as e:
if e.error.is_path() and e.error.get_path().is_not_found():
return False
raise
# Get a unique path in case of collision
remote_full_path = f"/{remote_path}" # Dropbox paths should start with /
remote_full_path = remote_full_path.replace('//', '/') # Clean double slashes
# Check for potential file collision and get a unique name if needed
dropbox_path = get_unique_filename(remote_full_path, check_exists_in_dropbox)
# Upload the file
logger.info(f"Uploading {filename} to Dropbox at {dropbox_path}")
with open(file_path, 'rb') as file_data:
# Use files_upload_session for large files to avoid timeouts
file_size = os.path.getsize(file_path)
if file_size > 10 * 1024 * 1024: # 10 MB threshold for chunked upload
cursor = None
chunk_size = 4 * 1024 * 1024 # 4 MB chunks
file_data.seek(0)
# Start upload session
session_start = dbx.files_upload_session_start(file_data.read(chunk_size))
cursor = dropbox.files.UploadSessionCursor(session_start.session_id, file_data.tell())
# Upload chunks until we reach the end
while file_data.tell() < file_size:
if (file_size - file_data.tell()) <= chunk_size:
dbx.files_upload_session_finish(file_data.read(chunk_size), cursor, commit)
# Last chunk
dbx.files_upload_session_finish(
file_data.read(chunk_size),
cursor,
dropbox.files.CommitInfo(path=dropbox_path, mode=dropbox.files.WriteMode.overwrite)
)
else:
# More chunks to upload
dbx.files_upload_session_append_v2(file_data.read(chunk_size), cursor)
cursor.offset = file_data.tell()
print(f"[INFO] Successfully uploaded {filename} to Dropbox at {dropbox_path}.")
return {"status": "Completed", "file": file_path}
except Exception as e:
error_msg = f"[ERROR] Failed to upload {filename} to Dropbox: {str(e)}"
print(error_msg)
else:
# Small file, direct upload
file_data.seek(0)
dbx.files_upload(
file_data.read(),
dropbox_path,
mode=dropbox.files.WriteMode.overwrite
)
logger.info(f"Successfully uploaded {filename} to Dropbox at {dropbox_path}")
return {
"status": "Completed",
"file_path": file_path,
"dropbox_path": dropbox_path
}
except AuthError:
error_msg = f"[ERROR] Authentication failed while uploading {filename} to Dropbox. Check token."
logger.error(error_msg)
raise Exception(error_msg)
except ApiError as e:
error_msg = f"[ERROR] Failed to upload {filename} to Dropbox: {e}"
logger.error(error_msg)
raise Exception(error_msg)
except Exception as e:
error_msg = f"[ERROR] Unexpected error uploading {filename} to Dropbox: {e}"
logger.error(error_msg)
raise Exception(error_msg)
app/tasks/upload_to_email.py
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env python3
import os
import json
import smtplib
import socket
import logging
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.application import MIMEApplication
from email.mime.image import MIMEImage
from datetime import datetime
from pathlib import Path
from jinja2 import Environment, FileSystemLoader, select_autoescape
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
logger = logging.getLogger(__name__)
def get_email_template(template_name="default.html"):
"""
Load email template from one of these locations in order of precedence:
1. Custom template from workdir/templates/email/
2. Default template from app/templates/email/
"""
# First try to load from workdir (user customizable location)
try:
workdir_template_path = os.path.join(settings.workdir, "templates", "email")
if os.path.exists(workdir_template_path):
env = Environment(
loader=FileSystemLoader(workdir_template_path),
autoescape=select_autoescape(['html', 'xml'])
)
env.globals['now'] = datetime.now # Add the now function to the Jinja environment
template = env.get_template(template_name)
logger.info(f"Using custom email template from workdir: {template_name}")
return template
except Exception as e:
logger.warning(f"Failed to load custom email template: {str(e)}")
# Fallback to built-in template
try:
# Get the app directory path (where this file is)
current_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
app_template_path = os.path.join(current_dir, "templates", "email")
env = Environment(
loader=FileSystemLoader(app_template_path),
autoescape=select_autoescape(['html', 'xml'])
)
env.globals['now'] = datetime.now # Add the now function to the Jinja environment
template = env.get_template(template_name)
logger.info(f"Using built-in email template: {template_name}")
return template
except Exception as e:
logger.error(f"Failed to load built-in email template: {str(e)}")
raise ValueError(f"Could not find any valid email template: {str(e)}")
def extract_metadata_from_file(file_path):
"""
Try to extract metadata from a file using several methods:
1. Check for a .json metadata file with the same name
2. Extract metadata from PDF if it's embedded
Returns a dictionary of metadata or None if not found
"""
metadata = {}
# Check for separate metadata JSON file
metadata_path = os.path.splitext(file_path)[0] + '.json'
if os.path.exists(metadata_path):
try:
with open(metadata_path, 'r', encoding='utf-8') as f:
metadata = json.load(f)
logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
return metadata
except Exception as e:
logger.warning(f"Failed to load metadata from JSON file: {str(e)}")
# TODO: For PDF files, try to extract embedded metadata using PyPDF2
# This would require additional dependencies, so for now we'll just check for external JSON
return metadata
def attach_logo(msg):
"""Attach the DocuNova logo to the email with proper Content-ID."""
try:
# Try to find logo in workdir first (for customization)
custom_logo_path = os.path.join(settings.workdir, "templates", "email", "logo.png")
if os.path.exists(custom_logo_path):
logo_path = custom_logo_path
else:
# Use built-in logo
app_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
logo_path = os.path.join(app_dir, "static", "logo.png")
# Fallback to logo in frontend/static if app/static doesn't exist
if not os.path.exists(logo_path):
logo_path = os.path.join(app_dir, "..", "frontend", "static", "logo.png")
if os.path.exists(logo_path):
with open(logo_path, 'rb') as img:
logo_data = img.read()
# Determine image MIME type based on extension
mimetype = 'image/svg+xml' if logo_path.endswith('.svg') else 'image/png'
logo_attach = MIMEImage(logo_data, mimetype)
logo_attach.add_header('Content-ID', '<logo>')
logo_attach.add_header('Content-Disposition', 'inline', filename='logo.png')
msg.attach(logo_attach)
logger.info(f"Logo attached from {logo_path}")
return True
else:
logger.warning("Could not find logo file")
return False
except Exception as e:
logger.warning(f"Error attaching logo: {str(e)}")
return False
@celery.task(base=BaseTaskWithRetry)
def upload_to_email(file_path: str, recipients=None, subject=None, message=None, template_name="default.html", include_metadata=True):
"""
Sends a file via email to the specified recipients.
If recipients is None, uses the configured default email recipient.
"""
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
# Extract filename
filename = os.path.basename(file_path)
# Check if email settings are configured
if not settings.email_host:
error_msg = "Email host is not configured"
logger.error(error_msg)
return {"status": "Skipped", "reason": error_msg}
# Log email configuration for debugging
logger.debug(f"Email config - Host: {settings.email_host}, Port: {settings.email_port}, "
f"Username: {settings.email_username}, TLS: {settings.email_use_tls}")
# Use provided recipients or fall back to default
if not recipients:
if not settings.email_default_recipient:
error_msg = "No recipients specified and no default recipient configured"
logger.error(error_msg)
return {"status": "Skipped", "reason": error_msg}
recipients = [settings.email_default_recipient]
elif isinstance(recipients, str):
recipients = [recipients] # Convert single email to list
# Use provided subject or create default
if not subject:
subject = f"DocuNova Document: {filename}"
# Extract document metadata if available
metadata = {}
if include_metadata:
metadata = extract_metadata_from_file(file_path)
try:
# Create the email
msg = MIMEMultipart('related') # Changed to 'related' to properly handle inline images
msg['From'] = settings.email_sender or settings.email_username
msg['To'] = ", ".join(recipients)
msg['Subject'] = subject
# Create alternative part for HTML content
alt_part = MIMEMultipart('alternative')
msg.attach(alt_part)
# Attach logo to the email
has_logo = attach_logo(msg)
# Load and render template
template = get_email_template(template_name)
# Context data for the template
context = {
"filename": filename,
"message": message or f"Attached is the document: {filename}",
"app_name": "DocuNova",
"app_url": f"https://{settings.external_hostname}" if settings.external_hostname else None,
"custom_message": message,
"metadata": metadata,
"has_metadata": bool(metadata),
"has_logo": has_logo,
"current_year": datetime.now().year
}
# Render HTML body
html_content = template.render(**context)
alt_part.attach(MIMEText(html_content, 'html'))
# Attach the file
with open(file_path, "rb") as file:
attachment = MIMEApplication(file.read(), _subtype="pdf")
attachment.add_header('Content-Disposition', f'attachment; filename="{filename}"')
msg.attach(attachment)
try:
# First try to resolve the hostname
socket.gethostbyname(settings.email_host)
# Connect to the SMTP server
with smtplib.SMTP(settings.email_host, settings.email_port, timeout=30) as server:
# Use TLS if specified
if settings.email_use_tls:
server.starttls()
# Login if credentials are provided
if settings.email_username and settings.email_password:
server.login(settings.email_username, settings.email_password)
# Send the email
server.send_message(msg)
logger.info(f"Successfully sent {filename} via email to {', '.join(recipients)}")
return {
"status": "Completed",
"file": file_path,
"recipients": recipients,
"subject": subject,
"metadata_included": bool(metadata),
"logo_included": has_logo
}
except socket.gaierror as e:
error_msg = f"Failed to resolve email host: {settings.email_host} - {str(e)}"
logger.error(error_msg)
return {"status": "Failed", "reason": error_msg, "error": str(e)}
except ConnectionRefusedError as e:
error_msg = f"Connection refused to SMTP server {settings.email_host}:{settings.email_port} - {str(e)}"
logger.error(error_msg)
return {"status": "Failed", "reason": error_msg, "error": str(e)}
except TimeoutError as e:
error_msg = f"Connection timeout to SMTP server {settings.email_host}:{settings.email_port} - {str(e)}"
logger.error(error_msg)
return {"status": "Failed", "reason": error_msg, "error": str(e)}
except Exception as e:
error_msg = f"Failed to send {filename} via email: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
app/tasks/upload_to_ftp.py
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
import os
import ftplib
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
import logging
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry)
def upload_to_ftp(file_path: str):
"""Uploads a file to an FTP server in the configured folder."""
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
# Extract filename
filename = os.path.basename(file_path)
# Check if FTP settings are configured
if not settings.ftp_host:
error_msg = "FTP host is not configured"
logger.error(error_msg)
raise ValueError(error_msg)
try:
# Connect to FTP server
ftp = ftplib.FTP()
ftp.connect(
host=settings.ftp_host,
port=settings.ftp_port or 21
)
# Login with credentials
ftp.login(
user=settings.ftp_username,
passwd=settings.ftp_password
)
# Change to target directory if specified
if settings.ftp_folder:
try:
# Try to navigate to the directory, create if it doesn't exist
ftp_folder = settings.ftp_folder
# Remove leading slash if present
if ftp_folder.startswith('/'):
ftp_folder = ftp_folder[1:]
# Try to change to the directory
try:
ftp.cwd(ftp_folder)
except ftplib.error_perm:
# Create directory structure if it doesn't exist
folders = ftp_folder.split('/')
current_dir = ''
for folder in folders:
if folder:
current_dir += f"/{folder}"
try:
ftp.cwd(current_dir)
except ftplib.error_perm:
ftp.mkd(current_dir)
ftp.cwd(current_dir)
except ftplib.Error as e:
error_msg = f"Failed to change/create directory on FTP server: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
# Upload the file
with open(file_path, 'rb') as file_data:
ftp.storbinary(f'STOR {filename}', file_data)
# Close FTP connection
ftp.quit()
logger.info(f"Successfully uploaded {filename} to FTP server at {settings.ftp_host}")
return {
"status": "Completed",
"file": file_path,
"ftp_host": settings.ftp_host,
"ftp_path": f"{settings.ftp_folder}/{filename}" if settings.ftp_folder else filename
}
except Exception as e:
error_msg = f"Failed to upload {filename} to FTP server: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
app/tasks/upload_to_google_drive.py
+103
View File
@@ -0,0 +1,103 @@
"""
app/tasks/upload_to_google_drive.py
"""
import os
import json
import logging
from googleapiclient.discovery import build
from googleapiclient.http import MediaFileUpload
from google.oauth2.service_account import Credentials
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
logger = logging.getLogger(__name__)
def get_google_drive_service():
"""
Authenticate with Google Drive API using service account credentials
and return an authorized service object.
"""
try:
# Load service account credentials from settings
if not settings.google_drive_credentials_json:
logger.error("Google Drive credentials not configured")
return None
credentials_dict = json.loads(settings.google_drive_credentials_json)
credentials = Credentials.from_service_account_info(
credentials_dict,
scopes=['https://www.googleapis.com/auth/drive']
)
# Delegate to user if specified
if settings.google_drive_delegate_to:
credentials = credentials.with_subject(settings.google_drive_delegate_to)
# Build and return the service
service = build('drive', 'v3', credentials=credentials)
return service
except Exception as e:
logger.error(f"Failed to authenticate with Google Drive: {str(e)}")
return None
@celery.task(base=BaseTaskWithRetry)
def upload_to_google_drive(file_path: str):
"""Uploads a file to Google Drive in the configured folder."""
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
# Extract filename from path
filename = os.path.basename(file_path)
try:
# Get Google Drive service
service = get_google_drive_service()
if not service:
raise Exception("Failed to initialize Google Drive service")
# Prepare the file metadata
file_metadata = {
'name': filename,
}
# If folder ID is specified, set parent folder
if settings.google_drive_folder_id:
file_metadata['parents'] = [settings.google_drive_folder_id]
# Upload file with metadata
media = MediaFileUpload(
file_path,
mimetype='application/pdf',
resumable=True
)
file = service.files().create(
body=file_metadata,
media_body=media,
fields='id,name,webViewLink'
).execute()
# Log success details
file_id = file.get('id')
web_view_link = file.get('webViewLink')
logger.info(f"Successfully uploaded {filename} to Google Drive with ID: {file_id}")
logger.info(f"File accessible at: {web_view_link}")
return {
"status": "Completed",
"file_path": file_path,
"google_drive_file_id": file_id,
"google_drive_web_link": web_view_link
}
except Exception as e:
error_msg = f"Failed to upload {filename} to Google Drive: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
app/tasks/upload_to_nextcloud.py
+116 -25
View File
@@ -1,37 +1,128 @@
#!/usr/bin/env python3
import os
import logging
import requests
from requests.auth import HTTPBasicAuth
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry)
def upload_to_nextcloud(file_path: str):
"""Uploads a file to Nextcloud in the configured folder."""
"""
Upload a file to Nextcloud WebDAV.
"""
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
# Extract filename
error_msg = f"File not found: {file_path}"
logger.error(error_msg)
raise FileNotFoundError(error_msg)
# For Nextcloud, we need to check for 'nextcloud_upload_url' instead of 'nextcloud_url'
# This is what's shown in your env view
if not (getattr(settings, 'nextcloud_upload_url', None) and
getattr(settings, 'nextcloud_username', None) and
getattr(settings, 'nextcloud_password', None)):
logger.info("Nextcloud upload skipped: Missing configuration")
return {"status": "Skipped", "reason": "Nextcloud settings not configured"}
filename = os.path.basename(file_path)
# Construct the full upload URL
nextcloud_url = f"{settings.nextcloud_upload_url}/{settings.nextcloud_folder}/{filename}"
# Read file content
with open(file_path, "rb") as file_data:
response = requests.put(
nextcloud_url,
auth=(settings.nextcloud_username, settings.nextcloud_password),
data=file_data
)
# Check if upload was successful
if response.status_code in (200, 201):
print(f"[INFO] Successfully uploaded {filename} to Nextcloud at {nextcloud_url}.")
return {"status": "Completed", "file": file_path}
else:
error_msg = f"[ERROR] Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
print(error_msg)
sanitized_filename = sanitize_filename(filename)
try:
# Prepare WebDAV URL - use nextcloud_upload_url instead of nextcloud_url
webdav_url = settings.nextcloud_upload_url
if not webdav_url.endswith('/'):
webdav_url += '/'
# Calculate remote path based on local file structure
remote_base = getattr(settings, 'nextcloud_folder', '') or ""
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
full_url = f"{webdav_url}/{remote_path}"
# Remove any double slashes (except in http://)
full_url = full_url.replace('://', '$PLACEHOLDER$')
while '//' in full_url:
full_url = full_url.replace('//', '/')
full_url = full_url.replace('$PLACEHOLDER$', '://')
# Function to check if file exists in Nextcloud
def check_exists_in_nextcloud(path):
check_url = f"{webdav_url}{os.path.dirname(path)}"
try:
response = requests.request(
'PROPFIND',
check_url,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
headers={'Depth': '1'},
timeout=10
)
return path in response.text
except Exception:
# If we can't check, assume it doesn't exist
return False
# Check for potential file collision and get a unique name if needed
remote_path = get_unique_filename(remote_path, check_exists_in_nextcloud)
full_url = f"{webdav_url}/{remote_path}"
# Fix double slashes again
full_url = full_url.replace('://', '$PLACEHOLDER$')
while '//' in full_url:
full_url = full_url.replace('//', '/')
full_url = full_url.replace('$PLACEHOLDER$', '://')
# Create necessary parent folders
parent_dirs = os.path.dirname(remote_path)
if parent_dirs:
current_path = ""
for folder in parent_dirs.split('/'):
if not folder:
continue
current_path += f"{folder}/"
mkdir_url = f"{webdav_url}/{current_path}"
# Fix double slashes
mkdir_url = mkdir_url.replace('://', '$PLACEHOLDER$')
while '//' in mkdir_url:
mkdir_url = mkdir_url.replace('//', '/')
mkdir_url = mkdir_url.replace('$PLACEHOLDER$', '://')
requests.request(
'MKCOL',
mkdir_url,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
timeout=10
)
# Upload the file
logger.info(f"Uploading {filename} to Nextcloud at {full_url}")
with open(file_path, 'rb') as file_data:
response = requests.put(
full_url,
data=file_data,
auth=HTTPBasicAuth(settings.nextcloud_username, settings.nextcloud_password),
headers={'Content-Type': 'application/octet-stream'},
timeout=60 # Longer timeout for larger files
)
if response.status_code in (201, 204): # Created or No Content
logger.info(f"Successfully uploaded {filename} to Nextcloud at {remote_path}")
return {
"status": "Completed",
"file_path": file_path,
"nextcloud_path": remote_path,
"response_code": response.status_code
}
else:
error_msg = f"[ERROR] Failed to upload {filename} to Nextcloud: {response.status_code} - {response.text}"
logger.error(error_msg)
raise Exception(error_msg)
except Exception as e:
error_msg = f"[ERROR] Failed to upload {filename} to Nextcloud: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
app/tasks/upload_to_onedrive.py
+206
View File
@@ -0,0 +1,206 @@
#!/usr/bin/env python3
import os
import time
import logging
import requests
import msal
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
logger = logging.getLogger(__name__)
def get_onedrive_token():
"""
Get an access token for Microsoft Graph API using the appropriate flow.
For personal accounts, uses refresh token flow.
For organizational accounts, uses client credentials flow if refresh token isn't provided.
"""
# Check for required settings
if not settings.onedrive_client_id or not settings.onedrive_client_secret:
raise ValueError("OneDrive client ID and client secret must be configured")
# Use refresh token flow (works for both personal and org accounts)
if settings.onedrive_refresh_token:
# Use MSAL to get token from refresh token
app = msal.PublicClientApplication(settings.onedrive_client_id)
# Request new token using refresh token
token_response = app.acquire_token_by_refresh_token(
refresh_token=settings.onedrive_refresh_token,
scopes=["https://graph.microsoft.com/Files.ReadWrite"]
)
if "access_token" not in token_response:
error = token_response.get("error", "")
error_desc = token_response.get("error_description", "Unknown error")
raise ValueError(f"Failed to get access token: {error} - {error_desc}")
return token_response["access_token"]
# No refresh token - try client credentials (only works for org accounts)
elif settings.onedrive_tenant_id and settings.onedrive_tenant_id != "common":
authority = f"https://login.microsoftonline.com/{settings.onedrive_tenant_id}"
app = msal.ConfidentialClientApplication(
client_id=settings.onedrive_client_id,
client_credential=settings.onedrive_client_secret,
authority=authority
)
# Acquire token for application
token_response = app.acquire_token_for_client(
scopes=["https://graph.microsoft.com/.default"]
)
if "access_token" not in token_response:
error = token_response.get("error", "")
error_desc = token_response.get("error_description", "Unknown error")
raise ValueError(f"Failed to get access token: {error} - {error_desc}")
return token_response["access_token"]
else:
raise ValueError("For personal Microsoft accounts, ONEDRIVE_REFRESH_TOKEN must be configured")
def create_upload_session(filename, folder_path, access_token):
"""Creates an upload session for large files in Microsoft Graph API."""
# Construct the API endpoint
base_url = "https://graph.microsoft.com/v1.0/me/drive"
# Format the folder path correctly
if folder_path:
# Remove leading/trailing slashes
folder_path = folder_path.strip('/')
# Replace spaces with %20
folder_path = folder_path.replace(' ', '%20')
item_path = f"/root:/{folder_path}/{filename}:/createUploadSession"
else:
item_path = f"/root:/{filename}:/createUploadSession"
url = f"{base_url}{item_path}"
headers = {
"Authorization": f"Bearer {access_token}",
"Content-Type": "application/json"
}
response = requests.post(url, headers=headers)
if response.status_code == 200:
return response.json().get("uploadUrl")
else:
error_msg = f"Failed to create upload session: {response.status_code} - {response.text}"
logger.error(error_msg)
raise Exception(error_msg)
def upload_large_file(file_path, upload_url):
"""
Upload a large file to OneDrive using the upload session URL.
Uses chunked upload for reliability.
"""
# Get file size
file_size = os.path.getsize(file_path)
# Define chunk size (10 MB)
chunk_size = 10 * 1024 * 1024
# Open and read file in chunks
with open(file_path, 'rb') as f:
# Process file in chunks
chunk_number = 0
while True:
chunk = f.read(chunk_size)
if not chunk:
break
# Get the position in the file
chunk_start = chunk_number * chunk_size
chunk_end = chunk_start + len(chunk) - 1
# Prepare content range header
content_range = f"bytes {chunk_start}-{chunk_end}/{file_size}"
# Upload chunk
headers = {
"Content-Length": str(len(chunk)),
"Content-Range": content_range
}
# Try to upload chunk with retries
max_retries = 3
retry_delay = 2 # seconds
for attempt in range(max_retries):
try:
response = requests.put(
upload_url,
headers=headers,
data=chunk
)
# Check if successful
if response.status_code in (201, 202):
# 201 = Created (final chunk), 202 = Accepted (more chunks coming)
break
else:
logger.warning(f"Chunk upload failed (attempt {attempt+1}): {response.status_code}")
if attempt < max_retries - 1:
time.sleep(retry_delay * (attempt + 1))
except Exception as e:
logger.warning(f"Chunk upload error (attempt {attempt+1}): {str(e)}")
if attempt < max_retries - 1:
time.sleep(retry_delay * (attempt + 1))
if response.status_code not in (201, 202):
raise Exception(f"Failed to upload chunk after {max_retries} attempts: {response.status_code} - {response.text}")
# Move to next chunk
chunk_number += 1
# If we get here, all chunks were uploaded successfully
# The last response should contain the file metadata
return response.json()
@celery.task(base=BaseTaskWithRetry)
def upload_to_onedrive(file_path: str):
"""Uploads a file to OneDrive in the configured folder."""
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
# Extract filename
filename = os.path.basename(file_path)
# Check if OneDrive settings are configured
if not settings.onedrive_client_id:
error_msg = "OneDrive client ID is not configured"
logger.error(error_msg)
raise ValueError(error_msg)
try:
# Get access token
access_token = get_onedrive_token()
# Create upload session
upload_url = create_upload_session(filename, settings.onedrive_folder_path, access_token)
# Upload the file
result = upload_large_file(file_path, upload_url)
# Log success
web_url = result.get("webUrl", "Not available")
logger.info(f"Successfully uploaded {filename} to OneDrive at path {settings.onedrive_folder_path}")
logger.info(f"File accessible at: {web_url}")
return {
"status": "Completed",
"file_path": file_path,
"onedrive_path": f"{settings.onedrive_folder_path}/{filename}",
"web_url": web_url
}
except Exception as e:
error_msg = f"Failed to upload {filename} to OneDrive: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
app/tasks/upload_to_paperless.py
+13 -13
View File
@@ -82,31 +82,31 @@ def poll_task_for_document_id(task_id: str) -> int:
)
@celery.task(base=BaseTaskWithRetry)
def upload_to_paperless(file_path: str) -> Dict[str, Any]:
"""
Uploads a PDF to Paperless with minimal metadata (filename and date only).
1. Extracts the filename and date from the file.
2. POSTs the PDF to Paperless => returns a quoted UUID string (task_id).
3. Polls /api/tasks/?task_id=<uuid> until SUCCESS or FAILURE => doc_id.
def upload_to_paperless(file_path: str):
"""Uploads a file to Paperless-ngx."""
Returns a dict with status, the paperless_task_id, paperless_document_id, and file_path.
"""
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
base_name = os.path.basename(file_path)
# Extract filename
filename = os.path.basename(file_path)
# Check if Paperless settings are configured
if not settings.paperless_host or not settings.paperless_ngx_api_token:
error_msg = "Paperless-ngx credentials are not fully configured"
logger.error(error_msg)
raise ValueError(error_msg)
# Upload the PDF
post_url = _paperless_api_url("/api/documents/post_document/")
with open(file_path, "rb") as f:
files = {
"document": (base_name, f, "application/pdf"),
"document": (filename, f, "application/pdf"),
}
data = {"title": base_name} # Title = Filename (no additional metadata)
data = {"title": filename} # Title = Filename (no additional metadata)
try:
logger.debug("Posting document to Paperless: file=%s", base_name)
logger.debug("Posting document to Paperless: file=%s", filename)
resp = requests.post(post_url, headers=_get_headers(), files=files, data=data)
resp.raise_for_status()
except requests.exceptions.RequestException as exc:
app/tasks/upload_to_s3.py
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env python3
import os
import logging
import boto3
from botocore.exceptions import ClientError
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry)
def upload_to_s3(file_path: str):
"""Uploads a file to Amazon S3 in the configured bucket and folder."""
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
# Extract filename
filename = os.path.basename(file_path)
# Check if S3 settings are configured
if not settings.s3_bucket_name:
error_msg = "S3 bucket name is not configured"
logger.error(error_msg)
raise ValueError(error_msg)
if not settings.aws_access_key_id or not settings.aws_secret_access_key:
error_msg = "AWS credentials are not configured"
logger.error(error_msg)
raise ValueError(error_msg)
try:
# Create S3 client
s3_client = boto3.client(
's3',
region_name=settings.aws_region,
aws_access_key_id=settings.aws_access_key_id,
aws_secret_access_key=settings.aws_secret_access_key
)
# Construct the S3 key (path within the bucket)
if settings.s3_folder_prefix:
# Ensure folder prefix ends with a slash
folder_prefix = settings.s3_folder_prefix
if not folder_prefix.endswith('/'):
folder_prefix += '/'
s3_key = f"{folder_prefix}{filename}"
else:
s3_key = filename
# Prepare extra arguments
extra_args = {
'StorageClass': settings.s3_storage_class
}
# Add ACL if configured
if settings.s3_acl:
extra_args['ACL'] = settings.s3_acl
# Upload file
s3_client.upload_file(
file_path,
settings.s3_bucket_name,
s3_key,
ExtraArgs=extra_args
)
# Generate URL to the file (useful for public files)
# For private files, this is just a reference and won't be accessible directly
s3_url = f"https://{settings.s3_bucket_name}.s3.{settings.aws_region}.amazonaws.com/{s3_key}"
logger.info(f"Successfully uploaded {filename} to S3 bucket {settings.s3_bucket_name} at path {s3_key}")
return {
"status": "Completed",
"file": file_path,
"s3_bucket": settings.s3_bucket_name,
"s3_key": s3_key,
"s3_url": s3_url
}
except ClientError as e:
error_msg = f"Failed to upload {filename} to S3: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
except Exception as e:
error_msg = f"Error uploading {filename} to S3: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
app/tasks/upload_to_sftp.py
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env python3
import os
import logging
import paramiko
from pathlib import Path
from app.config import settings
from app.celery_app import celery
from app.tasks.retry_config import BaseTaskWithRetry
from app.utils.filename_utils import get_unique_filename, sanitize_filename, extract_remote_path
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry)
def upload_to_sftp(file_path: str):
"""
Upload a file to an SFTP server.
"""
if not os.path.exists(file_path):
error_msg = f"File not found: {file_path}"
logger.error(error_msg)
raise FileNotFoundError(error_msg)
if not (settings.sftp_host and settings.sftp_port and settings.sftp_username):
logger.info("SFTP upload skipped: Missing configuration")
return {"status": "Skipped", "reason": "SFTP settings not configured"}
filename = os.path.basename(file_path)
sanitized_filename = sanitize_filename(filename)
# SSH client for SFTP connection
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
# Setup connection parameters
connect_kwargs = {
"hostname": settings.sftp_host,
"port": settings.sftp_port,
"username": settings.sftp_username,
}
# Check for authentication methods - use key if available, otherwise try password
sftp_key_path = getattr(settings, 'sftp_private_key', None)
sftp_key_passphrase = getattr(settings, 'sftp_private_key_passphrase', None)
if sftp_key_path and os.path.exists(sftp_key_path):
logger.info(f"Using SSH key authentication with key: {sftp_key_path}")
connect_kwargs["key_filename"] = sftp_key_path
if sftp_key_passphrase:
connect_kwargs["passphrase"] = sftp_key_passphrase
elif settings.sftp_password:
logger.info("Using password authentication for SFTP")
connect_kwargs["password"] = settings.sftp_password
else:
error_msg = "No authentication method available for SFTP (no key or password)"
logger.error(error_msg)
raise Exception(error_msg)
# Connect to the server
logger.info(f"Connecting to SFTP server at {settings.sftp_host}:{settings.sftp_port}")
ssh.connect(**connect_kwargs)
# Open SFTP session
sftp = ssh.open_sftp()
# Calculate remote path based on local file structure
remote_base = settings.sftp_folder or ""
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
# Function to check if file exists in SFTP server
def check_exists_in_sftp(path):
try:
sftp.stat(path)
return True
except FileNotFoundError:
return False
# Check for potential file collision and get a unique name if needed
remote_path = get_unique_filename(remote_path, check_exists_in_sftp)
# Create parent directories if needed
remote_dir = os.path.dirname(remote_path)
if remote_dir:
try:
# Try to create the full directory path
current_dir = ""
for dir_part in remote_dir.split("/"):
if not dir_part:
continue
current_dir += f"/{dir_part}"
try:
sftp.stat(current_dir)
except FileNotFoundError:
logger.info(f"Creating directory on SFTP server: {current_dir}")
sftp.mkdir(current_dir)
except Exception as e:
logger.warning(f"Failed to create directory structure {remote_dir}: {str(e)}")
# Upload the file
logger.info(f"Uploading {filename} to SFTP at {remote_path}")
sftp.put(file_path, remote_path)
logger.info(f"Successfully uploaded {filename} to SFTP at {remote_path}")
# Close connections
sftp.close()
ssh.close()
return {
"status": "Completed",
"file_path": file_path,
"sftp_path": remote_path
}
except Exception as e:
# Make sure connections are closed
try:
if 'sftp' in locals():
sftp.close()
ssh.close()
except:
pass
error_msg = f"Failed to upload {filename} to SFTP server: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
app/tasks/upload_to_webdav.py
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
import os
import requests
from urllib.parse import urljoin
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
import logging
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry)
def upload_to_webdav(file_path: str):
"""Uploads a file to a WebDAV server in the configured folder."""
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
# Extract filename
filename = os.path.basename(file_path)
# Check if WebDAV settings are configured
if not settings.webdav_url:
error_msg = "WebDAV URL is not configured"
logger.error(error_msg)
raise ValueError(error_msg)
# Construct the full upload URL
webdav_folder = settings.webdav_folder or ""
# Ensure folder doesn't have leading slash if we're joining it to the base URL
if webdav_folder and webdav_folder.startswith("/"):
webdav_folder = webdav_folder[1:]
# Join the base URL and folder path
target_url = urljoin(settings.webdav_url, webdav_folder)
# Ensure URL ends with a slash for proper joining with filename
if not target_url.endswith("/"):
target_url += "/"
# Construct final URL with filename
webdav_url = urljoin(target_url, filename)
# Read file content
try:
with open(file_path, "rb") as file_data:
response = requests.put(
webdav_url,
auth=(settings.webdav_username, settings.webdav_password),
data=file_data,
verify=settings.webdav_verify_ssl if hasattr(settings, "webdav_verify_ssl") else True
)
# Check if upload was successful
if response.status_code in (200, 201, 204):
logger.info(f"Successfully uploaded {filename} to WebDAV at {webdav_url}.")
return {"status": "Completed", "file": file_path, "url": webdav_url}
else:
error_msg = f"Failed to upload {filename} to WebDAV: {response.status_code} - {response.text}"
logger.error(error_msg)
raise Exception(error_msg)
except Exception as e:
error_msg = f"Error uploading {filename} to WebDAV: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
app/tasks/upload_with_rclone.py
+169
View File
@@ -0,0 +1,169 @@
#!/usr/bin/env python3
import os
import subprocess
import json
import tempfile
import logging
from pathlib import Path
from app.config import settings
from app.tasks.retry_config import BaseTaskWithRetry
from app.celery_app import celery
logger = logging.getLogger(__name__)
@celery.task(base=BaseTaskWithRetry)
def upload_with_rclone(file_path: str, destination: str):
"""
Uploads a file using rclone to the specified destination.
Args:
file_path: Path to the file to upload
destination: Rclone destination in format "remote:path/to/folder"
e.g. "gdrive:Uploads" or "dropbox:Documents/Uploads"
"""
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
# Extract filename
filename = os.path.basename(file_path)
# Check if rclone is installed and config exists
rclone_config_path = os.path.join(settings.workdir, "rclone.conf")
if not os.path.exists(rclone_config_path):
error_msg = f"Rclone configuration not found at {rclone_config_path}"
logger.error(error_msg)
raise ValueError(error_msg)
try:
# Split destination into remote and path
if ":" not in destination:
raise ValueError(f"Invalid destination format: {destination}. Expected format: remote:path")
remote, remote_path = destination.split(":", 1)
# Ensure the remote path exists (create folders if needed)
mkdir_cmd = [
"rclone",
"mkdir",
"--config", rclone_config_path,
destination
]
subprocess.run(mkdir_cmd, check=True, capture_output=True)
# Construct the upload command
upload_cmd = [
"rclone",
"copy",
"--config", rclone_config_path,
file_path,
destination,
"--progress"
]
# Execute the upload command
result = subprocess.run(upload_cmd, check=True, capture_output=True, text=True)
# Check if upload was successful
if result.returncode == 0:
# Try to get a public link if possible
try:
link_cmd = [
"rclone",
"link",
"--config", rclone_config_path,
f"{destination}/{filename}"
]
link_result = subprocess.run(link_cmd, capture_output=True, text=True)
public_url = link_result.stdout.strip() if link_result.returncode == 0 else None
except Exception:
public_url = None
logger.info(f"Successfully uploaded {filename} to {destination}")
return {
"status": "Completed",
"file": file_path,
"destination": destination,
"public_url": public_url
}
else:
error_msg = f"Failed to upload {filename} to {destination}: {result.stderr}"
logger.error(error_msg)
raise Exception(error_msg)
except subprocess.CalledProcessError as e:
error_msg = f"Rclone error: {e.stderr.decode('utf-8') if hasattr(e.stderr, 'decode') else e.stderr}"
logger.error(error_msg)
raise Exception(error_msg)
except Exception as e:
error_msg = f"Error uploading {filename} to {destination}: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
@celery.task(base=BaseTaskWithRetry)
def send_to_all_rclone_destinations(file_path: str):
"""
Uploads a file to all configured rclone destinations.
Destinations are loaded from the rclone configuration file.
"""
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
# Extract filename
filename = os.path.basename(file_path)
# Path to rclone config
rclone_config_path = os.path.join(settings.workdir, "rclone.conf")
if not os.path.exists(rclone_config_path):
error_msg = f"Rclone configuration not found at {rclone_config_path}"
logger.error(error_msg)
raise ValueError(error_msg)
# Get list of configured destinations from rclone
try:
remotes_cmd = ["rclone", "listremotes", "--config", rclone_config_path]
result = subprocess.run(remotes_cmd, check=True, capture_output=True, text=True)
if result.returncode == 0:
# Process the list of remotes
remotes = [r.strip() for r in result.stdout.splitlines() if r.strip()]
# Target directories for each remote (from settings)
remote_paths = {}
for remote in remotes:
remote_name = remote.rstrip(':')
path_setting_name = f"rclone_{remote_name}_path"
if hasattr(settings, path_setting_name) and getattr(settings, path_setting_name):
remote_paths[remote] = getattr(settings, path_setting_name)
else:
# Default to root of remote if not specified
remote_paths[remote] = ""
# Queue upload tasks for each configured destination
results = {}
for remote, path in remote_paths.items():
full_destination = f"{remote}{path}"
if path and not path.endswith('/'):
full_destination += '/'
logger.info(f"Queueing {file_path} for upload to {full_destination}")
task = upload_with_rclone.delay(file_path, full_destination)
results[f"rclone_{remote.rstrip(':')}_task_id"] = task.id
return {
"status": "Queued",
"file_path": file_path,
"tasks": results
}
else:
error_msg = f"Failed to list rclone remotes: {result.stderr}"
logger.error(error_msg)
raise Exception(error_msg)
except Exception as e:
error_msg = f"Error setting up rclone uploads for {filename}: {str(e)}"
logger.error(error_msg)
raise Exception(error_msg)
app/templates/email/.gitkeep
+2
View File
@@ -0,0 +1,2 @@
# This directory contains email templates
# The .gitkeep file ensures the directory is included in git
app/templates/email/default.html
+161
View File
@@ -0,0 +1,161 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Document from DocuNova</title>
<style>
body {
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
line-height: 1.6;
color: #333;
max-width: 600px;
margin: 0 auto;
padding: 20px;
}
.header {
text-align: center;
margin-bottom: 30px;
}
.logo {
max-width: 200px;
margin-bottom: 15px;
}
.container {
background-color: #f9f9f9;
border-radius: 8px;
padding: 25px;
border: 1px solid #e0e0e0;
}
.footer {
margin-top: 30px;
font-size: 12px;
color: #666;
text-align: center;
border-top: 1px solid #eee;
padding-top: 20px;
}
.button {
display: inline-block;
background-color: #3182ce;
color: white !important;
padding: 10px 20px;
text-decoration: none;
border-radius: 4px;
margin-top: 15px;
text-align: center;
}
.message {
margin-bottom: 20px;
}
.attachment-info {
background-color: #ebf8ff;
border-left: 4px solid #3182ce;
padding: 10px 15px;
margin-bottom: 20px;
}
.metadata-section {
background-color: #f0fff4;
border-left: 4px solid #38a169;
padding: 10px 15px;
margin: 20px 0;
}
.metadata-title {
font-weight: bold;
margin-bottom: 10px;
color: #2f855a;
}
.metadata-table {
width: 100%;
border-collapse: collapse;
}
.metadata-table th, .metadata-table td {
text-align: left;
padding: 8px;
border-bottom: 1px solid #ddd;
}
.metadata-table th {
width: 40%;
color: #4a5568;
}
.metadata-value {
word-break: break-word;
}
.document-type {
display: inline-block;
background-color: #3182ce;
color: white;
padding: 4px 10px;
border-radius: 12px;
font-size: 0.85em;
margin-top: 5px;
}
</style>
</head>
<body>
<div class="header">
{% if has_logo %}
<img src="cid:logo" alt="DocuNova Logo" class="logo">
{% endif %}
<h1>Document Delivery</h1>
</div>
<div class="container">
<p class="message">
{% if custom_message %}
{{ custom_message }}
{% else %}
A document has been shared with you from DocuNova.
{% endif %}
</p>
<div class="attachment-info">
<strong>Attached Document:</strong> {{ filename }}
</div>
{% if has_metadata %}
<div class="metadata-section">
<div class="metadata-title">Document Metadata</div>
{% if metadata.document_type %}
<span class="document-type">{{ metadata.document_type }}</span>
{% endif %}
<table class="metadata-table">
{% for key, value in metadata.items() %}
{% if key != "document_type" and value %}
<tr>
<th>{{ key | replace("_", " ") | title }}</th>
<td class="metadata-value">
{% if value is mapping %}
{% for subkey, subvalue in value.items() %}
<strong>{{ subkey | replace("_", " ") | title }}:</strong> {{ subvalue }}<br>
{% endfor %}
{% elif value is iterable and value is not string %}
{{ value | join(", ") }}
{% else %}
{{ value }}
{% endif %}
</td>
</tr>
{% endif %}
{% endfor %}
</table>
</div>
{% endif %}
<p>You can find the attached document in this email. Please let us know if you have any questions.</p>
{% if app_url %}
<p>
<a href="{{ app_url }}" class="button">Visit DocuNova</a>
</p>
{% endif %}
</div>
<div class="footer">
<p>This is an automated message from DocuNova Document Processing System.</p>
<p>© {{ now().year }} DocuNova</p>
</div>
</body>
</html>
app/utils.py
+5 -33
View File
@@ -1,35 +1,7 @@
# app/utils.py
import hashlib
from app.database import SessionLocal
from app.models import ProcessingLog
# This file is deprecated. Functions have been moved to the utils package.
# To avoid breaking existing imports, we'll import and re-export the functions
from app.utils.file_operations import hash_file
from app.utils.logging import log_task_progress
def hash_file(filepath, chunk_size=65536):
"""
Returns the SHA-256 hash of the file at 'filepath'.
Reads the file in chunks to handle large files efficiently.
"""
sha256 = hashlib.sha256()
with open(filepath, "rb") as f:
while True:
data = f.read(chunk_size)
if not data:
break
sha256.update(data)
return sha256.hexdigest()
def log_task_progress(task_id, step_name, status, message=None, file_id=None):
"""
Logs the progress of a Celery task to the database.
"""
with SessionLocal() as db:
log_entry = ProcessingLog(
task_id=task_id,
step_name=step_name,
status=status,
message=message,
file_id=file_id,
)
db.add(log_entry)
db.commit()
# These functions are now available directly from the app.utils package
app/utils/__init__.py
+10
View File
@@ -0,0 +1,10 @@
"""
Utility functions and helpers for the document processor application.
"""
# Import functions to make them available through the package
from app.utils.file_operations import hash_file
from app.utils.logging import log_task_progress
# Export all the functions that should be available when importing from app.utils
__all__ = ['hash_file', 'log_task_progress']
app/utils/config_validator.py
+324
View File
@@ -0,0 +1,324 @@
#!/usr/bin/env python3
import os
import socket
import logging
import inspect
from app.config import settings
logger = logging.getLogger(__name__)
def validate_email_config():
"""Validates email configuration settings"""
issues = []
# Check for required email settings
if not getattr(settings, 'email_host', None):
issues.append("EMAIL_HOST is not configured")
if not getattr(settings, 'email_port', None):
issues.append("EMAIL_PORT is not configured")
# Test SMTP server connectivity if host is configured
if getattr(settings, 'email_host', None) and getattr(settings, 'email_port', None):
try:
# Attempt to resolve the hostname
socket.gethostbyname(settings.email_host)
except socket.gaierror:
issues.append(f"Cannot resolve email host: {settings.email_host}")
# Check for authentication settings
if not getattr(settings, 'email_username', None):
issues.append("EMAIL_USERNAME is not configured")
if not getattr(settings, 'email_password', None):
issues.append("EMAIL_PASSWORD is not configured")
return issues
def validate_storage_configs():
"""Validates configuration for all storage providers"""
issues = {}
# Validate Dropbox config
dropbox_issues = []
if not (getattr(settings, 'dropbox_app_key', None) and
getattr(settings, 'dropbox_app_secret', None) and
getattr(settings, 'dropbox_refresh_token', None)):
dropbox_issues.append("Dropbox credentials are not fully configured")
issues['dropbox'] = dropbox_issues
# Validate Nextcloud config
nextcloud_issues = []
if not (getattr(settings, 'nextcloud_upload_url', None) and
getattr(settings, 'nextcloud_username', None) and
getattr(settings, 'nextcloud_password', None)):
nextcloud_issues.append("Nextcloud credentials are not fully configured")
issues['nextcloud'] = nextcloud_issues
# Validate SFTP config
sftp_issues = []
if not getattr(settings, 'sftp_host', None):
sftp_issues.append("SFTP_HOST is not configured")
sftp_key_path = getattr(settings, 'sftp_private_key', None)
if sftp_key_path and not os.path.exists(sftp_key_path):
sftp_issues.append(f"SFTP_KEY_PATH file not found: {sftp_key_path}")
if not sftp_key_path and not getattr(settings, 'sftp_password', None):
sftp_issues.append("Neither SFTP_KEY_PATH nor SFTP_PASSWORD is configured")
issues['sftp'] = sftp_issues
# Validate Email sending
email_issues = []
if not getattr(settings, 'email_host', None):
email_issues.append("EMAIL_HOST is not configured")
if not getattr(settings, 'email_default_recipient', None):
email_issues.append("EMAIL_DEFAULT_RECIPIENT is not configured")
issues['email'] = email_issues
# Validate S3
s3_issues = []
if not getattr(settings, 's3_bucket_name', None):
s3_issues.append("S3_BUCKET_NAME is not configured")
if not (getattr(settings, 'aws_access_key_id', None) and
getattr(settings, 'aws_secret_access_key', None)):
s3_issues.append("AWS credentials are not configured")
issues['s3'] = s3_issues
# Validate FTP
ftp_issues = []
if not getattr(settings, 'ftp_host', None):
ftp_issues.append("FTP_HOST is not configured")
if not getattr(settings, 'ftp_username', None):
ftp_issues.append("FTP_USERNAME is not configured")
if not getattr(settings, 'ftp_password', None):
ftp_issues.append("FTP_PASSWORD is not configured")
issues['ftp'] = ftp_issues
# Validate WebDAV
webdav_issues = []
if not getattr(settings, 'webdav_url', None):
webdav_issues.append("WEBDAV_URL is not configured")
if not getattr(settings, 'webdav_username', None):
webdav_issues.append("WEBDAV_USERNAME is not configured")
if not getattr(settings, 'webdav_password', None):
webdav_issues.append("WEBDAV_PASSWORD is not configured")
issues['webdav'] = webdav_issues
# Validate Google Drive
gdrive_issues = []
if not getattr(settings, 'google_drive_credentials_json', None):
gdrive_issues.append("GOOGLE_DRIVE_CREDENTIALS_JSON is not configured")
if not getattr(settings, 'google_drive_folder_id', None):
gdrive_issues.append("GOOGLE_DRIVE_FOLDER_ID is not configured")
issues['google_drive'] = gdrive_issues
# Validate Paperless
paperless_issues = []
if not getattr(settings, 'paperless_host', None):
paperless_issues.append("PAPERLESS_HOST is not configured")
if not getattr(settings, 'paperless_ngx_api_token', None):
paperless_issues.append("PAPERLESS_NGX_API_TOKEN is not configured")
issues['paperless'] = paperless_issues
# Validate OneDrive
onedrive_issues = []
if not (getattr(settings, 'onedrive_client_id', None) and
getattr(settings, 'onedrive_client_secret', None) and
getattr(settings, 'onedrive_refresh_token', None)):
onedrive_issues.append("OneDrive credentials are not fully configured")
issues['onedrive'] = onedrive_issues
return issues
def get_provider_status():
"""Get the status of each provider for the dashboard"""
providers = {
"Email": {
"configured": bool(getattr(settings, 'email_host', None) and
getattr(settings, 'email_username', None) and
getattr(settings, 'email_password', None)),
"icon": "mail",
"url": getattr(settings, 'email_host', None) or "",
"description": f"Send to {getattr(settings, 'email_default_recipient', 'Not configured')}"
},
"Dropbox": {
"configured": bool(getattr(settings, 'dropbox_app_key', None) and
getattr(settings, 'dropbox_app_secret', None) and
getattr(settings, 'dropbox_refresh_token', None)),
"icon": "dropbox",
"url": "https://dropbox.com",
"description": f"Upload to folder: {getattr(settings, 'dropbox_folder', 'Root')}"
},
"Nextcloud": {
"configured": bool(getattr(settings, 'nextcloud_upload_url', None) and
getattr(settings, 'nextcloud_username', None)),
"icon": "cloud",
"url": getattr(settings, 'nextcloud_upload_url', "").split('/remote.php')[0] if getattr(settings, 'nextcloud_upload_url', None) else "",
"description": f"Upload to folder: {getattr(settings, 'nextcloud_folder', 'Root')}"
},
"SFTP": {
"configured": bool(getattr(settings, 'sftp_host', None) and
getattr(settings, 'sftp_username', None) and
(getattr(settings, 'sftp_password', None) or getattr(settings, 'sftp_private_key', None))),
"icon": "server",
"url": f"sftp://{getattr(settings, 'sftp_host', '')}:{getattr(settings, 'sftp_port', 22)}",
"description": f"Upload to {getattr(settings, 'sftp_host', 'Not configured')}:{getattr(settings, 'sftp_folder', '/')}"
},
"Paperless": {
"configured": bool(getattr(settings, 'paperless_host', None) and
getattr(settings, 'paperless_ngx_api_token', None)),
"icon": "file-text",
"url": getattr(settings, 'paperless_host', ""),
"description": "Document management system"
},
"S3": {
"configured": bool(getattr(settings, 's3_bucket_name', None) and
getattr(settings, 'aws_access_key_id', None)),
"icon": "database",
"url": f"https://s3.console.aws.amazon.com/s3/buckets/{getattr(settings, 's3_bucket_name', '')}",
"description": f"Bucket: {getattr(settings, 's3_bucket_name', 'Not configured')}"
},
"FTP": {
"configured": bool(getattr(settings, 'ftp_host', None) and
getattr(settings, 'ftp_username', None)),
"icon": "hard-drive",
"url": f"ftp://{getattr(settings, 'ftp_host', '')}:{getattr(settings, 'ftp_port', 21)}",
"description": f"Upload to {getattr(settings, 'ftp_host', 'Not configured')}:{getattr(settings, 'ftp_folder', '/')}"
},
"WebDAV": {
"configured": bool(getattr(settings, 'webdav_url', None) and
getattr(settings, 'webdav_username', None)),
"icon": "globe",
"url": getattr(settings, 'webdav_url', ""),
"description": f"Upload to {getattr(settings, 'webdav_folder', '/')}"
},
"Google Drive": {
"configured": bool(getattr(settings, 'google_drive_credentials_json', None)),
"icon": "google",
"url": "https://drive.google.com",
"description": f"Folder ID: {getattr(settings, 'google_drive_folder_id', 'Not configured')}"
},
"OneDrive": {
"configured": bool(getattr(settings, 'onedrive_client_id', None) and
getattr(settings, 'onedrive_refresh_token', None)),
"icon": "microsoft",
"url": "https://onedrive.live.com",
"description": f"Upload to folder: {getattr(settings, 'onedrive_folder_path', 'Not configured')}"
}
}
return providers
def dump_all_settings():
"""Dump all settings to the log for debugging"""
logger.info("================ SETTINGS DUMP ================")
# Get all attributes from settings object
attributes = inspect.getmembers(settings, lambda a: not inspect.isroutine(a))
settings_dict = {a[0]: a[1] for a in attributes
if not a[0].startswith('_') and not callable(a[1])}
# Sort keys for better readability
for key in sorted(settings_dict.keys()):
value = settings_dict[key]
# Hide sensitive values
if any(sensitive in key.lower() for sensitive in ['password', 'secret', 'token', 'key']):
if value:
value = "******** [HIDDEN FOR SECURITY]"
logger.info(f" {key} = {value}")
# Also log all environment variables
logger.info("----------- ENVIRONMENT VARIABLES -----------")
env_vars_to_log = {}
for key in sorted(os.environ.keys()):
value = os.environ[key]
# Hide sensitive values
if any(sensitive in key.lower() for sensitive in ['password', 'secret', 'token', 'key']):
if value:
value = "******** [HIDDEN FOR SECURITY]"
env_vars_to_log[key] = value
for key in sorted(env_vars_to_log.keys()):
logger.info(f" {key} = {env_vars_to_log[key]}")
logger.info("=============================================")
def get_settings_for_display(show_values=False):
"""Get all settings organized by category for display in UI"""
# Get all attributes from settings object
attributes = inspect.getmembers(settings, lambda a: not inspect.isroutine(a))
settings_dict = {a[0]: a[1] for a in attributes
if not a[0].startswith('_') and not callable(a[1])}
# Categorize settings
categories = {
"Core": [],
"Email": [],
"IMAP": [],
"Storage": [],
"Authentication": [],
"Integration": [],
"Other": []
}
# Sort keys for better readability
for key in sorted(settings_dict.keys()):
value = settings_dict[key]
# Mask sensitive values if show_values is False
display_value = value
if not show_values or any(sensitive in key.lower() for sensitive in ['password', 'secret', 'token', 'key']):
if value:
display_value = "******** [HIDDEN]"
else:
display_value = None
# Categorize by key prefix
setting_item = {"name": key, "value": display_value, "is_configured": value is not None and value != ""}
if key.startswith(('email_', 'smtp_')):
categories["Email"].append(setting_item)
elif key.startswith('imap'):
categories["IMAP"].append(setting_item)
elif key.startswith(('s3_', 'aws_', 'dropbox_', 'nextcloud_', 'sftp_', 'ftp_', 'google_drive_')):
categories["Storage"].append(setting_item)
elif key.startswith(('auth_', 'jwt_', 'oauth_')):
categories["Authentication"].append(setting_item)
elif key.startswith(('paperless_', 'tesseract_', 'azure_')):
categories["Integration"].append(setting_item)
elif key in ('workdir', 'external_hostname', 'debug', 'version', 'env', 'log_level'):
categories["Core"].append(setting_item)
else:
categories["Other"].append(setting_item)
# Remove empty categories
return {k: v for k, v in categories.items() if v}
def check_all_configs():
"""Run all configuration validations and log results"""
logger.info("Validating application configuration...")
# Check if debug is enabled and dump all settings if it is
if hasattr(settings, 'debug') and settings.debug:
dump_all_settings()
# Check email config
email_issues = validate_email_config()
if email_issues:
logger.warning(f"Email configuration issues: {', '.join(email_issues)}")
else:
logger.info("Email configuration OK")
# Check storage configs
storage_issues = validate_storage_configs()
for provider, issues in storage_issues.items():
if issues:
logger.warning(f"{provider.capitalize()} configuration issues: {', '.join(issues)}")
else:
logger.info(f"{provider.capitalize()} configuration OK")
# Return all identified issues
return {
'email': email_issues,
'storage': storage_issues
}
app/utils/file_operations.py
+15
View File
@@ -0,0 +1,15 @@
import hashlib
def hash_file(filepath, chunk_size=65536):
"""
Returns the SHA-256 hash of the file at 'filepath'.
Reads the file in chunks to handle large files efficiently.
"""
sha256 = hashlib.sha256()
with open(filepath, "rb") as f:
while True:
data = f.read(chunk_size)
if not data:
break
sha256.update(data)
return sha256.hexdigest()
app/utils/filename_utils.py
+131
View File
@@ -0,0 +1,131 @@
import os
import re
import uuid
import logging
from datetime import datetime
from pathlib import Path
logger = logging.getLogger(__name__)
def get_unique_filename(original_path, check_exists_func=None):
"""
Generates a unique filename by appending a timestamp or counter when a collision occurs.
Args:
original_path (str): The original file path
check_exists_func (callable): Function that checks if file exists in target system.
Takes a path string and returns True if exists, False otherwise.
If None, will use local filesystem check.
Returns:
str: A unique filename that doesn't collide with existing files
"""
if check_exists_func is None:
check_exists_func = os.path.exists
path = Path(original_path)
directory = str(path.parent)
filename = path.name
name, ext = os.path.splitext(filename)
# If file doesn't exist, return the original
if not check_exists_func(original_path):
return original_path
# Try timestamp-based suffix first (more user-friendly)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
new_filename = f"{name}_{timestamp}{ext}"
new_path = os.path.join(directory, new_filename)
if not check_exists_func(new_path):
logger.info(f"Renamed '{filename}' to '{new_filename}' to avoid collision")
return new_path
# If timestamp-based name also exists, try random UUID
uuid_str = str(uuid.uuid4())[:8] # Use first 8 chars of UUID for brevity
new_filename = f"{name}_{uuid_str}{ext}"
new_path = os.path.join(directory, new_filename)
if not check_exists_func(new_path):
logger.info(f"Renamed '{filename}' to '{new_filename}' using UUID to avoid collision")
return new_path
# If that still exists (very unlikely), use incremental numbering
counter = 1
while counter < 1000: # Limit to avoid infinite loop
new_filename = f"{name}_{counter}{ext}"
new_path = os.path.join(directory, new_filename)
if not check_exists_func(new_path):
logger.info(f"Renamed '{filename}' to '{new_filename}' using counter to avoid collision")
return new_path
counter += 1
# If we got here, something is weird - just use a full UUID
new_filename = f"{name}_{str(uuid.uuid4())}{ext}"
new_path = os.path.join(directory, new_filename)
logger.warning(f"Had to use full UUID to rename '{filename}' to '{new_filename}'")
return new_path
def sanitize_filename(filename):
"""
Sanitize a filename to ensure it's valid across different file systems.
Args:
filename (str): The filename to sanitize
Returns:
str: A sanitized filename
"""
# Replace characters that are problematic in various filesystems
# Keep only alphanumeric, dash, underscore, period, and space
sanitized = re.sub(r'[^\w\-\. ]', '_', filename)
# Replace multiple spaces/underscores with single ones
sanitized = re.sub(r'__+', '_', sanitized)
sanitized = re.sub(r' +', ' ', sanitized)
# Trim leading/trailing spaces and periods which cause issues in Windows
sanitized = sanitized.strip('. ')
# Ensure the filename isn't empty after sanitization
if not sanitized or sanitized == '.':
sanitized = f"document_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
return sanitized
def extract_remote_path(local_path, base_dir, remote_base=None):
"""
Extracts the appropriate remote path based on a local path structure.
Args:
local_path (str): The local file path
base_dir (str): The local base directory to remove from path
remote_base (str, optional): Remote base directory to prepend
Returns:
str: The calculated remote path
"""
# Convert both paths to use forward slashes for consistency
local_path = local_path.replace('\\', '/')
base_dir = base_dir.replace('\\', '/')
# Make sure base_dir ends with a slash
if not base_dir.endswith('/'):
base_dir += '/'
# Remove the base directory from the local path
if local_path.startswith(base_dir):
relative_path = local_path[len(base_dir):]
else:
# If local_path is not within base_dir, just use the filename
relative_path = os.path.basename(local_path)
# Prepend the remote base if provided
if remote_base:
# Ensure remote_base ends with slash
if not remote_base.endswith('/'):
remote_base += '/'
return remote_base + relative_path
return relative_path
app/utils/logging.py
+17
View File
@@ -0,0 +1,17 @@
from app.database import SessionLocal
from app.models import ProcessingLog
def log_task_progress(task_id, step_name, status, message=None, file_id=None):
"""
Logs the progress of a Celery task to the database.
"""
with SessionLocal() as db:
log_entry = ProcessingLog(
task_id=task_id,
step_name=step_name,
status=status,
message=message,
file_id=file_id,
)
db.add(log_entry)
db.commit()
docker-compose.yaml
+6 -7
View File
@@ -1,7 +1,8 @@
services:
api:
image: christianlouis/document-processor:latest
build:
context: .
dockerfile: Dockerfile
container_name: document_api
# We'll keep the code in /app, but set working_dir to the shared data directory
@@ -26,12 +27,12 @@ services:
# Mount the shared working directory for data
volumes:
# optional: mount your code if you want local dev changes to reflect
# - ./app:/app
- /var/docparse/workdir:/workdir
worker:
image: christianlouis/document-processor:latest
build:
context: .
dockerfile: Dockerfile
container_name: document_worker
# same shared working directory
@@ -49,8 +50,6 @@ services:
# Mount the shared directory (and optionally your code if you want dev mode)
volumes:
# optional: mount your code if you want local dev changes
# - ./app:/app
- /var/docparse/workdir:/workdir
gotenberg:
docs/API.md
+15
View File
@@ -101,6 +101,21 @@ Reprocess a specific file.
}
```
**POST** `/send_to_google_drive/`
Send a processed file to Google Drive.
**Parameters**:
- `file_path`: Path to the file to upload
**Response**:
```json
{
"task_id": "a1b2c3d4-e5f6-7g8h-9i0j-k1l2m3n4o5p6",
"status": "queued"
}
```
## Error Handling
Errors follow standard HTTP status codes with descriptive messages:
docs/ConfigurationGuide.md
+438 -6
View File
@@ -55,12 +55,12 @@ DocuNova can monitor multiple IMAP mailboxes for document attachments. Each mail
### Dropbox
| **Variable** | **Description** | **How to Obtain** |
|-------------------------|--------------------------------------------------|------------------------------------------------------------------------------------|
| `DROPBOX_APP_KEY` | Dropbox API app key. | [Dropbox Developer Console](https://www.dropbox.com/developers/apps/create) |
| `DROPBOX_APP_SECRET` | Dropbox API app secret. | [Dropbox Developer Console](https://www.dropbox.com/developers/apps/create) |
| `DROPBOX_REFRESH_TOKEN` | OAuth2 refresh token for Dropbox. | Follow Dropbox OAuth flow to retrieve |
| `DROPBOX_FOLDER` | Default folder path for Dropbox uploads. | e.g. `"/Documents/Uploads"` |
| **Variable** | **Description** | **How to Obtain** |
|-------------------------|--------------------------------------------------|---------------------------------------------------------|
| `DROPBOX_APP_KEY` | Dropbox API app key. | [Dropbox Developer Console](#setting-up-dropbox-integration) |
| `DROPBOX_APP_SECRET` | Dropbox API app secret. | [Dropbox Developer Console](#setting-up-dropbox-integration) |
| `DROPBOX_REFRESH_TOKEN` | OAuth2 refresh token for Dropbox. | Follow steps in [Dropbox Setup](#setting-up-dropbox-integration) |
| `DROPBOX_FOLDER` | Default folder path for Dropbox uploads. | e.g. `"/Documents/Uploads"` (leading slash optional) |
### Nextcloud
@@ -71,6 +71,221 @@ DocuNova can monitor multiple IMAP mailboxes for document attachments. Each mail
| `NEXTCLOUD_PASSWORD` | Nextcloud login password. |
| `NEXTCLOUD_FOLDER` | Destination folder in Nextcloud (e.g. `"/Documents/Uploads"`). |
### Google Drive
| **Variable** | **Description** | **How to Obtain** |
|---------------------------------|-------------------------------------------------------|------------------------------------------------------|
| `GOOGLE_DRIVE_CREDENTIALS_JSON` | JSON string containing service account credentials | [Google Cloud Console](#setting-up-google-drive-api) |
| `GOOGLE_DRIVE_FOLDER_ID` | Google Drive folder ID for file uploads | See [folder ID instructions](#get-google-drive-folder-id) |
| `GOOGLE_DRIVE_DELEGATE_TO` | Email address to delegate permissions (optional) | User email in your Google Workspace |
### WebDAV
| **Variable** | **Description** |
|-------------------------|---------------------------------------------------------------|
| `WEBDAV_URL` | WebDAV server URL (e.g. `https://webdav.example.com/path`). |
| `WEBDAV_USERNAME` | WebDAV authentication username. |
| `WEBDAV_PASSWORD` | WebDAV authentication password. |
| `WEBDAV_FOLDER` | Destination folder on WebDAV server (e.g. `"/Documents/Uploads"`). |
| `WEBDAV_VERIFY_SSL` | Whether to verify SSL certificates (default: `True`). |
### FTP
| **Variable** | **Description** |
|-------------------------|---------------------------------------------------------------|
| `FTP_HOST` | FTP server hostname or IP address. |
| `FTP_PORT` | FTP port (default: `21`). |
| `FTP_USERNAME` | FTP authentication username. |
| `FTP_PASSWORD` | FTP authentication password. |
| `FTP_FOLDER` | Destination folder on FTP server (e.g. `"/Documents/Uploads"`). |
### SFTP
| **Variable** | **Description** |
|------------------------------|-------------------------------------------------------|
| `SFTP_HOST` | SFTP server hostname or IP address. |
| `SFTP_PORT` | SFTP port (default: `22`). |
| `SFTP_USERNAME` | SFTP authentication username. |
| `SFTP_PASSWORD` | SFTP authentication password (if not using private key). |
| `SFTP_FOLDER` | Destination folder on SFTP server. |
| `SFTP_PRIVATE_KEY` | Path to private key file for authentication (optional). |
| `SFTP_PRIVATE_KEY_PASSPHRASE`| Passphrase for private key if required (optional). |
### Email
| **Variable** | **Description** |
|----------------------------|----------------------------------------------------------|
| `EMAIL_HOST` | SMTP server hostname. |
| `EMAIL_PORT` | SMTP port (default: `587`). |
| `EMAIL_USERNAME` | SMTP authentication username. |
| `EMAIL_PASSWORD` | SMTP authentication password. |
| `EMAIL_USE_TLS` | Whether to use TLS (default: `True`). |
| `EMAIL_SENDER` | From address (e.g., `"DocuNova <docunova@example.com>"`). |
| `EMAIL_DEFAULT_RECIPIENT` | Default recipient email if none specified in the task. |
### OneDrive / Microsoft Graph
| **Variable** | **Description** | **How to Obtain** |
|---------------------------------|-------------------------------------------------------|------------------------------------------------------|
| `ONEDRIVE_CLIENT_ID` | Azure AD application client ID | [Microsoft Azure Portal](#setting-up-onedrive-integration) |
| `ONEDRIVE_CLIENT_SECRET` | Azure AD application client secret | [Microsoft Azure Portal](#setting-up-onedrive-integration) |
| `ONEDRIVE_TENANT_ID` | Azure AD tenant ID: use "common" for personal accounts or your tenant ID for corporate accounts | [Microsoft Azure Portal](#setting-up-onedrive-integration) |
| `ONEDRIVE_REFRESH_TOKEN` | OAuth 2.0 refresh token (required for personal accounts) | Follow steps in [Personal OneDrive Setup](#personal-onedrive-setup) |
| `ONEDRIVE_FOLDER_PATH` | Folder path in OneDrive for storing documents | e.g. `/Documents/Uploads` or `Documents/Uploads` |
## Setting up OneDrive Integration
There are two main types of Microsoft accounts that can be used with OneDrive integration:
1. **Personal Microsoft Accounts** - These include accounts with @outlook.com, @hotmail.com, @live.com domains, or personal Microsoft accounts linked to other email addresses (like Gmail)
2. **Work/School Microsoft Accounts** - These are accounts managed by an organization through Microsoft 365 or Azure Active Directory
The setup process differs slightly based on which account type you're using.
### Common Setup Steps (All Account Types)
1. **Register an application in Azure Active Directory**:
- Go to the [Azure Portal](https://portal.azure.com/)
- Navigate to "Azure Active Directory" > "App registrations"
- Click "New registration"
- Enter a name for your application (e.g., "DocuNova")
- For "Supported account types", select the appropriate option:
- For personal accounts: "Accounts in any organizational directory and personal Microsoft accounts"
- For corporate accounts only: "Accounts in this organizational directory only"
- For Redirect URI, select "Web" and enter a URL you can access (e.g., `http://localhost:8000/auth/callback`)
- Click "Register"
2. **Get Application (client) ID**:
- After registration, note the "Application (client) ID" from the overview page
- Set this value as `ONEDRIVE_CLIENT_ID`
3. **Create a client secret**:
- In your application page, go to "Certificates & secrets"
- Under "Client secrets," click "New client secret"
- Add a description and select an expiration period
- Click "Add" and immediately copy the secret value (it will only be shown once)
- Set this value as `ONEDRIVE_CLIENT_SECRET`
### For Personal Microsoft Accounts
If you're using a personal Microsoft account (@outlook.com, @hotmail.com, or personal accounts linked to other emails):
1. **Set Tenant ID to "common"**:
- Set `ONEDRIVE_TENANT_ID=common` in your configuration
2. **Configure API permissions**:
- In your application page, go to "API permissions"
- Click "Add a permission"
- Select "Microsoft Graph" > "Delegated permissions"
- Search for and add the following permissions:
- `Files.ReadWrite` (Allows the app to read and write files that the user has access to)
- `offline_access` (Needed for refresh tokens)
- Click "Add permissions"
3. **Generate a Refresh Token**:
- Use the following URL (replace CLIENT_ID and REDIRECT_URI with your values):
```
https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=YOUR_CLIENT_ID&response_type=code&redirect_uri=YOUR_REDIRECT_URI&response_mode=query&scope=offline_access%20Files.ReadWrite
```
- Open this URL in your browser
- Sign in with your personal Microsoft account
- After authentication, you'll be redirected to your redirect URI with a code parameter in the URL
- Copy the code value from the URL (everything after "code=")
4. **Exchange Code for Refresh Token**:
- Use the following command to exchange the code for tokens:
```bash
curl -X POST https://login.microsoftonline.com/common/oauth2/v2.0/token \
-H "Content-Type: application/x-www-form-urlencoded" \
-d "client_id=YOUR_CLIENT_ID&scope=offline_access Files.ReadWrite&code=YOUR_AUTH_CODE&redirect_uri=YOUR_REDIRECT_URI&grant_type=authorization_code&client_secret=YOUR_CLIENT_SECRET"
```
- From the response JSON, copy the `refresh_token` value
- Set this as `ONEDRIVE_REFRESH_TOKEN` in your configuration
### For Corporate Microsoft Accounts
If you're using a work/school account provided by your organization:
1. **Get your Tenant ID**:
- In the Azure Portal, find your "Tenant ID" (also called "Directory ID")
- It will be in the Azure Active Directory overview or properties section
- Set this value as `ONEDRIVE_TENANT_ID` in your configuration
2. **Configuration based on use case**:
**Option A: Access your own OneDrive (Interactive Login)**
This option requires a refresh token just like personal accounts:
- Follow the same steps as for personal accounts, but use your work email to sign in
- Make sure to set `ONEDRIVE_TENANT_ID` to your organization's tenant ID instead of "common"
- Set the refresh token you receive as `ONEDRIVE_REFRESH_TOKEN`
**Option B: Access OneDrive as a system service (App-only access)**
This option is for service accounts or automated systems with no user interaction:
- In API permissions, add "Application permissions" instead of "Delegated permissions"
- Add `Files.ReadWrite.All` permission under "Application permissions"
- Click "Grant admin consent" (requires admin privileges)
- In this case, `ONEDRIVE_REFRESH_TOKEN` is not needed as the app will use client credentials flow
- Note: This approach can only access specific shared folders or sites, not personal OneDrives
### Troubleshooting OAuth Login Issues
If you encounter errors during authentication:
1. **Check account permissions**:
- Ensure your Microsoft account has the necessary permissions to grant access
- For corporate accounts, check if your admin has restricted third-party app access
2. **Permission errors**:
- Verify the app registration has the correct API permissions
- For corporate accounts, ensure an admin has consented to the permissions
3. **Refresh token expired**:
- If uploads stop working, you may need to generate a new refresh token
- Repeat the process to get a new authorization code and refresh token
### Configuration Examples
**Personal Microsoft Account:**
```dotenv
ONEDRIVE_CLIENT_ID=12345678-1234-1234-1234-123456789012
ONEDRIVE_CLIENT_SECRET=your_client_secret
ONEDRIVE_TENANT_ID=common
ONEDRIVE_REFRESH_TOKEN=your_refresh_token
ONEDRIVE_FOLDER_PATH=Documents/Uploads
```
**Corporate Account with Interactive Login:**
```dotenv
ONEDRIVE_CLIENT_ID=12345678-1234-1234-1234-123456789012
ONEDRIVE_CLIENT_SECRET=your_client_secret
ONEDRIVE_TENANT_ID=87654321-4321-4321-4321-210987654321
ONEDRIVE_REFRESH_TOKEN=your_refresh_token
ONEDRIVE_FOLDER_PATH=Documents/Uploads
```
**Corporate Account with App-Only Access:**
```dotenv
ONEDRIVE_CLIENT_ID=12345678-1234-1234-1234-123456789012
ONEDRIVE_CLIENT_SECRET=your_client_secret
ONEDRIVE_TENANT_ID=87654321-4321-4321-4321-210987654321
# No refresh token needed for app-only access
ONEDRIVE_FOLDER_PATH=Documents/Uploads
```
### Amazon S3
| **Variable** | **Description** | **How to Obtain** |
|---------------------------------|-------------------------------------------------------|------------------------------------------------------|
| `AWS_ACCESS_KEY_ID` | AWS IAM access key ID | [AWS IAM Console](#setting-up-amazon-s3-integration) |
| `AWS_SECRET_ACCESS_KEY` | AWS IAM secret access key | [AWS IAM Console](#setting-up-amazon-s3-integration) |
| `AWS_REGION` | AWS region where your S3 bucket is located (default: `us-east-1`) | [AWS S3 Console](https://s3.console.aws.amazon.com/) |
| `S3_BUCKET_NAME` | Name of your S3 bucket | [AWS S3 Console](https://s3.console.aws.amazon.com/) |
| `S3_FOLDER_PREFIX` | Optional prefix/folder path for uploaded files | e.g. `documents/` or `uploads/2023/` (include trailing slash) |
| `S3_STORAGE_CLASS` | Storage class for uploaded objects (default: `STANDARD`) | [S3 Storage Classes](https://aws.amazon.com/s3/storage-classes/) |
| `S3_ACL` | Access control for uploaded files (default: `private`) | `private`, `public-read`, etc. |
## Configuration Examples
### Minimal Configuration
@@ -127,8 +342,225 @@ NEXTCLOUD_UPLOAD_URL=https://nc.example.com/remote.php/dav/files/username
NEXTCLOUD_USERNAME=username
NEXTCLOUD_PASSWORD=password
NEXTCLOUD_FOLDER=/Documents/Uploads
# Google Drive
GOOGLE_DRIVE_CREDENTIALS_JSON={"type":"service_account","project_id":"..."}
GOOGLE_DRIVE_FOLDER_ID=1a2b3c4d5e6f7g8h9i0j
GOOGLE_DRIVE_DELEGATE_TO=optional-user@example.com
# WebDAV
WEBDAV_URL=https://webdav.example.com/path
WEBDAV_USERNAME=username
WEBDAV_PASSWORD=password
WEBDAV_FOLDER=/Documents/Uploads
WEBDAV_VERIFY_SSL=True
# FTP
FTP_HOST=ftp.example.com
FTP_PORT=21
FTP_USERNAME=username
FTP_PASSWORD=password
FTP_FOLDER=/Documents/Uploads
# SFTP
SFTP_HOST=sftp.example.com
SFTP_PORT=22
SFTP_USERNAME=username
SFTP_PASSWORD=password
SFTP_FOLDER=/Documents/Uploads
# SFTP_PRIVATE_KEY=/path/to/key.pem
# SFTP_PRIVATE_KEY_PASSPHRASE=passphrase
# Email
EMAIL_HOST=smtp.example.com
EMAIL_PORT=587
EMAIL_USERNAME=docunova@example.com
EMAIL_PASSWORD=password
EMAIL_USE_TLS=True
EMAIL_SENDER=DocuNova System <docunova@example.com>
EMAIL_DEFAULT_RECIPIENT=recipient@example.com
# OneDrive (Personal Account)
ONEDRIVE_CLIENT_ID=12345678-1234-1234-1234-123456789012
ONEDRIVE_CLIENT_SECRET=your_client_secret
ONEDRIVE_TENANT_ID=common
ONEDRIVE_REFRESH_TOKEN=your_refresh_token
ONEDRIVE_FOLDER_PATH=Documents/Uploads
# Amazon S3
AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
AWS_REGION=us-east-1
S3_BUCKET_NAME=my-document-bucket
S3_FOLDER_PREFIX=documents/uploads/2023/ # Will place files in this subfolder
S3_STORAGE_CLASS=STANDARD
S3_ACL=private
```
## Setting up Google Drive API
To use the Google Drive integration, follow these steps:
1. **Create a Google Cloud Project**:
- Go to the [Google Cloud Console](https://console.cloud.google.com/)
- Create a new project or select an existing one
- Navigate to "APIs & Services" > "Library"
- Search for and enable the "Google Drive API"
2. **Create Service Account**:
- Go to "APIs & Services" > "Credentials"
- Click "Create Credentials" > "Service Account"
- Fill in the service account details and click "Create"
- Add appropriate roles (e.g., "Editor" for full access)
- Click "Continue" and then "Done"
3. **Generate Service Account Key**:
- Find your service account in the list and click on it
- Go to the "Keys" tab
- Click "Add Key" > "Create New Key"
- Choose JSON format and click "Create"
- The key file will be downloaded automatically
4. **Configure DocuNova**:
- Open the downloaded JSON key file
- Set the entire JSON content as the `GOOGLE_DRIVE_CREDENTIALS_JSON` environment variable
- For security, ensure the JSON is properly escaped if your deployment method requires it
### Get Google Drive Folder ID
To find your Google Drive folder ID:
1. Navigate to the desired folder in Google Drive web interface
2. The URL will look like: `https://drive.google.com/drive/folders/1a2b3c4d5e6f7g8h9i0j`
3. The string after "folders/" is your folder ID (in this example: `1a2b3c4d5e6f7g8h9i0j`)
4. Set this value as `GOOGLE_DRIVE_FOLDER_ID` in your configuration
### Domain-Wide Delegation (Optional)
If you need the service account to access files on behalf of users in your Google Workspace:
1. In your [Google Workspace Admin Console](https://admin.google.com/), go to:
- Security > API Controls > Domain-wide Delegation
2. Click "Add new" and provide:
- Client ID: your service account's client ID (found in the JSON credentials file)
- OAuth Scopes: `https://www.googleapis.com/auth/drive`
3. Set `GOOGLE_DRIVE_DELEGATE_TO` to the email address of the user to impersonate
This setup is only relevant for Google Workspace environments where you need the service account to access user-specific files.
## Setting up Dropbox Integration
To use the Dropbox integration, you'll need to create a Dropbox app and generate OAuth2 credentials:
1. **Create a Dropbox App**:
- Go to the [Dropbox Developer Apps Console](https://www.dropbox.com/developers/apps)
- Click "Create app"
- Select "Scoped access" for API
- Choose "Full Dropbox" access (or "App folder" for more restricted access)
- Give your app a name (e.g., "DocuNova")
- Click "Create app"
2. **Configure App Permissions**:
- In your app's settings page, go to the "Permissions" tab
- Enable the following permissions:
- `files.content.write` (to upload files)
- `files.content.read` (if you need to read file content)
- Click "Submit" to save changes
3. **Get App Key and Secret**:
- On your app's settings page, find the "App key" and "App secret"
- Set these as `DROPBOX_APP_KEY` and `DROPBOX_APP_SECRET` in your configuration
4. **Generate a Refresh Token**:
- Go to the "OAuth 2" tab in your app settings
- Add a redirect URI: `http://localhost` (this is for the authorization flow)
- Generate an authorization URL with these instructions:
```
https://www.dropbox.com/oauth2/authorize?client_id=YOUR_APP_KEY&response_type=code&token_access_type=offline
```
- Replace `YOUR_APP_KEY` with your app key
- Open this URL in your browser
- Authorize the app when prompted
- You'll be redirected to `localhost` with a code parameter in the URL
- Copy this code parameter
5. **Exchange the Code for a Refresh Token**:
- Use this curl command to exchange the code for tokens:
```bash
curl -X POST https://api.dropboxapi.com/oauth2/token \
-d code=YOUR_AUTH_CODE \
-d grant_type=authorization_code \
-d client_id=YOUR_APP_KEY \
-d client_secret=YOUR_APP_SECRET \
-d redirect_uri=http://localhost
```
- From the response, copy the `refresh_token` value
6. **Configure DocuNova**:
- Set `DROPBOX_APP_KEY`, `DROPBOX_APP_SECRET`, and `DROPBOX_REFRESH_TOKEN` with your values
- Set `DROPBOX_FOLDER` to the path where files should be uploaded
The system will use the refresh token to automatically generate short-lived access tokens when needed, so you shouldn't need to worry about token expiration.
## Setting up Amazon S3 Integration
To use the Amazon S3 integration, you'll need an AWS account and an S3 bucket:
1. **Create an S3 bucket**:
- Go to the [Amazon S3 Console](https://s3.console.aws.amazon.com/)
- Click "Create bucket"
- Enter a globally unique name for your bucket
- Select your preferred AWS region
- Configure other settings as needed (block public access is recommended)
- Click "Create bucket"
2. **Create an IAM User with S3 Access**:
- Go to the [AWS IAM Console](https://console.aws.amazon.com/iam/)
- Navigate to "Users" and click "Add users"
- Enter a name (e.g., "docunova-s3-access")
- For access type, select "Programmatic access"
- Click "Next: Permissions"
- Choose "Attach existing policies directly" and search for "AmazonS3FullAccess"
- For more security, you can create a custom policy limiting access to just your bucket
- Click through to review and create the user
- On the final page, you'll see the Access Key ID and Secret Access Key
- Save these credentials securely as they won't be shown again
3. **Configure DocuNova**:
- Set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to the credentials from step 2
- Set `AWS_REGION` to the region where your bucket was created (e.g., "us-east-1")
- Set `S3_BUCKET_NAME` to your bucket name
- Set `S3_FOLDER_PREFIX` to organize files in specific subfolder paths (e.g., "invoices/" or "documents/2023/")
- Optionally customize `S3_STORAGE_CLASS` and `S3_ACL` for your storage needs
4. **Optional: Create a Custom IAM Policy** (for better security):
- In IAM console, go to "Policies" and click "Create policy"
- Use the JSON editor and paste a policy like this (replace `your-bucket-name`):
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::your-bucket-name",
"arn:aws:s3:::your-bucket-name/*"
]
}
]
}
```
- After creating the policy, attach it to your user instead of the broader AmazonS3FullAccess
## Selective Service Configuration
You can choose which document storage services to use by only including the relevant environment variables. For example, if you only want to use Dropbox, include only the Dropbox variables and omit the Paperless NGX and Nextcloud variables.
docs/ConfigurationTroubleshooting.md
+89
View File
@@ -0,0 +1,89 @@
# Configuration Troubleshooting Guide
This guide helps you resolve common configuration issues with DocuNova's various integrations.
## SMTP Email Configuration
### Common Error: "Name or service not known"
This error occurs when the system can't resolve the hostname of your SMTP server.
**Solutions:**
1. Verify the `EMAIL_HOST` value in your `.env` file is correct
2. Check that your DNS is working properly:
```bash
nslookup your-smtp-server.com
```
3. If running in Docker, verify the container has network access
4. Try using an IP address instead of a hostname
### Common Error: "Connection refused"
This error indicates the SMTP server is unreachable on the specified port.
**Solutions:**
1. Verify `EMAIL_PORT` is correct (typically 587 for TLS, 465 for SSL)
2. Check if the email server is running and accessible from your network
3. Test connectivity using telnet:
```bash
telnet your-smtp-server.com 587
```
4. Check firewall settings to ensure the port is open
## Storage Provider Configuration
### Missing Configuration Attributes
When you see errors like "Settings object has no attribute 'nextcloud_url'" or similar:
**Solutions:**
1. Make sure all required environment variables are defined in your `.env` file
2. Check for typos in variable names
3. Ensure the application has loaded the environment variables
4. Restart the application after making changes to environment variables
### Example Configuration for Common Providers
#### Dropbox
```
DROPBOX_TOKEN=your_oauth2_token
DROPBOX_FOLDER=/DocuNova
```
#### Nextcloud
```
NEXTCLOUD_URL=https://your-nextcloud-instance.com
NEXTCLOUD_USERNAME=your_username
NEXTCLOUD_PASSWORD=your_secure_password
NEXTCLOUD_FOLDER=/Documents
```
#### SFTP
```
SFTP_HOST=your-sftp-server.com
SFTP_PORT=22
SFTP_USERNAME=your_username
# Use either password or key authentication:
SFTP_PASSWORD=your_secure_password
# Or:
SFTP_PRIVATE_KEY=/path/to/private_key.pem
SFTP_PRIVATE_KEY_PASSPHRASE=optional_passphrase
SFTP_FOLDER=/upload/path
```
## Testing Configuration
You can use the configuration validator to test your settings:
```bash
# Inside the container:
python -c "from app.utils.config_validator import check_all_configs; check_all_configs()"
```
## Debugging Tips
1. Check the application logs for specific error messages
2. Verify network connectivity from your application server to the external services
3. Ensure all required credentials are correct and access permissions are properly set
4. For OAuth services (like Dropbox), verify the token has not expired
5. If using Docker, check that environment variables are properly passed to the container
docs/README.md
+7 -1
View File
@@ -7,7 +7,13 @@ Welcome to the DocuNova documentation. This directory contains comprehensive gui
- [User Guide](UserGuide.md) - How to use DocuNova's features and interface
- [API Documentation](API.md) - Complete API reference for developers
- [Deployment Guide](DeploymentGuide.md) - How to deploy DocuNova in various environments
- [Configuration Guide](ConfigurationGuide.md) - All available configuration options
- [Configuration Guide](ConfigurationGuide.md) - All available configuration options, including:
- Dropbox integration
- Nextcloud integration
- Paperless NGX integration
- Google Drive integration
- Authentication setup
- Email processing configuration
- [Troubleshooting](Troubleshooting.md) - Solutions to common issues
## Additional Resources
frontend/static/logo.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.3 KiB

frontend/templates/base.html
+1
View File
@@ -36,6 +36,7 @@
<a href="/upload" class="text-gray-700 hover:text-gray-900">Upload</a>
<a href="/about" class="text-gray-700 hover:text-gray-900">About</a>
<a href="/files" class="text-gray-700 hover:text-gray-900">Files</a>
<a href="/status" class="text-gray-700 hover:text-gray-900">Status</a>
<!-- Dynamic Auth Section -->
<div id="authSection" class="text-gray-700 hover:text-gray-900"></div>
</div>
frontend/templates/env_debug.html
+82
View File
@@ -0,0 +1,82 @@
{% extends "base.html" %}
{% block title %}Environment Configuration{% endblock %}
{% block content %}
<div class="container mx-auto px-4 py-8">
<div class="mb-8">
<h1 class="text-3xl font-bold mb-2">Environment Configuration</h1>
<p class="text-gray-600">
This page displays the current configuration settings for the application. For security reasons,
sensitive values like passwords, tokens, and keys may be hidden.
</p>
<div class="bg-blue-100 border-l-4 border-blue-500 text-blue-700 p-4 my-4" role="alert">
<p><strong>Debug Mode:</strong> {{ "Enabled" if debug_enabled else "Disabled" }}</p>
<p><strong>App Version:</strong> {{ app_version }}</p>
</div>
</div>
{% for category, items in settings.items() %}
<div class="mb-8">
<h2 class="text-2xl font-semibold mb-4">{{ category }} Configuration</h2>
<div class="bg-white shadow overflow-hidden rounded-lg">
<table class="min-w-full divide-y divide-gray-200">
<thead class="bg-gray-50">
<tr>
<th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Setting</th>
<th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Value</th>
<th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Status</th>
</tr>
</thead>
<tbody class="bg-white divide-y divide-gray-200">
{% for item in items %}
<tr>
<td class="px-6 py-4 whitespace-nowrap text-sm font-medium text-gray-900">
{{ item.name }}
</td>
<td class="px-6 py-4 whitespace-nowrap text-sm text-gray-500">
{% if debug_enabled %}
{% if item.value is none %}
<span class="text-gray-400">NULL</span>
{% elif item.value == "" %}
<span class="text-gray-400">(empty string)</span>
{% else %}
{{ item.value }}
{% endif %}
{% else %}
<span class="text-gray-400">*** hidden in non-debug mode ***</span>
{% endif %}
</td>
<td class="px-6 py-4 whitespace-nowrap">
{% if item.is_configured %}
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-green-100 text-green-800">
Configured
</span>
{% else %}
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-red-100 text-red-800">
Not Configured
</span>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
{% endfor %}
<div class="mt-8 bg-gray-50 p-4 rounded-lg border border-gray-200">
<h3 class="text-lg font-medium text-gray-900">Environment Variables</h3>
<p class="text-sm text-gray-600 mt-1">
Configuration is loaded from environment variables or .env files.
Make sure your environment variables are correctly set.
</p>
<div class="mt-4">
<a href="/api/diagnostic/settings" class="inline-flex items-center px-4 py-2 border border-transparent text-sm font-medium rounded-md shadow-sm text-white bg-indigo-600 hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500">
View API Diagnostic
</a>
</div>
</div>
</div>
{% endblock %}
frontend/templates/status_dashboard.html
+112
View File
@@ -0,0 +1,112 @@
{% extends "base.html" %}
{% block title %}System Status{% endblock %}
{% block content %}
<div class="container mx-auto px-4 py-8">
<div class="mb-8">
<h1 class="text-3xl font-bold mb-2">System Status</h1>
<p class="text-gray-600">
This dashboard shows the status of all configured integrations and targets.
</p>
<div class="bg-blue-100 border-l-4 border-blue-500 text-blue-700 p-4 my-4" role="alert">
<p><strong>Debug Mode:</strong> {{ "Enabled" if debug_enabled else "Disabled" }}</p>
{% if last_check %}
<p><strong>Last Check:</strong> {{ last_check }}</p>
{% endif %}
</div>
</div>
<div class="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3">
{% for name, provider in providers.items() %}
<div class="bg-white overflow-hidden shadow rounded-lg">
<div class="px-4 py-5 sm:px-6 flex justify-between items-center">
<div class="flex items-center">
<span class="inline-flex items-center justify-center h-10 w-10 rounded-md {{ 'bg-green-500' if provider.configured else 'bg-gray-400' }} text-white mr-3">
<!-- Use appropriate icon based on provider -->
<svg xmlns="http://www.w3.org/2000/svg" class="h-6 w-6" fill="none" viewBox="0 0 24 24" stroke="currentColor">
{% if provider.icon == "mail" %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M3 8l7.89 5.26a2 2 0 002.22 0L21 8M5 19h14a2 2 0 002-2V7a2 2 0 00-2-2H5a2 2 0 00-2 2v10a2 2 0 002 2z" />
{% elif provider.icon == "cloud" %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M3 15a4 4 0 004 4h9a5 5 0 10-.1-9.999 5.002 5.002 0 10-9.78 2.096A4.001 4.001 0 003 15z" />
{% elif provider.icon == "dropbox" %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12l2 2 4-4m5.618-4.016A11.955 11.955 0 0112 2.944a11.955 11.955 0 01-8.618 3.04A12.02 12.02 0 003 9c0 5.591 3.824 10.29 9 11.622 5.176-1.332 9-6.03 9-11.622 0-1.042-.133-2.052-.382-3.016z" />
{% elif provider.icon == "file-text" %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z" />
{% elif provider.icon == "server" %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M5 12h14M5 12a2 2 0 01-2-2V6a2 2 0 012-2h14a2 2 0 012 2v4a2 2 0 01-2 2M5 12a2 2 0 00-2 2v4a2 2 0 002 2h14a2 2 0 002-2v-4a2 2 0 00-2-2m-2-4h.01M17 16h.01" />
{% elif provider.icon == "database" %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 7v10c0 2.21 3.582 4 8 4s8-1.79 8-4V7M4 7c0 2.21 3.582 4 8 4s8-1.79 8-4M4 7c0-2.21 3.582-4 8-4s8 1.79 8 4m0 5c0 2.21-3.582 4-8 4s-8-1.79-8-4" />
{% elif provider.icon == "hard-drive" %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M3 12h18M3 6h18M3 18h18" />
{% elif provider.icon == "globe" %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M3.055 11H5a2 2 0 012 2v1a2 2 0 002 2 2 2 0 012 2v2.945M8 3.935V5.5A2.5 2.5 0 0010.5 8h.5a2 2 0 012 2 2 2 0 104 0 2 2 0 012-2h1.064M15 20.488V18a2 2 0 012-2h3.064M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
{% elif provider.icon == "google" %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14" />
{% elif provider.icon == "microsoft" %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z" />
{% else %}
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
{% endif %}
</svg>
</span>
<h3 class="text-lg leading-6 font-medium text-gray-900">{{ name }}</h3>
</div>
{% if provider.configured %}
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-green-100 text-green-800">
Configured
</span>
{% else %}
<span class="px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-red-100 text-red-800">
Not Configured
</span>
{% endif %}
</div>
<div class="border-t border-gray-200">
<div class="px-4 py-5 sm:p-6">
<p class="text-sm text-gray-500">{{ provider.description }}</p>
{% if provider.url and provider.configured %}
<div class="mt-3">
<a href="{{ provider.url }}" target="_blank" class="text-sm text-blue-600 hover:text-blue-800">
{{ provider.url|truncate(30) }} <span aria-hidden="true">&rarr;</span>
</a>
</div>
{% endif %}
<div class="mt-4">
{% if provider.configured %}
<span class="inline-flex items-center px-2.5 py-0.5 rounded-md text-sm font-medium bg-green-100 text-green-800">
<svg class="-ml-0.5 mr-1.5 h-2 w-2 text-green-400" fill="currentColor" viewBox="0 0 8 8">
<circle cx="4" cy="4" r="3" />
</svg>
Active
</span>
{% else %}
<span class="inline-flex items-center px-2.5 py-0.5 rounded-md text-sm font-medium bg-gray-100 text-gray-800">
<svg class="-ml-0.5 mr-1.5 h-2 w-2 text-gray-400" fill="currentColor" viewBox="0 0 8 8">
<circle cx="4" cy="4" r="3" />
</svg>
Inactive
</span>
{% endif %}
</div>
</div>
</div>
</div>
{% endfor %}
</div>
<div class="mt-8 bg-gray-50 p-4 rounded-lg border border-gray-200">
<h3 class="text-lg font-medium text-gray-900">Configuration Settings</h3>
<p class="text-sm text-gray-600 mt-1">
For more detailed configuration settings and environment variables, check the environment debug page.
</p>
<div class="mt-4">
<a href="/env" class="inline-flex items-center px-4 py-2 border border-transparent text-sm font-medium rounded-md shadow-sm text-white bg-indigo-600 hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500">
View Detailed Configuration
</a>
</div>
</div>
</div>
{% endblock %}
frontend/templates/upload.html
+20 -17
View File
@@ -5,22 +5,25 @@
<div class="flex flex-col items-center justify-center p-8">
<h1 class="text-3xl font-bold mb-8">Upload a File</h1>
<div
id="dropZone"
class="border-4 border-dashed border-gray-300 rounded-lg p-8 bg-white text-center w-full max-w-lg"
ondrop="handleDrop(event)"
ondragover="handleDragOver(event)"
>
<p class="text-gray-500 mb-4">
Drag & drop a file here, or click to select a file.
</p>
<input
id="fileInput"
type="file"
class="hidden"
onchange="handleFileSelect(event)"
/>
</div>
<form action="/api/ui-upload" method="POST" enctype="multipart/form-data">
<div
id="dropZone"
class="border-4 border-dashed border-gray-300 rounded-lg p-8 bg-white text-center w-full max-w-lg"
ondrop="handleDrop(event)"
ondragover="handleDragOver(event)"
>
<p class="text-gray-500 mb-4">
Drag & drop a file here, or click to select a file.
</p>
<input
id="fileInput"
type="file"
class="hidden"
onchange="handleFileSelect(event)"
name="file"
/>
</div>
</form>
<div id="statusMessage" class="mt-4 text-gray-700"></div>
</div>
@@ -66,7 +69,7 @@
try {
let formData = new FormData();
formData.append("file", file);
const response = await fetch("/ui-upload", {
const response = await fetch("/api/ui-upload", {
method: "POST",
body: formData,
});
requirements.txt
+16 -1
View File
@@ -12,4 +12,19 @@ azure-ai-documentintelligence # Azure OCR service
authlib # Authentication
python-dotenv # Environment variables
starlette # ASGI toolkit (used by FastAPI)
alembic # Database migrations
alembic # Database migrations
# Google Drive API
google-api-python-client>=2.79.0
google-auth>=2.22.0
google-auth-oauthlib>=1.0.0
# OneDrive/Microsoft Graph API
msgraph-core>=1.0.0
msal>=1.20.0
# AWS S3
boto3>=1.28.0
# SFTP
paramiko>=3.4.0 # SSH/SFTP implementation for Python