Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 copilot-swe-agent[bot]
|
b3a85fe9c2 | ||
|
Christian Krakau-Louis
|
7481581e4e | ||
|
Christian Krakau-Louis
|
aae9a89d6b | ||
|
Christian Krakau-Louis
|
59db28d27b | ||
|
Christian Krakau-Louis
|
a92cace662 | ||
|
Christian Krakau-Louis
|
ffc049196b |
@@ -1,329 +1,56 @@
|
||||
# **Core Settings**
|
||||
WORKDIR=/workdir
|
||||
# **Config Variables**
|
||||
DATABASE_URL=sqlite:///./app/database.db
|
||||
REDIS_URL=redis://redis:6379/0
|
||||
EXTERNAL_HOSTNAME=docuelevate.example.com
|
||||
GOTENBERG_URL=http://gotenberg:3000
|
||||
ALLOW_FILE_DELETE=true # Allow deletion of file records
|
||||
WORKDIR=/workdir
|
||||
AWS_REGION="eu-central-1"
|
||||
AZURE_REGION="eastus"
|
||||
AZURE_ENDPOINT="https://<yourendpoint>.cognitiveservices.azure.com/"
|
||||
S3_BUCKET_NAME=<your_bucket_name>
|
||||
NEXTCLOUD_UPLOAD_URL=https://nextcloud.example.com/remote.php/dav/files/<USERNAME>
|
||||
NEXTCLOUD_FOLDER="<NEXTCLOUD_FOLDER_PATH>"
|
||||
PAPERLESS_NGX_URL=https://paperless.example.com/api/documents/post_document/
|
||||
PAPERLESS_HOST=https://paperless.example.com
|
||||
|
||||
# **UI / Appearance**
|
||||
# Default colour scheme: system (follow OS), light, or dark
|
||||
# Individual users can always override with the navbar dark-mode toggle.
|
||||
# UI_DEFAULT_COLOR_SCHEME=system
|
||||
|
||||
# **Batch Processing Settings**
|
||||
# Control throttling behavior for the /processall endpoint to prevent overwhelming downstream APIs
|
||||
PROCESSALL_THROTTLE_THRESHOLD=20 # Number of files above which throttling is applied (default: 20)
|
||||
PROCESSALL_THROTTLE_DELAY=3 # Delay in seconds between each task submission when throttling (default: 3)
|
||||
|
||||
# **Client-Side Upload Throttling**
|
||||
# Controls pacing when the browser uploads files (especially large directory drops).
|
||||
# The browser auto-detects rate-limit (HTTP 429) responses and backs off accordingly.
|
||||
UPLOAD_CONCURRENCY=3 # Max simultaneous uploads from the browser (default: 3)
|
||||
UPLOAD_QUEUE_DELAY_MS=500 # Delay (ms) between starting each upload slot (default: 500)
|
||||
|
||||
# **File Upload Size Limits** (Security - see SECURITY_AUDIT.md)
|
||||
# Maximum file upload size in bytes. Default: 1GB (1073741824 bytes)
|
||||
# Prevents resource exhaustion attacks. Adjust based on your server capacity.
|
||||
MAX_UPLOAD_SIZE=1073741824
|
||||
|
||||
# Maximum size for a single file chunk in bytes (optional)
|
||||
# If set and a file exceeds this size, it will be split into smaller chunks for processing
|
||||
# Default: None (no splitting). Example: 104857600 for 100MB chunks
|
||||
# MAX_SINGLE_FILE_SIZE=104857600
|
||||
|
||||
# **Request Body Size Limit** (Security - see SECURITY_AUDIT.md)
|
||||
# Maximum request body size in bytes for non-file-upload requests (JSON, form data, etc.).
|
||||
# Default: 1MB (1048576 bytes). File uploads are governed by MAX_UPLOAD_SIZE above.
|
||||
# Prevents memory exhaustion from oversized JSON/form payloads.
|
||||
# MAX_REQUEST_BODY_SIZE=1048576
|
||||
|
||||
# **Security Headers** (see SECURITY_AUDIT.md and docs/DeploymentGuide.md)
|
||||
# Disabled by default since most deployments use a reverse proxy (Traefik, Nginx, etc.)
|
||||
# that already adds these headers. Set to true only if deploying directly without a reverse proxy.
|
||||
# SECURITY_HEADERS_ENABLED=false
|
||||
|
||||
# If you enable security headers, you can also configure individual headers:
|
||||
# Strict-Transport-Security (HSTS) - Forces HTTPS connections
|
||||
# Only effective when served over HTTPS. Disable if not using HTTPS or if proxy adds this header
|
||||
# SECURITY_HEADER_HSTS_ENABLED=true
|
||||
# SECURITY_HEADER_HSTS_VALUE="max-age=31536000; includeSubDomains"
|
||||
|
||||
# Content-Security-Policy (CSP) - Controls resource loading
|
||||
# Customize based on your application's resource loading needs
|
||||
# Default allows self-hosted resources, inline scripts/styles, and external images
|
||||
# SECURITY_HEADER_CSP_ENABLED=true
|
||||
# SECURITY_HEADER_CSP_VALUE="default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;"
|
||||
|
||||
# X-Frame-Options - Prevents clickjacking attacks
|
||||
# Options: DENY (no framing), SAMEORIGIN (same origin framing only), ALLOW-FROM uri
|
||||
# SECURITY_HEADER_X_FRAME_OPTIONS_ENABLED=true
|
||||
# SECURITY_HEADER_X_FRAME_OPTIONS_VALUE="DENY"
|
||||
|
||||
# X-Content-Type-Options - Prevents MIME sniffing
|
||||
# Always set to 'nosniff' when enabled
|
||||
# SECURITY_HEADER_X_CONTENT_TYPE_OPTIONS_ENABLED=true
|
||||
|
||||
# **CORS (Cross-Origin Resource Sharing)** (see SECURITY_AUDIT.md – Infrastructure Security)
|
||||
# Disabled by default: most deployments rely on a reverse proxy (Traefik, Nginx, etc.) to inject
|
||||
# CORS headers. Set CORS_ENABLED=true only if DocuElevate is exposed directly without a proxy,
|
||||
# or if your proxy does not handle CORS. When enabled, only list the exact origins that need access.
|
||||
#
|
||||
# Rationale for reverse-proxy-first approach:
|
||||
# Traefik/Nginx already set Access-Control-Allow-Origin (and related headers) for every response,
|
||||
# so adding the middleware here would duplicate headers. When this flag is False the application
|
||||
# trusts the proxy layer to enforce CORS policy; set it to True for standalone / direct-access
|
||||
# deployments only.
|
||||
#
|
||||
# CORS_ENABLED=false
|
||||
#
|
||||
# Comma-separated list of allowed origins (use * to allow all - not recommended with credentials)
|
||||
# CORS_ALLOWED_ORIGINS=https://app.example.com,https://admin.example.com
|
||||
#
|
||||
# Allow cookies / Authorization headers in cross-origin requests
|
||||
# Must be False when CORS_ALLOWED_ORIGINS=* (browser security requirement)
|
||||
# CORS_ALLOW_CREDENTIALS=false
|
||||
#
|
||||
# Allowed HTTP methods (comma-separated)
|
||||
# CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,OPTIONS,PATCH
|
||||
#
|
||||
# Allowed request headers (use * to allow all)
|
||||
# CORS_ALLOWED_HEADERS=*
|
||||
|
||||
# **Rate Limiting** (see SECURITY_AUDIT.md and docs/API.md)
|
||||
# Protects against DoS attacks and API abuse by limiting request rates per IP/user
|
||||
# Enabled by default - highly recommended for production
|
||||
RATE_LIMITING_ENABLED=true
|
||||
|
||||
# Default rate limit for all API endpoints (format: count/period)
|
||||
# Periods can be: second, minute, hour, day
|
||||
# Default: 100 requests per minute per IP/user
|
||||
RATE_LIMIT_DEFAULT=100/minute
|
||||
|
||||
# Rate limit for file upload endpoints
|
||||
# Allows faster uploads while still preventing abuse
|
||||
# Default: 600 uploads per minute per IP/user
|
||||
RATE_LIMIT_UPLOAD=600/minute
|
||||
|
||||
# Rate limit for authentication endpoints
|
||||
# Strict limit to prevent brute force attacks
|
||||
# Default: 10 attempts per minute per IP
|
||||
RATE_LIMIT_AUTH=10/minute
|
||||
|
||||
# Note: Processing endpoints (OCR, metadata extraction) use built-in queue throttling
|
||||
# via Celery task queue to control processing rates and prevent upstream API overloads.
|
||||
# No additional API-level rate limit is needed for processing endpoints.
|
||||
|
||||
# **Authentication**
|
||||
AUTH_ENABLED=true
|
||||
# Generate a secure random string, for example:
|
||||
# python -c "import secrets; print(secrets.token_hex(32))"
|
||||
SESSION_SECRET=b39fd43f68d0491ca942f28a16e484b1e763fe9accf4445ca2669a5f3b179eb4
|
||||
ADMIN_USERNAME=admin
|
||||
ADMIN_PASSWORD=your_secure_password
|
||||
ADMIN_GROUP_NAME=admin
|
||||
|
||||
# **OpenID Connect/Authentik Settings**
|
||||
AUTHENTIK_CLIENT_ID=<yourAuthentikAppClientID>
|
||||
AUTHENTIK_CLIENT_SECRET=<yourAuthentikClientSecret>
|
||||
AUTHENTIK_CONFIG_URL=<ConfigUrlOfYourApp, e.g. https://authentik.example.com/application/o/docuelevate/.well-known/openid-configuration>
|
||||
OAUTH_PROVIDER_NAME="Authentik SSO"
|
||||
|
||||
# **AI/ML Services**
|
||||
# Select your AI provider: openai | azure | anthropic | gemini | ollama | openrouter | portkey | litellm
|
||||
AI_PROVIDER=openai
|
||||
# Model override (optional – falls back to OPENAI_MODEL when not set)
|
||||
# AI_MODEL=gpt-4o-mini
|
||||
|
||||
# --- OpenAI (AI_PROVIDER=openai) ---
|
||||
# **Tokens/API Credentials**
|
||||
AWS_ACCESS_KEY_ID="<AWS_ACCESS_KEY>"
|
||||
AWS_SECRET_ACCESS_KEY="<AWS_SECRET_ACCESS_KEY>"
|
||||
OPENAI_API_KEY="<OPENAI_API_KEY>"
|
||||
OPENAI_BASE_URL=https://api.openai.com/v1
|
||||
OPENAI_MODEL=gpt-4o-mini
|
||||
PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
|
||||
DROPBOX_APP_KEY=<DROPBOX_APP_KEY>
|
||||
DROPBOX_APP_SECRET=<DROPBOX_APP_SECRET>
|
||||
DROPBOX_REFRESH_TOKEN=<DROPBOX_REFRESH_TOKEN>
|
||||
AZURE_AI_KEY=<AZURE_AI_KEY>
|
||||
|
||||
# --- Anthropic Claude (AI_PROVIDER=anthropic) ---
|
||||
# ANTHROPIC_API_KEY=sk-ant-...
|
||||
# AI_MODEL=claude-3-5-sonnet-20241022
|
||||
|
||||
# --- Google Gemini (AI_PROVIDER=gemini) ---
|
||||
# GEMINI_API_KEY=AIza...
|
||||
# AI_MODEL=gemini-1.5-pro
|
||||
|
||||
# --- Ollama local LLMs (AI_PROVIDER=ollama) ---
|
||||
# OLLAMA_BASE_URL=http://localhost:11434
|
||||
# AI_MODEL=llama3.2
|
||||
|
||||
# --- OpenRouter (AI_PROVIDER=openrouter) ---
|
||||
# OPENROUTER_API_KEY=sk-or-...
|
||||
# AI_MODEL=anthropic/claude-3.5-sonnet
|
||||
|
||||
# --- Portkey AI Gateway (AI_PROVIDER=portkey) ---
|
||||
# PORTKEY_API_KEY=pk-...
|
||||
# PORTKEY_VIRTUAL_KEY=vk-... # optional – routes to provider credentials in Portkey vault
|
||||
# PORTKEY_CONFIG=pc-... # optional – saved Config ID for fallbacks / load balancing
|
||||
|
||||
# --- Azure OpenAI (AI_PROVIDER=azure) ---
|
||||
# OPENAI_API_KEY=<azure-key>
|
||||
# OPENAI_BASE_URL=https://my-resource.openai.azure.com
|
||||
# AZURE_OPENAI_API_VERSION=2024-02-01
|
||||
# AI_MODEL=gpt-4o # deployment name in Azure
|
||||
|
||||
# Azure Document Intelligence (OCR – separate from AI provider above)
|
||||
# **Email Settings**
|
||||
EMAIL_HOST=smtp.example.com
|
||||
EMAIL_PORT=587
|
||||
EMAIL_USERNAME=docuelevate@example.com
|
||||
EMAIL_PASSWORD=your_secure_email_password
|
||||
EMAIL_USE_TLS=True
|
||||
EMAIL_SENDER=DocuElevate System <docuelevate@example.com>
|
||||
EMAIL_DEFAULT_RECIPIENT=recipient@example.com
|
||||
# **User Credentials**
|
||||
ADMIN_USERNAME=admin
|
||||
ADMIN_PASSWORD=your_secure_password
|
||||
NEXTCLOUD_USERNAME=<NEXTCLOUD_USERNAME>
|
||||
NEXTCLOUD_PASSWORD=<NEXTCLOUD_PASSWORD>
|
||||
IMAP1_USERNAME=<IMAP1_USERNAME>
|
||||
IMAP1_PASSWORD=<IMAP1_PASSWORD>
|
||||
IMAP2_USERNAME=<IMAP2_USERNAME>
|
||||
IMAP2_PASSWORD=<IMAP2_PASSWORD>
|
||||
|
||||
# **IMAP Settings**
|
||||
IMAP1_HOST=mail.example.com
|
||||
IMAP1_PORT=993
|
||||
IMAP1_USERNAME=<IMAP1_USERNAME>
|
||||
IMAP1_PASSWORD=<IMAP1_PASSWORD>
|
||||
IMAP1_SSL=true
|
||||
IMAP1_POLL_INTERVAL_MINUTES=5
|
||||
IMAP1_DELETE_AFTER_PROCESS=false
|
||||
|
||||
IMAP2_HOST=imap.gmail.com
|
||||
IMAP2_PORT=993
|
||||
IMAP2_USERNAME=<IMAP2_USERNAME>
|
||||
IMAP2_PASSWORD=<IMAP2_PASSWORD>
|
||||
IMAP2_SSL=true
|
||||
IMAP2_POLL_INTERVAL_MINUTES=10
|
||||
IMAP2_DELETE_AFTER_PROCESS=false
|
||||
|
||||
# IMAP Readonly Mode (Feature Flag)
|
||||
# When true, IMAP processing will fetch and process attachments but will NOT modify
|
||||
# the mailbox state (no starring, labeling, deleting, or flag changes).
|
||||
# Use for pre-production instances that share a mailbox with production.
|
||||
IMAP_READONLY_MODE=false
|
||||
GOTENBERG_URL=http://gotenberg:3000
|
||||
|
||||
# **Storage/Document Services**
|
||||
# Amazon S3
|
||||
AWS_REGION=us-east-1
|
||||
AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
|
||||
AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
|
||||
S3_BUCKET_NAME=my-document-bucket
|
||||
S3_FOLDER_PREFIX=documents/uploads/2023/ # Organizes files in this subfolder
|
||||
S3_STORAGE_CLASS=STANDARD
|
||||
S3_ACL=private
|
||||
|
||||
# NextCloud
|
||||
NEXTCLOUD_UPLOAD_URL=https://nextcloud.example.com/remote.php/dav/files/<USERNAME>
|
||||
NEXTCLOUD_FOLDER="<NEXTCLOUD_FOLDER_PATH>"
|
||||
NEXTCLOUD_USERNAME=<NEXTCLOUD_USERNAME>
|
||||
NEXTCLOUD_PASSWORD=<NEXTCLOUD_PASSWORD>
|
||||
|
||||
# Paperless-ngx
|
||||
PAPERLESS_HOST=https://paperless.example.com
|
||||
PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
|
||||
# Optional: Name of the custom field in Paperless-ngx to store the "absender" (sender) value
|
||||
# If set, the extracted sender information will be automatically set as a custom field in Paperless
|
||||
# Example: PAPERLESS_CUSTOM_FIELD_ABSENDER=Absender
|
||||
# PAPERLESS_CUSTOM_FIELD_ABSENDER=
|
||||
|
||||
# Optional: JSON mapping of metadata fields to Paperless custom field names
|
||||
# This allows you to map multiple extracted metadata fields to custom fields in Paperless
|
||||
# The mapping format is: {"metadata_field_name": "PaperlessCustomFieldName", ...}
|
||||
# Available metadata fields: absender, empfaenger, correspondent, document_type, language,
|
||||
# kommunikationsart, kommunikationskategorie, reference_number, etc.
|
||||
# Example: PAPERLESS_CUSTOM_FIELDS_MAPPING={"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"}
|
||||
# PAPERLESS_CUSTOM_FIELDS_MAPPING=
|
||||
|
||||
# Dropbox
|
||||
DROPBOX_APP_KEY=<DROPBOX_APP_KEY>
|
||||
DROPBOX_APP_SECRET=<DROPBOX_APP_SECRET>
|
||||
DROPBOX_REFRESH_TOKEN=<DROPBOX_REFRESH_TOKEN>
|
||||
DROPBOX_FOLDER="/Documents/Uploads"
|
||||
|
||||
# Google Drive
|
||||
# Service Account Method:
|
||||
GOOGLE_DRIVE_CREDENTIALS_JSON={"type":"service_account","project_id":"your-project","private_key_id":"key-id","private_key":"-----BEGIN PRIVATE KEY-----\nYOUR_PRIVATE_KEY\n-----END PRIVATE KEY-----\n","client_email":"service-account@project.iam.gserviceaccount.com","client_id":"client-id","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/service-account%40project.iam.gserviceaccount.com"}
|
||||
GOOGLE_DRIVE_FOLDER_ID=<YOUR_FOLDER_ID>
|
||||
GOOGLE_DRIVE_DELEGATE_TO=<OPTIONAL_USER_EMAIL>
|
||||
|
||||
# OAuth Method (Alternative):
|
||||
GOOGLE_DRIVE_USE_OAUTH=false # Set to true to use OAuth instead of service account
|
||||
GOOGLE_DRIVE_CLIENT_ID=your-oauth-client-id # Required for OAuth method
|
||||
GOOGLE_DRIVE_CLIENT_SECRET=your-oauth-client-secret # Required for OAuth method
|
||||
GOOGLE_DRIVE_REFRESH_TOKEN=your-oauth-refresh-token # Required for OAuth method
|
||||
|
||||
# OneDrive
|
||||
ONEDRIVE_CLIENT_ID=your-client-id
|
||||
ONEDRIVE_CLIENT_SECRET=your-client-secret
|
||||
ONEDRIVE_TENANT_ID=common
|
||||
ONEDRIVE_REFRESH_TOKEN=your-refresh-token
|
||||
ONEDRIVE_FOLDER_PATH=Documents/Uploads
|
||||
|
||||
# WebDAV
|
||||
WEBDAV_URL=https://webdav.example.com/path
|
||||
WEBDAV_USERNAME=webdav_user
|
||||
WEBDAV_PASSWORD=your_secure_webdav_password
|
||||
WEBDAV_FOLDER=/Documents/Uploads
|
||||
WEBDAV_VERIFY_SSL=True
|
||||
|
||||
# FTP
|
||||
# Security Note: FTP_USE_TLS=True is strongly recommended for secure connections
|
||||
# Set FTP_ALLOW_PLAINTEXT=False in production to prevent unencrypted FTP
|
||||
FTP_HOST=ftp.example.com
|
||||
FTP_PORT=21
|
||||
FTP_USERNAME=ftp_user
|
||||
FTP_PASSWORD=your_secure_ftp_password
|
||||
FTP_FOLDER=/Documents/Uploads
|
||||
FTP_USE_TLS=True
|
||||
FTP_ALLOW_PLAINTEXT=True
|
||||
|
||||
# SFTP
|
||||
# Security Note: Host key verification is enabled by default (False)
|
||||
# Only set to True in development/testing environments if needed
|
||||
# When false, configure SSH known_hosts for proper host key verification
|
||||
SFTP_HOST=sftp.example.com
|
||||
SFTP_PORT=22
|
||||
SFTP_USERNAME=sftp_user
|
||||
SFTP_PASSWORD=your_secure_sftp_password
|
||||
# SFTP_PRIVATE_KEY=/path/to/private_key.pem
|
||||
# SFTP_PRIVATE_KEY_PASSPHRASE=optional_passphrase
|
||||
SFTP_FOLDER=/Documents/Uploads
|
||||
SFTP_DISABLE_HOST_KEY_VERIFICATION=False # Default is False (secure); set to True only for testing
|
||||
|
||||
# **HTTP Request Settings**
|
||||
# Timeout for HTTP requests - set higher to handle large PDF files (up to 1GB)
|
||||
HTTP_REQUEST_TIMEOUT=120 # Timeout in seconds (default: 120 for large file operations)
|
||||
|
||||
# **Notification Settings**
|
||||
# Configure notification services using Apprise URL format
|
||||
# See https://github.com/caronc/apprise#supported-notifications
|
||||
# Examples:
|
||||
# - Discord: discord://webhook_id/webhook_token
|
||||
# - Telegram: tgram://bot_token/chat_id
|
||||
# - Email: mailto://user:pass@example.com
|
||||
# - Pushover: pover://user_key/app_token
|
||||
# - Slack: slack://tokenA/tokenB/tokenC
|
||||
# - Matrix: matrix://username:password@domain/#room
|
||||
|
||||
# You can specify multiple notification URLs by separating them with commas
|
||||
NOTIFICATION_URLS=discord://webhook_id/webhook_token,mailto://user:pass@gmail.com,tgram://bot_token/chat_id
|
||||
|
||||
# Control when notifications are sent
|
||||
NOTIFY_ON_TASK_FAILURE=True
|
||||
NOTIFY_ON_CREDENTIAL_FAILURE=True
|
||||
NOTIFY_ON_STARTUP=True
|
||||
NOTIFY_ON_SHUTDOWN=False
|
||||
NOTIFY_ON_FILE_PROCESSED=True
|
||||
|
||||
# Uptime Kuma
|
||||
UPTIME_KUMA_URL=https://status.example.com/api/push/abcdef123456?status=up
|
||||
UPTIME_KUMA_PING_INTERVAL=5
|
||||
|
||||
# **Full-Text Search (Meilisearch)**
|
||||
# URL for the Meilisearch instance.
|
||||
# Default is "http://meilisearch:7700" — the Docker Compose / K8s service name —
|
||||
# so container-to-container networking works without extra configuration.
|
||||
# Override to "http://localhost:7700" only when running the API process outside Docker.
|
||||
MEILISEARCH_URL=http://meilisearch:7700
|
||||
# Optional master/API key for secured Meilisearch instances
|
||||
# MEILISEARCH_API_KEY=your_master_key_here
|
||||
MEILISEARCH_INDEX_NAME=documents
|
||||
ENABLE_SEARCH=True
|
||||
# ** needed for Authentik **
|
||||
AUTH_ENABLED=true
|
||||
SESSION_SECRET=<atLeast32Characters>
|
||||
AUTHENTIK_CLIENT_ID=<yourAuthentikAppClientID>
|
||||
AUTHENTIK_CLIENT_SECRET=<yourAuthentikAppClientSecret>
|
||||
AUTHENTIK_CONFIG_URL=<ConfigUrlOfYourApp, e.g. https://authentik.example.com/application/o/document-parser/.well-known/openid-configuration>
|
||||
@@ -1,38 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
1. Go to '...'
|
||||
2. Click on '....'
|
||||
3. Scroll down to '....'
|
||||
4. See error
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots**
|
||||
If applicable, add screenshots to help explain your problem.
|
||||
|
||||
**Desktop (please complete the following information):**
|
||||
- OS: [e.g. iOS]
|
||||
- Browser [e.g. chrome, safari]
|
||||
- Version [e.g. 22]
|
||||
|
||||
**Smartphone (please complete the following information):**
|
||||
- Device: [e.g. iPhone6]
|
||||
- OS: [e.g. iOS8.1]
|
||||
- Browser [e.g. stock browser, safari]
|
||||
- Version [e.g. 22]
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
||||
@@ -1,20 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
||||
@@ -1,347 +0,0 @@
|
||||
# Copilot Instructions for DocuElevate
|
||||
|
||||
## Project Overview
|
||||
|
||||
DocuElevate is an intelligent document processing system that automates handling, extraction, and processing of documents. It integrates with multiple cloud storage providers (Dropbox, Google Drive, OneDrive, S3, Nextcloud) and uses AI services (OpenAI, Azure Document Intelligence) for metadata extraction and OCR.
|
||||
|
||||
## Tech Stack
|
||||
|
||||
- **Backend**: FastAPI, SQLAlchemy, Celery, Redis
|
||||
- **Frontend**: Jinja2 templates, Tailwind CSS
|
||||
- **AI/ML**: OpenAI API, Azure Document Intelligence
|
||||
- **Auth**: Authentik (OAuth2), Basic Auth
|
||||
- **Infrastructure**: Docker, Docker Compose, Alembic (migrations)
|
||||
- **Testing**: Pytest, pytest-asyncio, httpx
|
||||
|
||||
## Supported Runtimes
|
||||
|
||||
- **Python**: 3.11+ (3.11 and 3.12 specified in pyproject.toml)
|
||||
- **Docker**: Production images use `python:3.14.1` / `python:3.14.1-slim`
|
||||
- **Redis**: Alpine-based (`redis:alpine`)
|
||||
- **Gotenberg**: `gotenberg/gotenberg:latest` for PDF conversion
|
||||
|
||||
## Build Commands
|
||||
|
||||
```bash
|
||||
# Install production dependencies
|
||||
pip install -r requirements.txt
|
||||
|
||||
# Install development dependencies (includes linters, test tools)
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
# Run the FastAPI development server
|
||||
uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload
|
||||
|
||||
# Run the Celery worker (requires Redis)
|
||||
celery -A app.celery_worker worker -B --loglevel=info -Q document_processor,default,celery
|
||||
|
||||
# Docker build and run
|
||||
docker compose up -d
|
||||
|
||||
# Database migrations
|
||||
alembic upgrade head # Apply all migrations
|
||||
alembic revision --autogenerate -m "description" # Create new migration
|
||||
```
|
||||
|
||||
## Test Commands
|
||||
|
||||
```bash
|
||||
# Run all tests with coverage (default via pyproject.toml addopts)
|
||||
pytest
|
||||
|
||||
# Run tests by marker
|
||||
pytest -m unit
|
||||
pytest -m integration
|
||||
pytest -m "not requires_external"
|
||||
|
||||
# Run a specific test file or test
|
||||
pytest tests/test_api.py -v
|
||||
pytest tests/test_api.py::test_function_name -v
|
||||
|
||||
# Coverage report
|
||||
pytest --cov=app --cov-report=term-missing
|
||||
pytest --cov=app --cov-report=html
|
||||
```
|
||||
|
||||
## Lint / Format Commands
|
||||
|
||||
```bash
|
||||
# Format and lint with Ruff (replaces Black, isort, Flake8, Bandit — all-in-one)
|
||||
ruff format app/ tests/
|
||||
ruff check app/ tests/ --fix
|
||||
|
||||
# Type checking with mypy
|
||||
mypy app/
|
||||
|
||||
# Check for dependency vulnerabilities
|
||||
safety check
|
||||
|
||||
# Run all pre-commit hooks at once (recommended — runs ruff, mypy, secret detection, etc.)
|
||||
pre-commit run --all-files
|
||||
```
|
||||
|
||||
## Agent Workflow (Follow for Every Task)
|
||||
|
||||
Follow these steps **in order** for every task — do not skip any:
|
||||
|
||||
1. **Understand** — read the issue/request in full before writing any code
|
||||
2. **Explore** — search the codebase for existing patterns and relevant implementations
|
||||
3. **Plan** — outline your changes as a checklist before starting
|
||||
4. **Implement** — make the smallest correct change that solves the problem
|
||||
5. **Test** — write or update tests; new code requires 100% test coverage
|
||||
6. **Document** — update all relevant docs in `docs/`; this is mandatory, not optional
|
||||
7. **Quality Gate** — run the single gate command below and fix every failure before committing:
|
||||
|
||||
```bash
|
||||
ruff format app/ tests/ && \
|
||||
ruff check app/ tests/ --fix && \
|
||||
safety check && \
|
||||
pytest --tb=short -q
|
||||
```
|
||||
|
||||
8. **Review** — re-read your own diff; confirm it is clean, secure, minimal, and well-documented
|
||||
|
||||
> All commands in the quality gate must exit with code 0. Never submit with failures.
|
||||
|
||||
## Core Principles
|
||||
|
||||
### Code Quality
|
||||
- **Security first**: treat security as a non-negotiable requirement, not an afterthought — review [SECURITY_AUDIT.md](../SECURITY_AUDIT.md) for every change
|
||||
- Write **clean, modern, well-documented code** — prioritize readability, maintainability, and idiomatic Python
|
||||
- Use **Ruff** for all formatting, linting, import sorting, and security scanning — `ruff format` + `ruff check --fix` (replaces Black, isort, Flake8, Bandit)
|
||||
- Line length: 120 characters (configured in `pyproject.toml`)
|
||||
- Use **type hints** for all function parameters and return values
|
||||
- Write **docstrings** for all public functions, classes, and modules
|
||||
- Maintain **100% test coverage** for new code
|
||||
|
||||
### Python Conventions
|
||||
- Use descriptive variable names (e.g., `user_document_path`, not `udp`)
|
||||
- Follow PEP 8 naming: `snake_case` for functions/variables, `PascalCase` for classes
|
||||
- Use modern Python 3.10+ type hints: `list[str]`, `dict[str, Any]`, `str | None` — avoid `List`, `Dict`, `Optional` from `typing`
|
||||
- Only import from `typing` for `Any`, `Callable`, `TypeVar`, `Protocol`, and other constructs unavailable natively
|
||||
- Prefer `pathlib.Path` over string paths for file operations
|
||||
- Use f-strings for string formatting, not `.format()` or `%`
|
||||
- Handle exceptions explicitly - avoid bare `except:` clauses
|
||||
|
||||
### Security Best Practices
|
||||
- **Never commit secrets or credentials** to the repository
|
||||
- Use environment variables for sensitive configuration (see `.env.demo`)
|
||||
- Validate and sanitize all user inputs
|
||||
- Use parameterized queries with SQLAlchemy (never raw SQL with user input)
|
||||
- Review [SECURITY_AUDIT.md](../SECURITY_AUDIT.md) before making security-related changes
|
||||
- Security linting is built into Ruff via `S` rules — runs automatically with `ruff check`; fix all `S`-prefixed findings
|
||||
- Run `safety check` to scan dependencies for known CVEs before submitting any PR
|
||||
|
||||
### FastAPI Patterns
|
||||
- Organize endpoints by feature in `app/api/` directory
|
||||
- Use dependency injection for database sessions and authentication
|
||||
- Return Pydantic models from endpoints for automatic validation
|
||||
- Use proper HTTP status codes (200, 201, 400, 401, 403, 404, 500)
|
||||
- Document endpoints with docstrings for OpenAPI documentation
|
||||
- Use `async def` for I/O-bound operations
|
||||
|
||||
### Database (SQLAlchemy)
|
||||
- All models are defined in `app/models.py`
|
||||
- Use Alembic for schema migrations (create migration for any model change)
|
||||
- Use declarative base for models
|
||||
- Define relationships with `relationship()` and proper `back_populates`
|
||||
- Use database sessions from `app.database.get_db()` dependency
|
||||
- Always close sessions in `finally` blocks or use context managers
|
||||
|
||||
### Celery Tasks
|
||||
- Define tasks in `app/tasks/` directory, organized by feature
|
||||
- Use descriptive task names: `module.action` (e.g., `document.process_ocr`)
|
||||
- Set appropriate retry policies and error handling
|
||||
- Log progress and errors using Python's `logging` module
|
||||
- Use `bind=True` for tasks that need access to task instance
|
||||
- Keep tasks idempotent when possible
|
||||
|
||||
### Frontend
|
||||
- Templates are in `frontend/templates/` using Jinja2
|
||||
- Static files (CSS, JS, images) in `frontend/static/`
|
||||
- Use Tailwind CSS utility classes (already configured)
|
||||
- Keep JavaScript minimal - prefer server-side rendering
|
||||
- Follow existing template structure and patterns
|
||||
|
||||
### Testing
|
||||
- Write tests in `tests/` directory, mirroring `app/` structure
|
||||
- Use pytest markers: `@pytest.mark.unit`, `@pytest.mark.integration`, etc.
|
||||
- Mock external services (OpenAI, Azure, cloud storage) in tests
|
||||
- Use `pytest.fixture` for test setup and teardown
|
||||
- Run tests with: `pytest -v`
|
||||
- Check coverage with: `pytest --cov=app --cov-report=term-missing`
|
||||
- **All tests must pass** before submitting changes — never leave failing tests
|
||||
- **All linters must pass** before submitting — run `pre-commit run --all-files`
|
||||
|
||||
### Configuration
|
||||
- All configuration is in `app/config.py` using Pydantic Settings
|
||||
- Use environment variables for configuration (12-factor app)
|
||||
- Provide sensible defaults when possible
|
||||
- Document all configuration options in `docs/ConfigurationGuide.md`
|
||||
|
||||
### Documentation
|
||||
- Keep documentation in `docs/` directory in Markdown format
|
||||
- **Always update** relevant docs when adding or changing any feature — documentation updates are mandatory, never optional
|
||||
- User-facing documentation should be clear and include examples
|
||||
- Reference existing docs: `docs/UserGuide.md`, `docs/API.md`, `docs/DeploymentGuide.md`
|
||||
- See [AGENTIC_CODING.md](../AGENTIC_CODING.md) for detailed development guide
|
||||
|
||||
### Error Handling
|
||||
- Use custom exceptions defined in application (follow existing patterns)
|
||||
- Log errors with context using Python's `logging` module
|
||||
- Return user-friendly error messages in API responses
|
||||
- Include error details in development, sanitize in production
|
||||
- In API endpoints, raise `HTTPException` with appropriate status codes (400, 401, 403, 404, 500)
|
||||
- In Celery tasks, use `self.retry(exc=e, countdown=60)` for transient errors; log and return error dict for permanent errors
|
||||
- Never use bare `except:` — always catch specific exception types
|
||||
- Wrap database operations in `try/except` with `db.rollback()` in the except block
|
||||
|
||||
### Logging Conventions
|
||||
- Use Python's built-in `logging` module: `import logging; logger = logging.getLogger(__name__)`
|
||||
- **Log levels**:
|
||||
- `logger.debug()` — detailed diagnostic information
|
||||
- `logger.info()` — general operational events (document processed, task started)
|
||||
- `logger.warning()` — recoverable issues (retrying, fallback used)
|
||||
- `logger.error()` — errors that need attention (failed operations)
|
||||
- `logger.critical()` — system-level failures requiring immediate action
|
||||
- **Always include context** in log messages: `logger.info(f"Processing document: {file_id}, user: {user_id}")`
|
||||
- **Never log sensitive data**: passwords, tokens, API keys, personal information
|
||||
- Use f-strings in log messages (consistent with project style)
|
||||
|
||||
### Architectural Boundaries
|
||||
- **`app/api/`** — REST API endpoints only; organize by feature
|
||||
- **`app/tasks/`** — Celery background tasks only; keep idempotent
|
||||
- **`app/views/`** — UI routes serving Jinja2 templates
|
||||
- **`app/utils/`** — Shared utility functions and helpers
|
||||
- **`app/routes/`** — **Deprecated**; being migrated to `app/api/` — do not add new code here
|
||||
- **`app/models.py`** — All SQLAlchemy models (single file)
|
||||
- **`app/config.py`** — All configuration via Pydantic Settings (single file)
|
||||
- **`app/database.py`** — Database engine and session setup (single file)
|
||||
- **`app/auth.py`** — Authentication logic (single file)
|
||||
- **`frontend/templates/`** — Jinja2 templates; do not mix backend logic
|
||||
- **`frontend/static/`** — CSS, JS, images; keep JavaScript minimal
|
||||
- **`tests/`** — Test files mirroring `app/` structure
|
||||
- **`migrations/`** — Alembic migration scripts; always auto-generate with `alembic revision --autogenerate`
|
||||
|
||||
### Don't Change Rules
|
||||
These files and directories are managed by automation or are critical infrastructure — **do not manually edit**:
|
||||
- **`VERSION`** — Managed by `python-semantic-release`; updated automatically on merge to main
|
||||
- **`CHANGELOG.md`** — Auto-generated from conventional commit messages by semantic-release
|
||||
- **`migrations/`** — Do not manually edit existing migration files; only create new ones via `alembic revision --autogenerate`
|
||||
- **Git tags and GitHub Releases** — Created automatically by semantic-release; never create manually
|
||||
- **`.pre-commit-config.yaml`** — Only change if adding/updating linting tools; do not remove existing hooks
|
||||
- **`pyproject.toml` `[tool.semantic_release]`** — Release configuration; do not modify without explicit approval
|
||||
- **`docker-compose.yaml` service names** — External systems depend on `api`, `worker`, `redis`, `gotenberg` names
|
||||
|
||||
### Dependencies
|
||||
- Add new dependencies to `requirements.txt` (production) or `requirements-dev.txt` (development)
|
||||
- Document any new dependencies and their licenses in README.md
|
||||
- Check for security vulnerabilities with `safety check`
|
||||
- Pin major versions, allow minor updates (e.g., `fastapi>=0.100.0,<1.0.0`)
|
||||
|
||||
### Git Workflow
|
||||
- Write clear, descriptive commit messages
|
||||
- **ALWAYS follow Conventional Commits format** (see below)
|
||||
- Keep commits focused and atomic
|
||||
- **All tests must pass** before committing — `pytest` must succeed with no failures
|
||||
- **All linters must pass** before committing — `pre-commit run --all-files` must succeed
|
||||
- Pre-commit hooks are configured (`.pre-commit-config.yaml`)
|
||||
|
||||
## Conventional Commits (REQUIRED)
|
||||
|
||||
All commit messages MUST follow the [Conventional Commits](https://www.conventionalcommits.org/) specification.
|
||||
|
||||
### Format
|
||||
```
|
||||
<type>(<scope>): <subject>
|
||||
|
||||
<body>
|
||||
|
||||
<footer>
|
||||
```
|
||||
|
||||
### Commit Types and Version Impact
|
||||
- **feat**: New feature → minor version bump (0.5.0 → 0.6.0)
|
||||
- **fix**: Bug fix → patch version bump (0.5.0 → 0.5.1)
|
||||
- **perf**: Performance improvement → patch version bump
|
||||
- **docs**: Documentation only → no version bump
|
||||
- **style**: Formatting changes → no version bump
|
||||
- **refactor**: Code refactoring → no version bump
|
||||
- **test**: Test changes → no version bump
|
||||
- **build**: Build system changes → no version bump
|
||||
- **ci**: CI/CD changes → no version bump
|
||||
- **chore**: Other changes → no version bump
|
||||
|
||||
### Breaking Changes
|
||||
For breaking changes (major version bump), add `!` after type or include `BREAKING CHANGE:` in footer:
|
||||
```
|
||||
feat(api)!: redesign authentication endpoints
|
||||
|
||||
BREAKING CHANGE: OAuth2 tokens now required instead of API keys.
|
||||
```
|
||||
Result: 0.5.0 → 1.0.0
|
||||
|
||||
### Scope Examples
|
||||
- `api` - REST API changes
|
||||
- `ui` - Frontend changes
|
||||
- `auth` - Authentication
|
||||
- `storage` - Storage providers
|
||||
- `ocr` - OCR processing
|
||||
- `tasks` - Celery tasks
|
||||
- `config` - Configuration
|
||||
- `docs` - Documentation
|
||||
|
||||
### Commit Examples
|
||||
```
|
||||
feat(storage): add Amazon S3 storage provider
|
||||
fix(ocr): handle PDFs without text layer
|
||||
docs: update deployment guide with Docker setup
|
||||
refactor(tasks): consolidate duplicate code
|
||||
test: add integration tests for upload API
|
||||
chore: update dependencies for security fixes
|
||||
```
|
||||
|
||||
## Semantic Release Process
|
||||
|
||||
### Automated Versioning
|
||||
DocuElevate uses `python-semantic-release` for automated version management:
|
||||
|
||||
1. **On merge to main**: semantic-release analyzes commit messages
|
||||
2. **Automatic actions**:
|
||||
- Determines next version from commit types
|
||||
- Updates `VERSION` file
|
||||
- Generates/updates `CHANGELOG.md`
|
||||
- Creates Git tag with `v` prefix (e.g., `v0.6.0`)
|
||||
- Creates GitHub Release with auto-generated notes
|
||||
- Triggers Docker image builds with version tag
|
||||
|
||||
### Agent Rules for Versioning
|
||||
- ✅ **DO**: Write conventional commit messages
|
||||
- ✅ **DO**: Use appropriate commit types for your changes
|
||||
- ✅ **DO**: Mark breaking changes explicitly
|
||||
- ❌ **DON'T**: Manually edit `VERSION` file
|
||||
- ❌ **DON'T**: Manually edit `CHANGELOG.md`
|
||||
- ❌ **DON'T**: Create version tags or GitHub Releases manually
|
||||
|
||||
These files are managed entirely by the semantic-release automation.
|
||||
|
||||
### File Organization
|
||||
- Place API endpoints in `app/api/` organized by feature
|
||||
- Background tasks go in `app/tasks/`
|
||||
- Utility functions in `app/utils/`
|
||||
- UI routes in `app/views/`
|
||||
- Database models in `app/models.py`
|
||||
- Configuration in `app/config.py`
|
||||
|
||||
### Common Patterns
|
||||
- Use modern Python 3.10+ type hints: `list[str]`, `dict[str, Any]`, `str | None`; only import from `typing` for `Any`, `Callable`, `TypeVar`, `Protocol`
|
||||
- Import FastAPI dependencies: `from fastapi import Depends, HTTPException, status`
|
||||
- Get DB session: `db: Session = Depends(get_db)`
|
||||
- Current user: `current_user: User = Depends(get_current_user)`
|
||||
- Logger: `import logging; logger = logging.getLogger(__name__)`
|
||||
|
||||
## Resources
|
||||
- [AGENTIC_CODING.md](../AGENTIC_CODING.md) - Comprehensive development guide
|
||||
- [CONTRIBUTING.md](../CONTRIBUTING.md) - Contribution guidelines
|
||||
- [SECURITY_AUDIT.md](../SECURITY_AUDIT.md) - Security considerations
|
||||
- [README.md](../README.md) - Project overview and quickstart
|
||||
@@ -1,38 +0,0 @@
|
||||
# GitHub Copilot Workspace Configuration
|
||||
# This file configures GitHub Copilot coding agent settings for the DocuElevate repository
|
||||
|
||||
# Network allowlist for external API services
|
||||
# These domains are required for integration tests and external service connectivity
|
||||
network:
|
||||
allowlist:
|
||||
# OpenAI API - Required for AI-powered metadata extraction and GPT integration
|
||||
- api.openai.com
|
||||
|
||||
# Google OAuth2 - Required for Google Drive integration and OAuth authentication
|
||||
- oauth2.googleapis.com
|
||||
- accounts.google.com
|
||||
- www.googleapis.com
|
||||
|
||||
# Azure Cognitive Services - Required for Azure Document Intelligence and OCR
|
||||
- test.cognitiveservices.azure.com
|
||||
- "*.cognitiveservices.azure.com"
|
||||
|
||||
# Additional Azure endpoints that may be needed
|
||||
- login.microsoftonline.com
|
||||
- graph.microsoft.com
|
||||
|
||||
# AWS S3 - Required for S3 storage integration tests
|
||||
- s3.amazonaws.com
|
||||
- "*.s3.amazonaws.com"
|
||||
|
||||
# Dropbox API - Required for Dropbox storage integration
|
||||
- api.dropboxapi.com
|
||||
- content.dropboxapi.com
|
||||
|
||||
# Example/test domains - Used in test fixtures and SMTP configuration tests
|
||||
- example.com
|
||||
- smtp.example.com
|
||||
|
||||
# Package registries (if needed for dependency installation during tests)
|
||||
- pypi.org
|
||||
- files.pythonhosted.org
|
||||
@@ -5,14 +5,8 @@
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "pip"
|
||||
directory: "/"
|
||||
- package-ecosystem: "" # See documentation for possible values
|
||||
directory: "/" # Location of package manifests
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
open-pull-requests-limit: 10
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/frontend/static"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 5
|
||||
|
||||
|
||||
@@ -1,270 +0,0 @@
|
||||
---
|
||||
applyTo: "docs/**/*.md"
|
||||
---
|
||||
|
||||
# Documentation Instructions
|
||||
|
||||
These instructions apply to all documentation files in the `docs/` directory.
|
||||
|
||||
## Documentation Structure
|
||||
- User-facing documentation in `docs/` directory
|
||||
- All documentation in Markdown format
|
||||
- Follow existing documentation style and structure
|
||||
|
||||
## Existing Documentation
|
||||
- `docs/UserGuide.md` - How to use DocuElevate
|
||||
- `docs/API.md` - API reference and examples
|
||||
- `docs/DeploymentGuide.md` - Deployment instructions
|
||||
- `docs/ConfigurationGuide.md` - Configuration options
|
||||
- `docs/Troubleshooting.md` - Common issues and solutions
|
||||
- `AGENTIC_CODING.md` - Development guide for AI agents
|
||||
- `CONTRIBUTING.md` - Contribution guidelines
|
||||
- `README.md` - Project overview and quickstart
|
||||
|
||||
## Markdown Style
|
||||
|
||||
### Headers
|
||||
```markdown
|
||||
# H1 - Document Title (only one per file)
|
||||
|
||||
## H2 - Major Sections
|
||||
|
||||
### H3 - Subsections
|
||||
|
||||
#### H4 - Minor subsections (use sparingly)
|
||||
```
|
||||
|
||||
### Code Blocks
|
||||
Always specify the language for syntax highlighting:
|
||||
````markdown
|
||||
```python
|
||||
def example_function():
|
||||
"""Example Python code."""
|
||||
return "Hello, World!"
|
||||
```
|
||||
|
||||
```bash
|
||||
# Shell commands
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"key": "value"
|
||||
}
|
||||
```
|
||||
````
|
||||
|
||||
### Lists
|
||||
```markdown
|
||||
- Unordered list item 1
|
||||
- Unordered list item 2
|
||||
- Nested item
|
||||
- Another nested item
|
||||
|
||||
1. Ordered list item 1
|
||||
2. Ordered list item 2
|
||||
3. Ordered list item 3
|
||||
```
|
||||
|
||||
### Links
|
||||
```markdown
|
||||
[Link text](https://example.com)
|
||||
[Internal link](./UserGuide.md)
|
||||
[Link to section](#installation)
|
||||
```
|
||||
|
||||
### Images
|
||||
```markdown
|
||||
|
||||
|
||||
<div align="center">
|
||||
<img src="path/to/image.png" alt="Descriptive alt text" width="80%" />
|
||||
<p><em>Image caption</em></p>
|
||||
</div>
|
||||
```
|
||||
|
||||
### Tables
|
||||
```markdown
|
||||
| Column 1 | Column 2 | Column 3 |
|
||||
|----------|----------|----------|
|
||||
| Value 1 | Value 2 | Value 3 |
|
||||
| Value 4 | Value 5 | Value 6 |
|
||||
```
|
||||
|
||||
### Admonitions and Notes
|
||||
```markdown
|
||||
> **Note:** This is an important note.
|
||||
|
||||
> **Warning:** This is a warning message.
|
||||
|
||||
> **Tip:** This is a helpful tip.
|
||||
```
|
||||
|
||||
## Content Guidelines
|
||||
|
||||
### Writing Style
|
||||
- Use clear, concise language
|
||||
- Write in second person (you/your) for user-facing docs
|
||||
- Use present tense
|
||||
- Avoid jargon; explain technical terms when necessary
|
||||
- Use active voice
|
||||
- Keep sentences short and focused
|
||||
|
||||
### Documentation Types
|
||||
|
||||
#### User Documentation
|
||||
- Focus on **how to use** features, not implementation details
|
||||
- Include step-by-step instructions
|
||||
- Provide examples for common use cases
|
||||
- Add screenshots or diagrams when helpful
|
||||
- Explain what each feature does and when to use it
|
||||
|
||||
Example:
|
||||
```markdown
|
||||
## Uploading Documents
|
||||
|
||||
To upload a document to DocuElevate:
|
||||
|
||||
1. Navigate to the Upload page
|
||||
2. Click "Choose File" and select your document
|
||||
3. Select the destination (Dropbox, Google Drive, etc.)
|
||||
4. Click "Upload"
|
||||
|
||||
The document will be automatically processed and stored in your selected destination.
|
||||
```
|
||||
|
||||
#### API Documentation
|
||||
- Document all endpoints with examples
|
||||
- Show request and response formats
|
||||
- Include authentication requirements
|
||||
- Provide example curl commands
|
||||
- Document error responses
|
||||
|
||||
Example:
|
||||
```markdown
|
||||
### POST /api/documents/upload
|
||||
|
||||
Upload a new document for processing.
|
||||
|
||||
**Authentication:** Required
|
||||
|
||||
**Request:**
|
||||
```bash
|
||||
curl -X POST "http://localhost:8000/api/documents/upload" \
|
||||
-H "Authorization: Bearer YOUR_TOKEN" \
|
||||
-F "file=@document.pdf"
|
||||
```
|
||||
|
||||
**Response (201 Created):**
|
||||
```json
|
||||
{
|
||||
"id": 123,
|
||||
"filename": "document.pdf",
|
||||
"status": "processing"
|
||||
}
|
||||
```
|
||||
```
|
||||
|
||||
#### Configuration Documentation
|
||||
- List all configuration options
|
||||
- Provide default values
|
||||
- Explain what each option does
|
||||
- Include example configurations
|
||||
- Note which options are required vs. optional
|
||||
|
||||
Example:
|
||||
```markdown
|
||||
### OPENAI_API_KEY
|
||||
|
||||
**Type:** String
|
||||
**Required:** Yes
|
||||
**Default:** None
|
||||
|
||||
Your OpenAI API key for metadata extraction.
|
||||
|
||||
```bash
|
||||
OPENAI_API_KEY=sk-...
|
||||
```
|
||||
```
|
||||
|
||||
#### Troubleshooting Documentation
|
||||
- Start with the symptom/error
|
||||
- Provide clear diagnosis steps
|
||||
- Offer solutions
|
||||
- Include common causes
|
||||
|
||||
Example:
|
||||
```markdown
|
||||
### Error: "Connection refused" when starting services
|
||||
|
||||
**Cause:** Docker services are not running or ports are already in use.
|
||||
|
||||
**Solution:**
|
||||
1. Check if Docker is running: `docker ps`
|
||||
2. Check port availability: `lsof -i :8000`
|
||||
3. Restart Docker services: `docker-compose restart`
|
||||
```
|
||||
|
||||
## Code Examples
|
||||
- Always test code examples before including them
|
||||
- Use realistic examples that users can adapt
|
||||
- Include comments explaining non-obvious parts
|
||||
- Show complete examples, not just fragments
|
||||
|
||||
## Version Information
|
||||
- Update documentation when changing features
|
||||
- Note version numbers when features are added
|
||||
- Mark deprecated features clearly
|
||||
|
||||
## Cross-References
|
||||
- Link to related documentation
|
||||
- Reference other sections when appropriate
|
||||
- Keep the documentation interconnected
|
||||
|
||||
Example:
|
||||
```markdown
|
||||
For deployment instructions, see the [Deployment Guide](./DeploymentGuide.md).
|
||||
|
||||
For API details, refer to the [API Documentation](./API.md).
|
||||
```
|
||||
|
||||
## Updating Documentation
|
||||
Documentation updates are **mandatory** — every PR that changes code must include matching documentation updates in the same PR. There are no exceptions.
|
||||
|
||||
When making code changes:
|
||||
1. **Update relevant documentation** in the same PR — never defer docs to a follow-up
|
||||
2. Check for outdated information in existing docs
|
||||
3. Add new sections for new features
|
||||
4. Update examples if behavior changes
|
||||
5. Review related documentation for consistency
|
||||
6. Update `docs/ConfigurationGuide.md` and `.env.demo` for any new or changed configuration options
|
||||
|
||||
## Screenshots and Diagrams
|
||||
- Use clear, high-quality images
|
||||
- Annotate screenshots when helpful
|
||||
- Keep diagrams simple and focused
|
||||
- Update screenshots when UI changes
|
||||
- Use consistent styling in diagrams
|
||||
|
||||
## Accessibility
|
||||
- Use descriptive alt text for images
|
||||
- Ensure proper heading hierarchy
|
||||
- Make links descriptive (avoid "click here")
|
||||
- Use semantic formatting (bold, italic, code) appropriately
|
||||
|
||||
## README.md Specific
|
||||
- Keep README concise and focused on getting started
|
||||
- Include badges for build status, version, license
|
||||
- Show the most important features first
|
||||
- Link to detailed documentation
|
||||
- Include quick start instructions
|
||||
- Add screenshots of the main interface
|
||||
|
||||
## Configuration Guide Updates
|
||||
When adding new configuration options:
|
||||
- Add to `docs/ConfigurationGuide.md`
|
||||
- Include type, default value, and description
|
||||
- Provide example usage
|
||||
- Note any dependencies on other config options
|
||||
- Update `.env.demo` with the new option
|
||||
@@ -1,197 +0,0 @@
|
||||
---
|
||||
applyTo: "frontend/**/*"
|
||||
---
|
||||
|
||||
# Frontend Instructions
|
||||
|
||||
These instructions apply to all files in the `frontend/` directory (templates, CSS, JavaScript, images).
|
||||
|
||||
## Templates (Jinja2)
|
||||
|
||||
### Location and Structure
|
||||
- All templates in `frontend/templates/`
|
||||
- Use template inheritance with `base.html`
|
||||
- Keep templates organized by feature
|
||||
|
||||
### Template Patterns
|
||||
```jinja2
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Document Upload - DocuElevate{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="container mx-auto px-4 py-8">
|
||||
<h1 class="text-2xl font-bold mb-4">{{ page_title }}</h1>
|
||||
|
||||
{% if error_message %}
|
||||
<div class="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded mb-4">
|
||||
{{ error_message }}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<form method="post" enctype="multipart/form-data">
|
||||
<!-- Form content -->
|
||||
</form>
|
||||
</div>
|
||||
{% endblock %}
|
||||
```
|
||||
|
||||
### Tailwind CSS Usage
|
||||
- Use Tailwind utility classes (already configured)
|
||||
- Follow responsive design: `md:`, `lg:` breakpoints
|
||||
- Use existing color scheme from the project
|
||||
- Common patterns:
|
||||
- Containers: `container mx-auto px-4`
|
||||
- Buttons: `bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded`
|
||||
- Cards: `bg-white shadow-md rounded-lg p-6`
|
||||
- Forms: `w-full px-3 py-2 border rounded`
|
||||
|
||||
### Static Files
|
||||
- CSS files in `frontend/static/css/`
|
||||
- JavaScript in `frontend/static/js/`
|
||||
- Images in `frontend/static/images/`
|
||||
- Reference with `{{ url_for('static', path='css/style.css') }}`
|
||||
|
||||
### JavaScript
|
||||
- Keep JavaScript minimal - prefer server-side rendering
|
||||
- Use vanilla JavaScript or minimal dependencies
|
||||
- Place scripts at the end of the body
|
||||
- Use `defer` or `async` for external scripts
|
||||
```html
|
||||
<script src="{{ url_for('static', path='js/upload.js') }}" defer></script>
|
||||
```
|
||||
|
||||
### Forms
|
||||
- Use CSRF protection when needed
|
||||
- Include proper validation
|
||||
- Show clear error messages
|
||||
- Use proper `method` (GET/POST) and `enctype` for file uploads
|
||||
```html
|
||||
<form method="post" enctype="multipart/form-data">
|
||||
<div class="mb-4">
|
||||
<label class="block text-gray-700 text-sm font-bold mb-2" for="file">
|
||||
Document File
|
||||
</label>
|
||||
<input
|
||||
type="file"
|
||||
id="file"
|
||||
name="file"
|
||||
class="w-full px-3 py-2 border rounded"
|
||||
required
|
||||
/>
|
||||
</div>
|
||||
<button type="submit" class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">
|
||||
Upload
|
||||
</button>
|
||||
</form>
|
||||
```
|
||||
|
||||
### Accessibility (WCAG 2.1 Level AA Required)
|
||||
|
||||
DocuElevate targets **WCAG 2.1 Level AA** compliance. Every template change **must** follow these rules.
|
||||
For the full guide with examples, see `docs/AccessibilityGuide.md`.
|
||||
|
||||
#### Semantic HTML (WCAG 1.3.1)
|
||||
- Use semantic elements: `<nav>`, `<main>`, `<article>`, `<section>`, `<header>`, `<footer>`
|
||||
- Use proper heading hierarchy: one `<h1>` per page, then `<h2>` → `<h3>` (never skip levels)
|
||||
- Use `<button>` for actions (not `<a>` or `<div>`) and `<a>` for navigation
|
||||
- Use `<table>` with `<caption>` or `aria-label`, `<thead>`/`<tbody>`, and `scope="col"`/`scope="row"` on headers
|
||||
|
||||
#### Images & Icons (WCAG 1.1.1)
|
||||
- All `<img>` elements **must** have an `alt` attribute — descriptive for content images, `alt=""` for purely decorative ones
|
||||
- Decorative Font Awesome `<i>` icons **must** have `aria-hidden="true"` when adjacent text already conveys meaning
|
||||
- Icon-only buttons **must** have `aria-label` describing the action (e.g., `aria-label="Delete file"`)
|
||||
|
||||
#### Keyboard Navigation (WCAG 2.1.1, 2.4.1, 2.4.7)
|
||||
- All interactive elements must be keyboard-reachable (native `<a>`, `<button>`, `<input>`, or add `tabindex="0"` + key handlers)
|
||||
- `base.html` provides a **skip-to-content** link (`<a href="#main-content" class="skip-link">`) — do not remove it
|
||||
- Never suppress focus indicators — the global `focus-visible` outline in `styles.css` is required
|
||||
- Custom interactive widgets (dropdowns, modals) must trap focus correctly
|
||||
|
||||
#### ARIA Attributes
|
||||
- `aria-label` — use on elements whose purpose isn't clear from visible text (icon-only buttons, unlabelled inputs)
|
||||
- `aria-hidden="true"` — use on purely decorative icons and elements that duplicate adjacent text
|
||||
- `aria-live="polite"` — add to any container whose content updates dynamically (status messages, search results, upload progress)
|
||||
- `aria-expanded` — add to buttons that toggle visibility of content (menus, accordions)
|
||||
- `aria-current="page"` — mark the current page's navigation link
|
||||
- `aria-sort` — use on sortable table column headers
|
||||
|
||||
#### Forms (WCAG 1.3.1, 3.3.2)
|
||||
- Every `<input>`, `<select>`, and `<textarea>` **must** have an associated `<label>` (via `for`/`id`) or `aria-label`
|
||||
- Error messages must be linked via `aria-describedby` or announced with `role="alert"`
|
||||
- Use `role="search"` on search form containers
|
||||
|
||||
#### Modals / Dialogs (WCAG 4.1.2)
|
||||
- Add `role="dialog"`, `aria-modal="true"`, and `aria-labelledby` pointing to the dialog title
|
||||
- Focus must move into the dialog when opened and return to the trigger when closed
|
||||
|
||||
#### Color & Contrast (WCAG 1.4.3, 1.4.1)
|
||||
- Text must meet 4.5:1 contrast ratio against its background (3:1 for large text)
|
||||
- Never rely on color alone to convey information — pair color with icons, text labels, or patterns
|
||||
- Dark-mode overrides in `styles.css` are WCAG AA-verified; maintain this when adding new colors
|
||||
|
||||
#### Touch Targets (WCAG 2.5.8)
|
||||
- All clickable/tappable elements must be at least 44×44 CSS pixels (`min-height:44px; min-width:44px`)
|
||||
|
||||
#### Automated Checks
|
||||
- The CI pipeline runs `djlint` on every PR to catch common accessibility regressions
|
||||
- Run locally: `djlint frontend/templates/ --lint`
|
||||
- Configuration is in `pyproject.toml` under `[tool.djlint]`
|
||||
|
||||
### Error Handling
|
||||
- Display user-friendly error messages
|
||||
- Use flash messages for feedback
|
||||
- Show loading states for async operations
|
||||
```jinja2
|
||||
{% with messages = get_flashed_messages(with_categories=true) %}
|
||||
{% if messages %}
|
||||
{% for category, message in messages %}
|
||||
<div class="bg-{{ category }}-100 border border-{{ category }}-400 text-{{ category }}-700 px-4 py-3 rounded mb-4">
|
||||
{{ message }}
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endwith %}
|
||||
```
|
||||
|
||||
### URL Generation
|
||||
- Always use `url_for()` for URLs, never hardcode
|
||||
- Examples:
|
||||
- Routes: `{{ url_for('upload_document') }}`
|
||||
- Static: `{{ url_for('static', path='css/style.css') }}`
|
||||
- API: `{{ url_for('api_document', document_id=doc.id) }}`
|
||||
|
||||
### Template Variables
|
||||
- Check if variables exist before using them
|
||||
- Use filters for formatting
|
||||
```jinja2
|
||||
{% if document %}
|
||||
<p>Uploaded: {{ document.created_at|datetime }}</p>
|
||||
<p>Size: {{ document.file_size|filesizeformat }}</p>
|
||||
{% else %}
|
||||
<p>No document found</p>
|
||||
{% endif %}
|
||||
```
|
||||
|
||||
### Common Components
|
||||
- Follow existing patterns for headers, footers, navigation
|
||||
- Reuse template blocks and includes
|
||||
- Keep components modular
|
||||
```jinja2
|
||||
{% include 'components/navigation.html' %}
|
||||
{% include 'components/document_card.html' with document=doc %}
|
||||
```
|
||||
|
||||
## UI/UX Guidelines
|
||||
- Maintain consistent spacing using Tailwind's scale (4, 8, 16, etc.)
|
||||
- Use the existing color palette from the design
|
||||
- Ensure mobile responsiveness
|
||||
- Show loading indicators for long operations
|
||||
- Provide feedback for user actions (success/error messages)
|
||||
- Keep the interface clean and minimal
|
||||
|
||||
## Performance
|
||||
- Optimize images (compress, use appropriate formats)
|
||||
- Minimize JavaScript bundle size
|
||||
- Use lazy loading for images when appropriate
|
||||
- Cache static assets
|
||||
@@ -1,165 +0,0 @@
|
||||
---
|
||||
applyTo: "app/**/*.py"
|
||||
---
|
||||
|
||||
# Python Backend Instructions
|
||||
|
||||
These instructions apply to all Python code in the `app/` directory.
|
||||
|
||||
## Code Style
|
||||
- Use **Ruff** for all formatting, linting, import sorting, and security scanning — `ruff format app/ tests/ && ruff check app/ tests/ --fix`
|
||||
- Line length: 120 characters (configured in `pyproject.toml` `[tool.ruff]`)
|
||||
- All functions must have type hints for parameters and return values
|
||||
- Use modern Python 3.10+ type hints: `list[str]`, `dict[str, Any]`, `str | None`
|
||||
- Only import from `typing` for `Any`, `Callable`, `TypeVar`, `Protocol` (not `Dict`, `List`, `Optional`, `Union`)
|
||||
|
||||
## Import Order (enforced by Ruff `I` rules)
|
||||
```python
|
||||
# Standard library imports
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any # Only for Any, Callable, TypeVar, Protocol
|
||||
|
||||
# Third-party imports
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# Local application imports
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import Document, User
|
||||
```
|
||||
|
||||
## Function Definitions
|
||||
```python
|
||||
def process_document(
|
||||
file_path: Path,
|
||||
user_id: int,
|
||||
metadata: dict[str, Any] | None = None
|
||||
) -> DocumentMetadata:
|
||||
"""
|
||||
Process a document and extract metadata.
|
||||
|
||||
Args:
|
||||
file_path: Path to the document file
|
||||
user_id: ID of the user uploading the document
|
||||
metadata: Optional additional metadata
|
||||
|
||||
Returns:
|
||||
DocumentMetadata object with extracted information
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If file doesn't exist
|
||||
ProcessingError: If processing fails
|
||||
"""
|
||||
pass
|
||||
```
|
||||
|
||||
## FastAPI Endpoints
|
||||
- Use dependency injection for DB sessions and auth
|
||||
- Return Pydantic models for automatic validation
|
||||
- Use proper status codes from `fastapi.status`
|
||||
- Add detailed docstrings for OpenAPI docs
|
||||
```python
|
||||
from fastapi import APIRouter, Depends, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
router = APIRouter(prefix="/api/documents", tags=["documents"])
|
||||
|
||||
@router.post("/", status_code=status.HTTP_201_CREATED)
|
||||
async def create_document(
|
||||
file: UploadFile,
|
||||
db: Session = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user)
|
||||
) -> DocumentResponse:
|
||||
"""Create and process a new document."""
|
||||
pass
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
- Use custom exceptions from the application
|
||||
- Log errors with context using `logging.getLogger(__name__)`
|
||||
- Return user-friendly error messages
|
||||
- Never expose internal details in production errors
|
||||
```python
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
result = process_file(file_path)
|
||||
except FileNotFoundError:
|
||||
logger.error(f"File not found: {file_path}")
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
except Exception as e:
|
||||
logger.exception(f"Error processing file: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail="Processing failed")
|
||||
```
|
||||
|
||||
## Database Operations
|
||||
- Use SQLAlchemy ORM, never raw SQL with user input
|
||||
- Use `get_db()` dependency for sessions
|
||||
- Always commit in try/except blocks
|
||||
```python
|
||||
from sqlalchemy.orm import Session
|
||||
from app.database import get_db
|
||||
|
||||
def create_document(db: Session, document_data: dict) -> Document:
|
||||
"""Create a new document in the database."""
|
||||
db_document = Document(**document_data)
|
||||
try:
|
||||
db.add(db_document)
|
||||
db.commit()
|
||||
db.refresh(db_document)
|
||||
return db_document
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
raise
|
||||
```
|
||||
|
||||
## Celery Tasks
|
||||
- Define in `app/tasks/` directory
|
||||
- Use descriptive names: `module.action`
|
||||
- Set retry policies
|
||||
- Log progress and errors
|
||||
```python
|
||||
from celery import shared_task
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@shared_task(bind=True, max_retries=3)
|
||||
def process_ocr(self, document_id: int) -> dict[str, Any]:
|
||||
"""Process OCR for a document."""
|
||||
try:
|
||||
# Processing logic
|
||||
logger.info(f"Processing OCR for document {document_id}")
|
||||
return {"status": "success"}
|
||||
except Exception as exc:
|
||||
logger.exception(f"OCR processing failed for {document_id}")
|
||||
raise self.retry(exc=exc, countdown=60)
|
||||
```
|
||||
|
||||
## Configuration
|
||||
- All settings in `app/config.py` using Pydantic Settings
|
||||
- Use environment variables, never hardcode values
|
||||
- Provide defaults when sensible
|
||||
```python
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
class Settings(BaseSettings):
|
||||
openai_api_key: str
|
||||
max_file_size: int = 10485760 # 10MB default
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
```
|
||||
|
||||
## Security (First and Foremost)
|
||||
- **Security first**: treat every change as a potential attack surface — review `SECURITY_AUDIT.md` before making any security-related change
|
||||
- Never commit secrets, tokens, or credentials
|
||||
- Validate and sanitize all user inputs
|
||||
- Use parameterized queries — never raw SQL with user data
|
||||
- Sanitize file paths; check file permissions before access
|
||||
- Security linting is built into Ruff via `S` rules — fix all `S`-prefixed findings before committing
|
||||
- Run `safety check` before submitting any PR to catch dependency CVEs
|
||||
@@ -1,246 +0,0 @@
|
||||
---
|
||||
applyTo: "tests/**/*.py"
|
||||
---
|
||||
|
||||
# Testing Instructions
|
||||
|
||||
These instructions apply to all test files in the `tests/` directory.
|
||||
|
||||
## Test Organization
|
||||
- Mirror the structure of `app/` directory in `tests/`
|
||||
- Name test files with `test_` prefix (e.g., `test_api.py`)
|
||||
- Group related tests in classes with `Test` prefix
|
||||
- Use descriptive test function names: `test_<what>_<condition>_<expected>`
|
||||
|
||||
## Pytest Configuration
|
||||
- Configuration in `pytest.ini`
|
||||
- Run tests: `pytest -v`
|
||||
- With coverage: `pytest --cov=app --cov-report=term-missing`
|
||||
- Run specific markers: `pytest -m unit` or `pytest -m integration`
|
||||
|
||||
## Test Markers
|
||||
Use pytest markers to categorize tests:
|
||||
```python
|
||||
import pytest
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_document_validation():
|
||||
"""Test document validation logic."""
|
||||
pass
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_document_upload_api():
|
||||
"""Test document upload endpoint."""
|
||||
pass
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_large_file_processing():
|
||||
"""Test processing of large files."""
|
||||
pass
|
||||
|
||||
@pytest.mark.requires_external
|
||||
def test_openai_integration():
|
||||
"""Test OpenAI API integration."""
|
||||
pass
|
||||
```
|
||||
|
||||
Available markers:
|
||||
- `unit` - Unit tests for individual functions/methods
|
||||
- `integration` - Integration tests for API endpoints and workflows
|
||||
- `slow` - Tests that take significant time to run
|
||||
- `security` - Security-related tests
|
||||
- `requires_external` - Tests requiring external services (OpenAI, Azure, etc.)
|
||||
- `requires_db` - Tests requiring database
|
||||
- `requires_redis` - Tests requiring Redis
|
||||
|
||||
## Fixtures
|
||||
Use pytest fixtures for test setup and teardown:
|
||||
```python
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from app.database import Base
|
||||
|
||||
@pytest.fixture
|
||||
def db_session():
|
||||
"""Provide a database session for tests."""
|
||||
engine = create_engine("sqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
|
||||
yield session
|
||||
|
||||
session.close()
|
||||
Base.metadata.drop_all(engine)
|
||||
|
||||
@pytest.fixture
|
||||
def sample_document():
|
||||
"""Provide a sample document for tests."""
|
||||
return {
|
||||
"filename": "test.pdf",
|
||||
"content_type": "application/pdf",
|
||||
"size": 1024
|
||||
}
|
||||
```
|
||||
|
||||
## API Testing with FastAPI
|
||||
Use `TestClient` from FastAPI:
|
||||
```python
|
||||
from fastapi.testclient import TestClient
|
||||
from app.main import app
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
def test_upload_document():
|
||||
"""Test document upload endpoint."""
|
||||
with open("tests/fixtures/sample.pdf", "rb") as f:
|
||||
response = client.post(
|
||||
"/api/documents/upload",
|
||||
files={"file": ("test.pdf", f, "application/pdf")}
|
||||
)
|
||||
|
||||
assert response.status_code == 201
|
||||
assert "id" in response.json()
|
||||
```
|
||||
|
||||
## Async Testing
|
||||
For async code, use `pytest-asyncio`:
|
||||
```python
|
||||
import pytest
|
||||
import httpx
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_document_processing():
|
||||
"""Test async document processing."""
|
||||
async with httpx.AsyncClient(app=app, base_url="http://test") as client:
|
||||
response = await client.get("/api/documents/1")
|
||||
assert response.status_code == 200
|
||||
```
|
||||
|
||||
## Mocking External Services
|
||||
Always mock external services in tests:
|
||||
```python
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_openai_metadata_extraction(mocker):
|
||||
"""Test metadata extraction with mocked OpenAI."""
|
||||
mock_response = {
|
||||
"document_type": "invoice",
|
||||
"amount": 100.00,
|
||||
"date": "2024-01-01"
|
||||
}
|
||||
|
||||
mocker.patch(
|
||||
"app.utils.openai_client.extract_metadata",
|
||||
return_value=mock_response
|
||||
)
|
||||
|
||||
result = extract_document_metadata("test.pdf")
|
||||
assert result["document_type"] == "invoice"
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_azure_ocr_processing(mocker):
|
||||
"""Test OCR with mocked Azure service."""
|
||||
mock_text = "Sample extracted text"
|
||||
|
||||
mocker.patch(
|
||||
"app.utils.azure_client.extract_text",
|
||||
return_value=mock_text
|
||||
)
|
||||
|
||||
result = perform_ocr("test.pdf")
|
||||
assert result == mock_text
|
||||
```
|
||||
|
||||
## Database Testing
|
||||
```python
|
||||
@pytest.mark.requires_db
|
||||
def test_create_document(db_session):
|
||||
"""Test document creation in database."""
|
||||
from app.models import Document
|
||||
|
||||
doc = Document(
|
||||
filename="test.pdf",
|
||||
user_id=1,
|
||||
file_path="/tmp/test.pdf"
|
||||
)
|
||||
db_session.add(doc)
|
||||
db_session.commit()
|
||||
|
||||
assert doc.id is not None
|
||||
assert doc.filename == "test.pdf"
|
||||
```
|
||||
|
||||
## Test Coverage Goals
|
||||
- Achieve **100% test coverage** for all new code — use `# pragma: no cover` only for genuinely unreachable or platform-specific branches, with an inline comment explaining why
|
||||
- Enforce the threshold: `pytest --cov=app --cov-fail-under=100`
|
||||
- Focus on critical paths and error handling
|
||||
- Test both success and failure scenarios
|
||||
- Don't test third-party library code
|
||||
|
||||
## Test Structure
|
||||
Follow the Arrange-Act-Assert pattern:
|
||||
```python
|
||||
def test_document_validation():
|
||||
"""Test that invalid documents are rejected."""
|
||||
# Arrange
|
||||
invalid_document = {
|
||||
"filename": "", # Empty filename
|
||||
"size": -1 # Invalid size
|
||||
}
|
||||
|
||||
# Act
|
||||
result = validate_document(invalid_document)
|
||||
|
||||
# Assert
|
||||
assert result.is_valid is False
|
||||
assert "filename" in result.errors
|
||||
assert "size" in result.errors
|
||||
```
|
||||
|
||||
## Parameterized Tests
|
||||
Use `pytest.mark.parametrize` for multiple test cases:
|
||||
```python
|
||||
@pytest.mark.parametrize("filename,expected", [
|
||||
("document.pdf", True),
|
||||
("image.jpg", True),
|
||||
("script.exe", False),
|
||||
("", False),
|
||||
])
|
||||
def test_allowed_file_types(filename, expected):
|
||||
"""Test file type validation."""
|
||||
result = is_allowed_file(filename)
|
||||
assert result == expected
|
||||
```
|
||||
|
||||
## Test Data
|
||||
- Place test fixtures in `tests/fixtures/` directory
|
||||
- Use small sample files for testing
|
||||
- Don't commit large test files
|
||||
- Clean up test files in teardown
|
||||
|
||||
## Error Testing
|
||||
Always test error conditions:
|
||||
```python
|
||||
def test_missing_file_raises_error():
|
||||
"""Test that missing files raise appropriate error."""
|
||||
with pytest.raises(FileNotFoundError):
|
||||
process_document("/nonexistent/file.pdf")
|
||||
|
||||
def test_invalid_api_request():
|
||||
"""Test API error handling."""
|
||||
response = client.post("/api/documents/", json={})
|
||||
assert response.status_code == 422 # Validation error
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
- Test one thing per test function
|
||||
- Use descriptive test names
|
||||
- Keep tests independent (no dependencies between tests)
|
||||
- Use fixtures for common setup
|
||||
- Mock external dependencies
|
||||
- Test edge cases and error conditions
|
||||
- Keep tests fast (use mocks for slow operations)
|
||||
- Clean up resources after tests
|
||||
@@ -1,364 +0,0 @@
|
||||
name: CI Pipeline
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
tags:
|
||||
- 'v*'
|
||||
- '[0-9]+.*'
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
env:
|
||||
IMAGE_NAME: christianlouis/docuelevate
|
||||
|
||||
jobs:
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
# Stage 1: Ruff Lint & Format (runs first to catch style issues early)
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
lint:
|
||||
name: Ruff Lint & Format
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install Ruff
|
||||
run: pip install ruff
|
||||
|
||||
- name: Show Ruff version (debug)
|
||||
run: ruff --version
|
||||
|
||||
- name: Check for merge conflict markers
|
||||
run: |
|
||||
if git grep -rn -E '^(<{7} |>{7} |={7}$)' -- '.'; then
|
||||
echo "ERROR: Merge conflict markers found in tracked files."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run Ruff Lint (check)
|
||||
# ruff check can --fix locally, but CI should only check (no modifications)
|
||||
run: ruff check app/ tests/
|
||||
|
||||
- name: Run Ruff Format check
|
||||
# ruff format only supports --check; do not pass --fix here
|
||||
run: ruff format --check app/ tests/
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
# Stage 1b: HTML Accessibility Lint (catches a11y regressions early)
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
html-lint:
|
||||
name: HTML Accessibility Lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install djLint
|
||||
run: pip install djlint>=1.36.0
|
||||
|
||||
- name: Lint HTML templates for accessibility
|
||||
run: djlint frontend/templates/ --lint
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
# Stage 2a: Dependency Vulnerability Scan (runs in parallel with lint)
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
dependency-scan:
|
||||
name: Dependency Vulnerability Scan
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install pip-audit
|
||||
run: pip install pip-audit>=2.7.0
|
||||
|
||||
- name: Run pip-audit on production dependencies
|
||||
run: pip-audit -r requirements.txt --desc on
|
||||
|
||||
- name: Run pip-audit on dev dependencies
|
||||
run: pip-audit -r requirements-dev.txt --desc on
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
# Stage 2b: Quick Tests (unit + basic integration — fast fail gate)
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
test-quick:
|
||||
name: Quick Tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
needs: [lint, html-lint, dependency-scan]
|
||||
services:
|
||||
redis:
|
||||
image: redis:7
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install Dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Run Quick Tests
|
||||
run: >
|
||||
pytest tests/ -v
|
||||
--timeout=120
|
||||
--cov=app --cov-report=xml --cov-report=term
|
||||
--junitxml=junit.xml -o junit_family=legacy
|
||||
-m "not e2e and not requires_docker and not requires_external and not slow"
|
||||
|
||||
- name: Upload coverage reports to Codecov
|
||||
if: ${{ !cancelled() }}
|
||||
uses: codecov/codecov-action@v5
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
files: ./coverage.xml
|
||||
fail_ci_if_error: false
|
||||
|
||||
- name: Upload test results to Codecov
|
||||
if: ${{ !cancelled() }}
|
||||
uses: codecov/codecov-action@v5
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
files: ./junit.xml
|
||||
report_type: test_results
|
||||
fail_ci_if_error: false
|
||||
|
||||
- name: Upload test artifacts
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: test-results-quick
|
||||
path: |
|
||||
junit.xml
|
||||
coverage.xml
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
# Stage 2c: Integration Tests (Docker containers, external services)
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
test-integration:
|
||||
name: Integration Tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
needs: [test-quick] # Only run after quick tests pass (fail early)
|
||||
services:
|
||||
redis:
|
||||
image: redis:7
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
rabbitmq:
|
||||
image: rabbitmq:3-management
|
||||
ports:
|
||||
- 5672:5672
|
||||
- 15672:15672
|
||||
options: >-
|
||||
--health-cmd "rabbitmq-diagnostics -q ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install Dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Run Integration Tests
|
||||
run: >
|
||||
pytest tests/ -v
|
||||
--timeout=300
|
||||
--junitxml=junit-integration.xml -o junit_family=legacy
|
||||
-m "(requires_docker or requires_external or slow) and not e2e"
|
||||
|
||||
- name: Upload integration test results
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: test-results-integration
|
||||
path: junit-integration.xml
|
||||
|
||||
mypy:
|
||||
name: Mypy
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint, html-lint, dependency-scan] # Wait for lint, HTML a11y lint, and dependency scan before running type checks
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install Dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Run Mypy
|
||||
run: mypy app/
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
# Stage 3: Build & Push Docker Image (only after all Stage 2 jobs pass)
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
build:
|
||||
name: Build & Push Docker Image
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test-quick, test-integration, lint, html-lint, mypy, dependency-scan]
|
||||
if: github.event_name == 'push'
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Generate Build Metadata
|
||||
run: |
|
||||
chmod +x scripts/generate_build_metadata.sh
|
||||
./scripts/generate_build_metadata.sh
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata for tags
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.IMAGE_NAME }}
|
||||
ghcr.io/${{ github.repository_owner }}/docuelevate
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=sha,prefix={{branch}}-
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
- name: Build and Push Docker Image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
sbom: true
|
||||
provenance: mode=max
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
# Stage 4: Update preprod K8s manifest (ArgoCD GitOps, only on main)
|
||||
# ══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
update-k8s-manifest:
|
||||
name: Update Preprod K8s Manifest
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
|
||||
steps:
|
||||
- name: Compute image tag
|
||||
id: tag
|
||||
run: |
|
||||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
echo "image=ghcr.io/${{ github.repository_owner }}/docuelevate:main-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=main-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout k8s-cluster-state
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: christianlouis/k8s-cluster-state
|
||||
token: ${{ secrets.GH_PAT }}
|
||||
path: k8s-cluster-state
|
||||
|
||||
- name: Update image tag in preprod manifest
|
||||
uses: mikefarah/yq@v4.44.6
|
||||
env:
|
||||
IMAGE: ${{ steps.tag.outputs.image }}
|
||||
with:
|
||||
cmd: |
|
||||
yq -i '(.. | select(tag == "!!str") | select(test("^(ghcr\\.io/christianlouis/docuelevate|christianlouis/docuelevate):"))) = strenv(IMAGE)' \
|
||||
k8s-cluster-state/apps/docuelevate/preprod/docuelevate-stack.yaml
|
||||
|
||||
- name: Commit and push
|
||||
run: |
|
||||
cd k8s-cluster-state
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add apps/docuelevate/preprod/docuelevate-stack.yaml
|
||||
if git diff --staged --quiet; then
|
||||
echo "No changes to commit -- image tag already up to date"
|
||||
else
|
||||
git commit -m "chore(preprod): update docuelevate image to ${{ steps.tag.outputs.tag }}"
|
||||
git push
|
||||
fi
|
||||
@@ -1,55 +0,0 @@
|
||||
name: "CodeQL Advanced"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
schedule:
|
||||
- cron: '37 1 * * 1'
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze (${{ matrix.language }})
|
||||
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
security-events: write
|
||||
packages: read
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- language: actions
|
||||
build-mode: none
|
||||
- language: javascript-typescript
|
||||
build-mode: none
|
||||
- language: python
|
||||
build-mode: none
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
|
||||
- name: Run manual build steps
|
||||
if: matrix.build-mode == 'manual'
|
||||
shell: bash
|
||||
run: |
|
||||
echo 'If you are using a "manual" build mode for one or more of the' \
|
||||
'languages you are analyzing, replace this with the commands to build' \
|
||||
'your code, for example:'
|
||||
echo ' make bootstrap'
|
||||
echo ' make release'
|
||||
exit 1
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
@@ -0,0 +1,18 @@
|
||||
name: Deploy to Production
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["Build and Push Docker Image"]
|
||||
types:
|
||||
- completed
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Call Deployment Webhook
|
||||
run: |
|
||||
curl -X POST https://docker2.kuechenserver.org/api/stacks/webhooks/960c7d8e-97ec-4175-a8dc-73f037b02349
|
||||
@@ -0,0 +1,54 @@
|
||||
name: Build and Push Docker Image
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and Push
|
||||
uses: docker/build-push-action@v4
|
||||
with:
|
||||
# Specify target platforms
|
||||
platforms: linux/amd64
|
||||
context: .
|
||||
file: Dockerfile
|
||||
push: true
|
||||
tags: |
|
||||
christianlouis/document-processor:latest
|
||||
christianlouis/document-processor:${{ github.sha }}
|
||||
ghcr.io/${{ github.repository_owner }}/document-processor:latest
|
||||
ghcr.io/${{ github.repository_owner }}/document-processor:${{ github.sha }}
|
||||
# Cache options (optional)
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@@ -1,77 +0,0 @@
|
||||
name: Semantic Release
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
release:
|
||||
name: Semantic Release
|
||||
runs-on: ubuntu-latest
|
||||
if: github.repository == 'christianlouis/DocuElevate'
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install python-semantic-release
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config --global user.name "github-actions[bot]"
|
||||
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Run Semantic Release
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
semantic-release version --print
|
||||
semantic-release version
|
||||
semantic-release publish
|
||||
|
||||
- name: Update changelog if no new version was released
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
if git diff --name-only HEAD~1 2>/dev/null | grep -q CHANGELOG.md; then
|
||||
echo "CHANGELOG.md was already updated by semantic-release version"
|
||||
else
|
||||
semantic-release changelog
|
||||
if ! git diff --quiet CHANGELOG.md; then
|
||||
git add CHANGELOG.md
|
||||
git commit -m "docs(changelog): update changelog [skip ci]"
|
||||
git push
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: Update build metadata files if changed
|
||||
run: |
|
||||
for f in VERSION BUILD_DATE GIT_SHA RUNTIME_INFO; do
|
||||
if [ -f "$f" ]; then
|
||||
git add -f "$f"
|
||||
fi
|
||||
done
|
||||
if ! git diff --staged --quiet; then
|
||||
git commit -m "chore(release): update build metadata files [skip ci]"
|
||||
git push
|
||||
fi
|
||||
@@ -1,94 +0,0 @@
|
||||
name: Ruff Auto-Fix
|
||||
|
||||
# This workflow automatically fixes ruff formatting and linting issues
|
||||
# and commits them back to the PR branch when issues are detected.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
paths:
|
||||
- '**.py'
|
||||
workflow_dispatch: # Allow manual triggering
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
ruff-auto-fix:
|
||||
name: Auto-fix Ruff Issues
|
||||
runs-on: ubuntu-latest
|
||||
# Only run on PRs from the same repository (not forks) for security
|
||||
if: github.event.pull_request.head.repo.full_name == github.repository
|
||||
|
||||
steps:
|
||||
- name: Checkout PR branch
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.head_ref }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install Ruff
|
||||
run: pip install ruff
|
||||
|
||||
- name: Run Ruff Check with Auto-fix
|
||||
run: |
|
||||
echo "Running ruff check with auto-fix..."
|
||||
ruff check app/ tests/ --fix || true
|
||||
|
||||
- name: Run Ruff Format
|
||||
run: |
|
||||
echo "Running ruff format..."
|
||||
ruff format app/ tests/
|
||||
|
||||
- name: Check for changes
|
||||
id: check_changes
|
||||
run: |
|
||||
if [[ -n $(git status --porcelain) ]]; then
|
||||
echo "changes=true" >> $GITHUB_OUTPUT
|
||||
echo "Changes detected after running ruff auto-fix"
|
||||
else
|
||||
echo "changes=false" >> $GITHUB_OUTPUT
|
||||
echo "No changes needed - code is already properly formatted"
|
||||
fi
|
||||
|
||||
- name: Commit and push changes
|
||||
if: steps.check_changes.outputs.changes == 'true'
|
||||
run: |
|
||||
git config --local user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git config --local user.name "github-actions[bot]"
|
||||
git add app/ tests/
|
||||
git commit -m "style: apply ruff auto-fix
|
||||
|
||||
- Auto-formatted code with ruff format
|
||||
- Applied ruff linting fixes with --fix
|
||||
|
||||
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>"
|
||||
git push
|
||||
|
||||
- name: Comment on PR
|
||||
if: steps.check_changes.outputs.changes == 'true'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: '✨ Ruff auto-fix applied! The code has been automatically formatted and linting issues have been fixed.\n\nPlease pull the latest changes:\n```bash\ngit pull\n```'
|
||||
})
|
||||
|
||||
- name: Summary
|
||||
run: |
|
||||
if [[ "${{ steps.check_changes.outputs.changes }}" == "true" ]]; then
|
||||
echo "✅ Ruff auto-fix completed and changes committed"
|
||||
else
|
||||
echo "✅ No changes needed - code is already properly formatted"
|
||||
fi
|
||||
@@ -0,0 +1,40 @@
|
||||
name: Run Tests & Linting
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install Dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
pip install pytest flake8 black mypy pylint
|
||||
|
||||
# - name: Run Tests
|
||||
# run: pytest tests/
|
||||
|
||||
- name: Run Linter (Flake8)
|
||||
run: flake8 app/
|
||||
continue-on-error: true
|
||||
|
||||
- name: Run Code Formatter (Black)
|
||||
run: black --check app/
|
||||
continue-on-error: true
|
||||
|
||||
- name: Run Type Checker (Mypy)
|
||||
run: mypy app/
|
||||
continue-on-error: true
|
||||
|
||||
- name: Run Linter (Pylint)
|
||||
run: pylint app/
|
||||
continue-on-error: true
|
||||
+171
-201
@@ -1,201 +1,171 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# Environment files - NEVER commit these!
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
*.env
|
||||
|
||||
# Secrets and credentials
|
||||
*secret*
|
||||
*credentials*.json
|
||||
!frontend/static/* # Allow static files even if they match patterns
|
||||
!docs/* # Allow documentation files
|
||||
|
||||
# Private keys
|
||||
*.pem
|
||||
*.key
|
||||
*.p12
|
||||
*.pfx
|
||||
id_rsa*
|
||||
ssh_host_*
|
||||
|
||||
# Database files - may contain sensitive data
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
database.db
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
junit.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# UV
|
||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
#uv.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
||||
.pdm.toml
|
||||
.pdm-python
|
||||
.pdm-build/
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
# Build metadata files - generated at build time
|
||||
GIT_SHA
|
||||
RUNTIME_INFO
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
.env
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# UV
|
||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
#uv.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
||||
.pdm.toml
|
||||
.pdm-python
|
||||
.pdm-build/
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
# Pre-commit hooks for code quality and security
|
||||
# Install: pip install pre-commit
|
||||
# Setup: pre-commit install
|
||||
# Run manually: pre-commit run --all-files
|
||||
|
||||
repos:
|
||||
# General file checks
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.5.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
- id: check-json
|
||||
- id: check-added-large-files
|
||||
args: ['--maxkb=1000']
|
||||
- id: check-merge-conflict
|
||||
- id: detect-private-key
|
||||
- id: detect-aws-credentials
|
||||
args: ['--allow-missing-credentials']
|
||||
|
||||
# Ruff - Fast Python linter and formatter (replaces Black, Flake8, isort, Bandit)
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.3.0
|
||||
hooks:
|
||||
- id: ruff
|
||||
args: [ --fix ]
|
||||
- id: ruff-format
|
||||
|
||||
# Type checking
|
||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
||||
rev: v1.8.0
|
||||
hooks:
|
||||
- id: mypy
|
||||
args: ['--ignore-missing-imports']
|
||||
additional_dependencies: ['types-requests']
|
||||
|
||||
# Secret detection
|
||||
- repo: https://github.com/Yelp/detect-secrets
|
||||
rev: v1.4.0
|
||||
hooks:
|
||||
- id: detect-secrets
|
||||
args: ['--baseline', '.secrets.baseline']
|
||||
exclude: |
|
||||
(?x)^(
|
||||
.+\.lock|
|
||||
.+\.json|
|
||||
.env.demo
|
||||
)$
|
||||
|
||||
# Conventional commits validation
|
||||
- repo: https://github.com/compilerla/conventional-pre-commit
|
||||
rev: v3.0.0
|
||||
hooks:
|
||||
- id: conventional-pre-commit
|
||||
stages: [commit-msg]
|
||||
args: []
|
||||
@@ -1,22 +0,0 @@
|
||||
# Read the Docs configuration file
|
||||
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
|
||||
|
||||
# Required
|
||||
version: 2
|
||||
|
||||
# Set the OS, Python version, and other tools you might need
|
||||
build:
|
||||
os: ubuntu-24.04
|
||||
tools:
|
||||
python: "3.13"
|
||||
|
||||
# Build documentation with Mkdocs
|
||||
mkdocs:
|
||||
configuration: mkdocs.yml
|
||||
|
||||
# Optionally, but recommended,
|
||||
# declare the Python requirements required to build your documentation
|
||||
# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
|
||||
python:
|
||||
install:
|
||||
- requirements: docs/requirements.txt
|
||||
Vendored
-7
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"python.testing.pytestArgs": [
|
||||
"tests"
|
||||
],
|
||||
"python.testing.unittestEnabled": false,
|
||||
"python.testing.pytestEnabled": true
|
||||
}
|
||||
@@ -1,754 +0,0 @@
|
||||
# Agentic Coding Guide for DocuElevate
|
||||
|
||||
**Version:** 1.0
|
||||
**Last Updated:** 2026-02-06
|
||||
|
||||
This guide helps AI coding agents work effectively with the DocuElevate codebase. It provides context, conventions, and best practices for autonomous code contributions.
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Project Overview
|
||||
|
||||
### What is DocuElevate?
|
||||
DocuElevate is an intelligent document processing system that:
|
||||
- Ingests documents from multiple sources (email, web upload, API)
|
||||
- Processes documents (OCR, PDF conversion, metadata extraction)
|
||||
- Stores documents in various cloud storage providers
|
||||
- Uses AI (OpenAI, Azure) for intelligent document classification and metadata extraction
|
||||
|
||||
### Tech Stack
|
||||
```
|
||||
Backend: FastAPI, SQLAlchemy, Celery, Redis
|
||||
Frontend: Jinja2 templates, Tailwind CSS
|
||||
AI/ML: OpenAI API, Azure Document Intelligence
|
||||
Storage: Dropbox, Google Drive, OneDrive, S3, Nextcloud, Paperless-NGX
|
||||
Auth: Authentik (OAuth2), Basic Auth
|
||||
Infra: Docker, Docker Compose, Alembic (migrations)
|
||||
```
|
||||
|
||||
### Key Directories
|
||||
```
|
||||
DocuElevate/
|
||||
├── app/
|
||||
│ ├── api/ # REST API endpoints
|
||||
│ ├── tasks/ # Celery background tasks
|
||||
│ ├── routes/ # Deprecated - being migrated to api/
|
||||
│ ├── views/ # UI routes and templates
|
||||
│ ├── utils/ # Utility functions
|
||||
│ ├── config.py # Configuration (Pydantic Settings)
|
||||
│ ├── database.py # SQLAlchemy setup
|
||||
│ ├── models.py # Database models
|
||||
│ ├── main.py # FastAPI app initialization
|
||||
│ └── auth.py # Authentication logic
|
||||
├── frontend/
|
||||
│ ├── static/ # CSS, JS, images
|
||||
│ └── templates/ # Jinja2 HTML templates
|
||||
├── tests/ # Pytest test suite
|
||||
├── docs/ # User documentation
|
||||
├── migrations/ # Alembic database migrations
|
||||
└── docker/ # Docker configuration
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🤖 Agent Guidelines
|
||||
|
||||
### Before Making Changes
|
||||
|
||||
1. **Understand the Context**
|
||||
- Read relevant documentation in `docs/`
|
||||
- Check `TODO.md` for current priorities
|
||||
- Review `SECURITY_AUDIT.md` for security considerations
|
||||
- Check `ROADMAP.md` for feature direction
|
||||
|
||||
2. **Check Existing Patterns**
|
||||
- Look at similar existing code first
|
||||
- Follow the established patterns in the codebase
|
||||
- Don't introduce new patterns without good reason
|
||||
|
||||
3. **Identify Dependencies**
|
||||
- Check if your change affects multiple modules
|
||||
- Ensure you understand the Celery task flow
|
||||
- Consider impact on database schema
|
||||
|
||||
### Documentation-First Principle
|
||||
|
||||
**Documentation is as important as tests and code.** Every change must include documentation updates in the same commit/PR.
|
||||
|
||||
| Change type | What to update |
|
||||
|-------------|---------------|
|
||||
| New feature | `docs/UserGuide.md`, `docs/API.md` (if API), `docs/ConfigurationGuide.md` (if config) |
|
||||
| New config option | `docs/ConfigurationGuide.md` and `.env.demo` |
|
||||
| New API endpoint | `docs/API.md` |
|
||||
| Bug fix (user-visible) | `docs/Troubleshooting.md` |
|
||||
| Deployment change | `docs/DeploymentGuide.md` |
|
||||
| Security change | `SECURITY_AUDIT.md` |
|
||||
| Breaking change | CHANGELOG (auto-generated) + migration notes in relevant docs |
|
||||
|
||||
**Never edit `CHANGELOG.md` or `VERSION` manually.** These are managed automatically by `python-semantic-release` on every merge to `main`.
|
||||
|
||||
### Code Conventions
|
||||
|
||||
#### Python Style
|
||||
```python
|
||||
# Use Ruff formatting (line length: 120)
|
||||
# Use type hints
|
||||
def process_document(file_path: str, metadata: Dict[str, Any]) -> DocumentMetadata:
|
||||
"""
|
||||
Process a document and extract metadata.
|
||||
|
||||
Args:
|
||||
file_path: Absolute path to the document file
|
||||
metadata: Additional metadata to include
|
||||
|
||||
Returns:
|
||||
DocumentMetadata object with extracted information
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If file doesn't exist
|
||||
ProcessingError: If processing fails
|
||||
"""
|
||||
pass
|
||||
|
||||
# Use descriptive variable names
|
||||
user_document_path = Path("/workdir/documents/invoice.pdf")
|
||||
ocr_result = extract_text_from_pdf(user_document_path)
|
||||
|
||||
# Prefer explicit over implicit
|
||||
if storage_provider == "dropbox":
|
||||
upload_to_dropbox(file_path, metadata)
|
||||
elif storage_provider == "google_drive":
|
||||
upload_to_google_drive(file_path, metadata)
|
||||
else:
|
||||
raise ValueError(f"Unknown storage provider: {storage_provider}")
|
||||
```
|
||||
|
||||
#### Configuration
|
||||
```python
|
||||
# Always use settings from config.py
|
||||
from app.config import settings
|
||||
|
||||
# Good
|
||||
api_key = settings.openai_api_key
|
||||
|
||||
# Bad - never hardcode
|
||||
api_key = "sk-abc123..."
|
||||
|
||||
# Check if optional services are configured
|
||||
if settings.dropbox_app_key:
|
||||
# Dropbox is configured
|
||||
upload_to_dropbox()
|
||||
```
|
||||
|
||||
#### Error Handling
|
||||
```python
|
||||
# Use appropriate exception types
|
||||
from fastapi import HTTPException, status
|
||||
|
||||
# API endpoints should return HTTP errors
|
||||
@router.get("/files/{file_id}")
|
||||
async def get_file(file_id: int):
|
||||
file = get_file_from_db(file_id)
|
||||
if not file:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"File with ID {file_id} not found"
|
||||
)
|
||||
return file
|
||||
|
||||
# Tasks should log and handle errors gracefully
|
||||
@celery_app.task(bind=True, max_retries=3)
|
||||
def process_document_task(self, file_path: str):
|
||||
try:
|
||||
result = process_document(file_path)
|
||||
return result
|
||||
except TemporaryError as e:
|
||||
logger.warning(f"Temporary error processing {file_path}: {e}")
|
||||
raise self.retry(exc=e, countdown=60)
|
||||
except PermanentError as e:
|
||||
logger.error(f"Permanent error processing {file_path}: {e}")
|
||||
# Don't retry permanent errors
|
||||
return {"error": str(e)}
|
||||
```
|
||||
|
||||
#### Testing
|
||||
```python
|
||||
# Mark tests appropriately
|
||||
@pytest.mark.unit
|
||||
def test_hash_file():
|
||||
"""Unit test for file hashing utility."""
|
||||
pass
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_upload_api_endpoint(client):
|
||||
"""Integration test for upload API."""
|
||||
pass
|
||||
|
||||
@pytest.mark.requires_external
|
||||
@pytest.mark.skip(reason="Requires OpenAI API key")
|
||||
def test_openai_metadata_extraction():
|
||||
"""Test actual OpenAI integration."""
|
||||
pass
|
||||
|
||||
# Use fixtures for common setup
|
||||
def test_document_processing(sample_pdf_path, db_session):
|
||||
"""Test uses fixtures from conftest.py"""
|
||||
pass
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📝 Common Tasks
|
||||
|
||||
### Adding a New API Endpoint
|
||||
|
||||
1. Create endpoint in `app/api/`:
|
||||
```python
|
||||
# app/api/my_feature.py
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from app.database import get_db
|
||||
from app.models import MyModel
|
||||
|
||||
router = APIRouter(prefix="/api/my-feature", tags=["my-feature"])
|
||||
|
||||
@router.get("/")
|
||||
async def list_items(db=Depends(get_db)):
|
||||
"""List all items."""
|
||||
items = db.query(MyModel).all()
|
||||
return items
|
||||
```
|
||||
|
||||
2. Register router in `app/api/__init__.py`:
|
||||
```python
|
||||
from app.api import my_feature
|
||||
|
||||
router.include_router(my_feature.router)
|
||||
```
|
||||
|
||||
3. Add tests in `tests/test_api_my_feature.py`
|
||||
|
||||
### Adding a New Celery Task
|
||||
|
||||
1. Create task in `app/tasks/`:
|
||||
```python
|
||||
# app/tasks/my_task.py
|
||||
from app.celery_app import celery_app
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@celery_app.task(bind=True, max_retries=3)
|
||||
def my_background_task(self, param: str):
|
||||
"""
|
||||
Description of what this task does.
|
||||
|
||||
Args:
|
||||
param: Description of parameter
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Processing task with param: {param}")
|
||||
# Task logic here
|
||||
return {"status": "success"}
|
||||
except Exception as e:
|
||||
logger.error(f"Task failed: {e}")
|
||||
raise self.retry(exc=e, countdown=60)
|
||||
```
|
||||
|
||||
2. Import in `app/tasks/__init__.py`
|
||||
3. Add tests in `tests/test_tasks.py`
|
||||
|
||||
### Adding a Database Model
|
||||
|
||||
1. Define model in `app/models.py`:
|
||||
```python
|
||||
class MyModel(Base):
|
||||
__tablename__ = "my_table"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
name = Column(String, nullable=False)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
```
|
||||
|
||||
2. Create migration:
|
||||
```bash
|
||||
cd /path/to/DocuElevate
|
||||
alembic revision --autogenerate -m "Add MyModel table"
|
||||
alembic upgrade head
|
||||
```
|
||||
|
||||
3. Add model to tests fixtures
|
||||
|
||||
### Adding a Storage Provider
|
||||
|
||||
1. Create provider module in `app/tasks/storage/`:
|
||||
```python
|
||||
# app/tasks/storage/my_provider.py
|
||||
from app.config import settings
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def upload_to_my_provider(file_path: str, metadata: dict) -> str:
|
||||
"""
|
||||
Upload file to My Provider.
|
||||
|
||||
Args:
|
||||
file_path: Local path to file
|
||||
metadata: Document metadata
|
||||
|
||||
Returns:
|
||||
URL or ID of uploaded file
|
||||
|
||||
Raises:
|
||||
ProviderError: If upload fails
|
||||
"""
|
||||
if not settings.my_provider_api_key:
|
||||
raise ValueError("MY_PROVIDER_API_KEY not configured")
|
||||
|
||||
# Implementation
|
||||
pass
|
||||
```
|
||||
|
||||
2. Add configuration to `app/config.py`:
|
||||
```python
|
||||
class Settings(BaseSettings):
|
||||
# ... existing settings ...
|
||||
my_provider_api_key: Optional[str] = None
|
||||
my_provider_endpoint: Optional[str] = None
|
||||
```
|
||||
|
||||
3. Add to `.env.demo`:
|
||||
```bash
|
||||
# My Provider
|
||||
MY_PROVIDER_API_KEY=your_api_key_here
|
||||
MY_PROVIDER_ENDPOINT=https://api.myprovider.com
|
||||
```
|
||||
|
||||
4. Add validator in `app/utils/config_validator/`
|
||||
5. Add tests with mocked API calls
|
||||
|
||||
---
|
||||
|
||||
## 🔒 Security Best Practices
|
||||
|
||||
### What to NEVER Do
|
||||
- ❌ Hardcode API keys, passwords, or secrets
|
||||
- ❌ Log sensitive data (passwords, tokens, API keys)
|
||||
- ❌ Accept unsanitized user input for file paths
|
||||
- ❌ Disable security features without documentation
|
||||
- ❌ Commit `.env` files or credentials
|
||||
|
||||
### What to ALWAYS Do
|
||||
- ✅ Use `settings` from `app/config.py` for all configuration
|
||||
- ✅ Validate and sanitize all user inputs
|
||||
- ✅ Use parameterized database queries (SQLAlchemy handles this)
|
||||
- ✅ Check file paths for directory traversal (`Path.resolve()`)
|
||||
- ✅ Use appropriate HTTP status codes (401, 403, 404, etc.)
|
||||
- ✅ Log security-relevant events
|
||||
- ✅ Add rate limiting for sensitive endpoints
|
||||
- ✅ Use HTTPS in production (documented in deployment guide)
|
||||
|
||||
### Input Validation Example
|
||||
```python
|
||||
from pathlib import Path
|
||||
from fastapi import HTTPException, status
|
||||
|
||||
def validate_file_path(file_path: str, base_dir: str = "/workdir") -> Path:
|
||||
"""Validate file path is within allowed directory."""
|
||||
try:
|
||||
path = Path(file_path).resolve()
|
||||
base = Path(base_dir).resolve()
|
||||
|
||||
# Ensure path is within base directory
|
||||
if not path.is_relative_to(base):
|
||||
raise ValueError("Path outside allowed directory")
|
||||
|
||||
return path
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid file path: {e}"
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🧪 Testing Strategy
|
||||
|
||||
### Test Coverage Goals
|
||||
- **Target:** 80% overall coverage
|
||||
- **Critical modules:** 90%+ (auth, config, database)
|
||||
- **Tasks:** 70%+ (complex to test with external services)
|
||||
- **API endpoints:** 85%+
|
||||
|
||||
### Test Types
|
||||
```python
|
||||
# Unit tests - fast, isolated, no external dependencies
|
||||
@pytest.mark.unit
|
||||
def test_hash_file_empty(tmp_path):
|
||||
"""Test hashing an empty file."""
|
||||
file = tmp_path / "empty.txt"
|
||||
file.write_text("")
|
||||
assert hash_file(str(file)) == "expected_hash"
|
||||
|
||||
# Integration tests - test multiple components together
|
||||
@pytest.mark.integration
|
||||
def test_upload_and_process(client, sample_pdf):
|
||||
"""Test full upload and processing flow."""
|
||||
response = client.post("/api/upload", files={"file": sample_pdf})
|
||||
assert response.status_code == 200
|
||||
|
||||
# External service tests - skipped by default
|
||||
@pytest.mark.requires_external
|
||||
@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="No API key")
|
||||
def test_real_openai_extraction():
|
||||
"""Test actual OpenAI API (skipped in CI)."""
|
||||
pass
|
||||
```
|
||||
|
||||
### Running Tests
|
||||
```bash
|
||||
# All tests
|
||||
pytest
|
||||
|
||||
# Specific category
|
||||
pytest -m unit
|
||||
pytest -m integration
|
||||
|
||||
# With coverage
|
||||
pytest --cov=app --cov-report=html
|
||||
|
||||
# Specific file
|
||||
pytest tests/test_api.py -v
|
||||
|
||||
# Skip external services
|
||||
pytest -m "not requires_external"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Performance Considerations
|
||||
|
||||
### Async/Await
|
||||
- FastAPI endpoints are async by default
|
||||
- Use `async def` for I/O-bound operations
|
||||
- Use regular `def` for CPU-bound operations
|
||||
|
||||
```python
|
||||
# Good - async for I/O
|
||||
@router.get("/files")
|
||||
async def list_files(db: Session = Depends(get_db)):
|
||||
files = db.query(FileRecord).all()
|
||||
return files
|
||||
|
||||
# Also good - sync for CPU-heavy
|
||||
@router.post("/hash")
|
||||
def hash_large_file(file: UploadFile):
|
||||
return compute_hash(file.file.read())
|
||||
```
|
||||
|
||||
### Database Queries
|
||||
```python
|
||||
# Good - single query with join
|
||||
files = db.query(FileRecord).options(
|
||||
joinedload(FileRecord.metadata)
|
||||
).filter(FileRecord.user_id == user_id).all()
|
||||
|
||||
# Bad - N+1 queries
|
||||
files = db.query(FileRecord).filter(FileRecord.user_id == user_id).all()
|
||||
for file in files:
|
||||
metadata = file.metadata # Triggers separate query each time
|
||||
```
|
||||
|
||||
### Celery Tasks
|
||||
```python
|
||||
# Long-running tasks should update progress
|
||||
@celery_app.task(bind=True)
|
||||
def process_large_batch(self, file_ids: List[int]):
|
||||
total = len(file_ids)
|
||||
for i, file_id in enumerate(file_ids):
|
||||
process_file(file_id)
|
||||
self.update_state(
|
||||
state='PROGRESS',
|
||||
meta={'current': i + 1, 'total': total}
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📚 Documentation Requirements
|
||||
|
||||
### Code Documentation
|
||||
```python
|
||||
def complex_function(param1: str, param2: int = 10) -> Dict[str, Any]:
|
||||
"""
|
||||
One-line summary of what the function does.
|
||||
|
||||
More detailed explanation if needed. Can span multiple
|
||||
lines and include examples.
|
||||
|
||||
Args:
|
||||
param1: Description of param1
|
||||
param2: Description of param2, defaults to 10
|
||||
|
||||
Returns:
|
||||
Dictionary containing:
|
||||
- key1: Description
|
||||
- key2: Description
|
||||
|
||||
Raises:
|
||||
ValueError: If param1 is empty
|
||||
FileNotFoundError: If file doesn't exist
|
||||
|
||||
Examples:
|
||||
>>> result = complex_function("test", 5)
|
||||
>>> print(result['key1'])
|
||||
'value'
|
||||
"""
|
||||
pass
|
||||
```
|
||||
|
||||
### API Documentation
|
||||
- Use FastAPI's automatic OpenAPI generation
|
||||
- Add descriptions to endpoints
|
||||
- Document request/response models
|
||||
- Include example requests/responses
|
||||
|
||||
```python
|
||||
@router.post(
|
||||
"/upload",
|
||||
response_model=UploadResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
summary="Upload a document",
|
||||
description="Upload a document for processing. Supports PDF, images, and Office documents.",
|
||||
responses={
|
||||
201: {"description": "Document uploaded successfully"},
|
||||
400: {"description": "Invalid file format"},
|
||||
413: {"description": "File too large"},
|
||||
}
|
||||
)
|
||||
async def upload_document(
|
||||
file: UploadFile = File(..., description="Document file to upload"),
|
||||
tags: List[str] = Query([], description="Optional tags for the document"),
|
||||
):
|
||||
"""Upload endpoint implementation."""
|
||||
pass
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🐛 Debugging
|
||||
|
||||
### Logging
|
||||
```python
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Use appropriate log levels
|
||||
logger.debug("Detailed information for debugging")
|
||||
logger.info("General information about operation")
|
||||
logger.warning("Warning about potential issue")
|
||||
logger.error("Error that needs attention")
|
||||
logger.critical("Critical error that needs immediate attention")
|
||||
|
||||
# Include context in logs
|
||||
logger.info(f"Processing document: {file_id}, user: {user_id}")
|
||||
|
||||
# Don't log sensitive data
|
||||
logger.info(f"User authenticated") # Good
|
||||
logger.info(f"Password: {password}") # BAD!
|
||||
```
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Import Errors**
|
||||
- Check if module is in `__init__.py`
|
||||
- Verify Python path includes project root
|
||||
- Look for circular imports
|
||||
|
||||
2. **Database Issues**
|
||||
- Check if migrations are up to date: `alembic upgrade head`
|
||||
- Verify DATABASE_URL is set correctly
|
||||
- Check if tables exist: `sqlite3 app/database.db .schema`
|
||||
|
||||
3. **Celery Issues**
|
||||
- Verify Redis is running: `redis-cli ping`
|
||||
- Check Celery worker logs
|
||||
- Ensure tasks are imported in `celery_worker.py`
|
||||
|
||||
4. **Test Failures**
|
||||
- Check if test database is clean (use fixtures)
|
||||
- Verify environment variables are set in `conftest.py`
|
||||
- Run single test to isolate issue: `pytest tests/test_file.py::test_name -v`
|
||||
|
||||
---
|
||||
|
||||
## 🔄 Git Workflow & Versioning
|
||||
|
||||
### Branch Names
|
||||
- `feature/description` - New features
|
||||
- `bugfix/description` - Bug fixes
|
||||
- `hotfix/description` - Urgent production fixes
|
||||
- `refactor/description` - Code refactoring
|
||||
- `docs/description` - Documentation updates
|
||||
|
||||
### Conventional Commits (REQUIRED)
|
||||
|
||||
**All commit messages MUST follow the Conventional Commits specification for automated versioning.**
|
||||
|
||||
#### Format
|
||||
```
|
||||
<type>(<scope>): <subject>
|
||||
|
||||
<body>
|
||||
|
||||
<footer>
|
||||
```
|
||||
|
||||
#### Commit Types and Version Bumps
|
||||
- **feat**: New feature → **minor version bump** (0.5.0 → 0.6.0)
|
||||
- **fix**: Bug fix → **patch version bump** (0.5.0 → 0.5.1)
|
||||
- **perf**: Performance improvement → **patch version bump**
|
||||
- **docs**: Documentation only → **no version bump**
|
||||
- **style**: Code style/formatting → **no version bump**
|
||||
- **refactor**: Code refactoring → **no version bump**
|
||||
- **test**: Test changes → **no version bump**
|
||||
- **build**: Build system changes → **no version bump**
|
||||
- **ci**: CI/CD changes → **no version bump**
|
||||
- **chore**: Other changes → **no version bump**
|
||||
|
||||
#### Breaking Changes
|
||||
Add `!` after type/scope or include `BREAKING CHANGE:` in footer for **major version bump**:
|
||||
```
|
||||
feat(api)!: redesign authentication endpoints
|
||||
|
||||
BREAKING CHANGE: OAuth2 tokens now required instead of API keys
|
||||
```
|
||||
Result: 0.5.0 → 1.0.0
|
||||
|
||||
#### Scope Examples
|
||||
- `api` - REST API changes
|
||||
- `ui` - Frontend/UI changes
|
||||
- `auth` - Authentication
|
||||
- `storage` - Storage providers
|
||||
- `ocr` - OCR processing
|
||||
- `tasks` - Celery tasks
|
||||
- `config` - Configuration
|
||||
|
||||
#### Good Commit Examples
|
||||
```
|
||||
feat(storage): add Amazon S3 storage provider
|
||||
|
||||
Implements S3StorageProvider with upload, download, delete operations.
|
||||
Includes configuration for bucket, region, and credentials.
|
||||
|
||||
Closes #123
|
||||
```
|
||||
|
||||
```
|
||||
fix(ocr): handle PDFs without text layer
|
||||
|
||||
Previously failed silently. Now properly processes through Azure.
|
||||
|
||||
Fixes #456
|
||||
```
|
||||
|
||||
```
|
||||
docs: update deployment guide with Docker Compose
|
||||
|
||||
Added step-by-step instructions for Docker Compose deployment.
|
||||
```
|
||||
|
||||
### Semantic Release Automation
|
||||
|
||||
DocuElevate uses `python-semantic-release` for automated version management.
|
||||
|
||||
#### How It Works
|
||||
1. **PR merges to main** with conventional commits
|
||||
2. **semantic-release analyzes** commit messages
|
||||
3. **Automatic updates**:
|
||||
- Bumps `VERSION` file
|
||||
- Updates `CHANGELOG.md`
|
||||
- Creates Git tag (e.g., `v0.6.0`)
|
||||
- Creates GitHub Release
|
||||
- Triggers Docker builds
|
||||
|
||||
#### Agent Rules
|
||||
- ✅ **DO**: Write conventional commit messages
|
||||
- ✅ **DO**: Use correct commit types
|
||||
- ✅ **DO**: Include `BREAKING CHANGE:` when applicable
|
||||
- ❌ **DON'T**: Manually edit `VERSION` file
|
||||
- ❌ **DON'T**: Manually edit `CHANGELOG.md`
|
||||
- ❌ **DON'T**: Create version tags or releases manually
|
||||
|
||||
### Pull Requests
|
||||
1. Create PR with descriptive title (conventional format if single change)
|
||||
2. Fill out PR template
|
||||
3. Link related issues
|
||||
4. Ensure CI passes
|
||||
5. Request reviews
|
||||
6. Address feedback
|
||||
7. Merge when approved (commits retain conventional format)
|
||||
|
||||
---
|
||||
|
||||
## ✅ Pre-commit Checklist
|
||||
|
||||
Before submitting code:
|
||||
|
||||
- [ ] Code follows style guide (Ruff formatted)
|
||||
- [ ] Commit messages use conventional commit format
|
||||
- [ ] All tests pass (`pytest`)
|
||||
- [ ] New code has tests
|
||||
- [ ] Coverage doesn't decrease
|
||||
- [ ] Documentation updated if needed
|
||||
- [ ] No secrets or credentials in code
|
||||
- [ ] Linting passes (`ruff check`)
|
||||
- [ ] Type hints added (`mypy` clean)
|
||||
- [ ] No manual edits to `VERSION` or `CHANGELOG.md`
|
||||
- [ ] Security scan passed (included in `ruff check`)
|
||||
|
||||
Run full check:
|
||||
```bash
|
||||
pytest --cov=app
|
||||
ruff check app/ tests/
|
||||
ruff format --check app/ tests/
|
||||
mypy app/
|
||||
```
|
||||
|
||||
**Note:** This project uses Ruff, which replaces Black, Flake8, isort, and Bandit with a single, faster tool.
|
||||
|
||||
---
|
||||
|
||||
## 🤝 Agent Collaboration
|
||||
|
||||
### When to Ask for Help
|
||||
- Breaking changes needed
|
||||
- Unsure about architecture decision
|
||||
- Security implications unclear
|
||||
- Performance impact unknown
|
||||
- Tests consistently failing
|
||||
|
||||
### How to Document Changes
|
||||
1. Update relevant documentation
|
||||
2. Add comments for complex logic
|
||||
3. Update TODO.md if introducing tech debt
|
||||
4. Note breaking changes in commit message
|
||||
5. Update API documentation if endpoints changed
|
||||
|
||||
---
|
||||
|
||||
## 📞 Resources
|
||||
|
||||
- **Main README:** [README.md](README.md)
|
||||
- **API Docs:** http://localhost:8000/docs (when running)
|
||||
- **User Guide:** [docs/UserGuide.md](docs/UserGuide.md)
|
||||
- **Deployment:** [docs/DeploymentGuide.md](docs/DeploymentGuide.md)
|
||||
- **Troubleshooting:** [docs/Troubleshooting.md](docs/Troubleshooting.md)
|
||||
- **GitHub Issues:** Track bugs and features
|
||||
- **GitHub Discussions:** Questions and community
|
||||
|
||||
---
|
||||
|
||||
*This guide is a living document. Improvements welcome via PR!*
|
||||
@@ -1,122 +0,0 @@
|
||||
# Agent Pre-PR Ruff Policy
|
||||
|
||||
This document defines the policy for Copilot/AI Agents when creating or updating pull requests in the DocuElevate repository.
|
||||
|
||||
## Required Steps Before Creating/Updating a PR
|
||||
|
||||
### 1. Run the Ruff Auto-Fix Script
|
||||
|
||||
Before creating or updating a branch or opening a PR, **ALWAYS** run:
|
||||
|
||||
```bash
|
||||
./scripts/run-ruff-and-fix.sh
|
||||
```
|
||||
|
||||
### 2. Handle Script Exit Codes
|
||||
|
||||
**If the script exits with code 0 (success):**
|
||||
- ✅ All ruff checks passed or were auto-fixed
|
||||
- Continue with PR creation process
|
||||
- Push the branch (including any auto-fix commits)
|
||||
|
||||
**If the script exits with code non-zero (failure):**
|
||||
- ❌ Stop PR creation immediately
|
||||
- Do NOT open the PR
|
||||
- Report the ruff failures to the human user
|
||||
- Include the full `ruff check` output in your response
|
||||
- Create an issue or report the failures for manual resolution
|
||||
|
||||
### 3. Handle Auto-Fix Commits
|
||||
|
||||
If the script commits fixes:
|
||||
- The script will automatically commit changes with message: `style: ruff auto-fixes`
|
||||
- Push the branch with the commit(s) before continuing PR creation
|
||||
- This ensures CI sees the fixed code when the PR is opened
|
||||
|
||||
## Example Agent Workflow
|
||||
|
||||
```
|
||||
1. Clone repository and create feature branch
|
||||
2. Make code changes to implement feature
|
||||
3. Run: ./scripts/run-ruff-and-fix.sh
|
||||
4. Check exit code:
|
||||
- If 0: Push branch and create PR
|
||||
- If non-zero: Report failures, do NOT create PR
|
||||
5. Run project tests (if ruff passed)
|
||||
6. If all checks pass: Create the PR
|
||||
7. If any check fails: Report failures instead of creating PR
|
||||
```
|
||||
|
||||
## Why This Policy Exists
|
||||
|
||||
1. **Prevent CI Failures**: Ensures PRs don't break ruff checks in CI
|
||||
2. **Auto-Fix Minor Issues**: Automatically fixes formatting and simple linting issues
|
||||
3. **Surface Manual Issues Early**: Identifies issues that need human attention before PR creation
|
||||
4. **Maintain Code Quality**: Enforces consistent code style across the repository
|
||||
|
||||
## Integration with Existing CI
|
||||
|
||||
DocuElevate has two workflows that handle ruff:
|
||||
|
||||
1. **`.github/workflows/ci.yml`** (Lint Job)
|
||||
- Runs `ruff check` (without --fix) on all pushes and PRs
|
||||
- Fails CI if issues are found
|
||||
- Runs early in the pipeline to catch style issues before tests
|
||||
|
||||
2. **`.github/workflows/ruff-auto-fix.yml`**
|
||||
- Runs on PRs when Python files change
|
||||
- Automatically applies `ruff --fix` and `ruff format`
|
||||
- Commits fixes back to the PR branch
|
||||
- Posts a comment notifying the author
|
||||
|
||||
This agent script ensures that most issues are caught and fixed **before** the PR is created, reducing the need for the auto-fix workflow to intervene.
|
||||
|
||||
## Local Development
|
||||
|
||||
Developers should also use this script or set up pre-commit hooks:
|
||||
|
||||
```bash
|
||||
# Install pre-commit hooks (recommended)
|
||||
pip install pre-commit
|
||||
pre-commit install
|
||||
|
||||
# Or run manually before committing
|
||||
./scripts/run-ruff-and-fix.sh
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Script fails with "ruff: command not found"
|
||||
|
||||
The script installs ruff automatically. If this fails:
|
||||
```bash
|
||||
pip install ruff
|
||||
```
|
||||
|
||||
### Script fails with Git errors
|
||||
|
||||
Ensure you're in a Git repository with proper configuration:
|
||||
```bash
|
||||
git config user.name "Your Name"
|
||||
git config user.email "your.email@example.com"
|
||||
```
|
||||
|
||||
### Ruff issues remain after --fix
|
||||
|
||||
Some issues cannot be auto-fixed (e.g., unused imports, complex logic issues). These require manual resolution:
|
||||
1. Review the ruff output
|
||||
2. Fix the issues manually
|
||||
3. Run the script again to verify
|
||||
|
||||
## Configuration
|
||||
|
||||
Ruff configuration is in `pyproject.toml` under `[tool.ruff]` and `[tool.ruff.lint]`.
|
||||
|
||||
Default settings:
|
||||
- Line length: 120 characters
|
||||
- Target Python version: 3.11+
|
||||
- Enabled rules: Pyflakes (F), pycodestyle (E, W), isort (I), bandit (S), flake8-bugbear (B), pylint (PL)
|
||||
|
||||
## Questions?
|
||||
|
||||
See the [Contributing Guide](CONTRIBUTING.md) for more information on code quality standards and development workflow.
|
||||
@@ -1 +0,0 @@
|
||||
2026-03-01T17:23:25Z
|
||||
-1024
File diff suppressed because it is too large
Load Diff
@@ -1,133 +0,0 @@
|
||||
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
We as members, contributors, and leaders pledge to make participation in our
|
||||
community a harassment-free experience for everyone, regardless of age, body
|
||||
size, visible or invisible disability, ethnicity, sex characteristics, gender
|
||||
identity and expression, level of experience, education, socio-economic status,
|
||||
nationality, personal appearance, race, caste, color, religion, or sexual
|
||||
identity and orientation.
|
||||
|
||||
We pledge to act and interact in ways that contribute to an open, welcoming,
|
||||
diverse, inclusive, and healthy community.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to a positive environment for our
|
||||
community include:
|
||||
|
||||
* Demonstrating empathy and kindness toward other people
|
||||
* Being respectful of differing opinions, viewpoints, and experiences
|
||||
* Giving and gracefully accepting constructive feedback
|
||||
* Accepting responsibility and apologizing to those affected by our mistakes,
|
||||
and learning from the experience
|
||||
* Focusing on what is best not just for us as individuals, but for the overall
|
||||
community
|
||||
|
||||
Examples of unacceptable behavior include:
|
||||
|
||||
* The use of sexualized language or imagery, and sexual attention or advances of
|
||||
any kind
|
||||
* Trolling, insulting or derogatory comments, and personal or political attacks
|
||||
* Public or private harassment
|
||||
* Publishing others' private information, such as a physical or email address,
|
||||
without their explicit permission
|
||||
* Other conduct which could reasonably be considered inappropriate in a
|
||||
professional setting
|
||||
|
||||
## Enforcement Responsibilities
|
||||
|
||||
Community leaders are responsible for clarifying and enforcing our standards of
|
||||
acceptable behavior and will take appropriate and fair corrective action in
|
||||
response to any behavior that they deem inappropriate, threatening, offensive,
|
||||
or harmful.
|
||||
|
||||
Community leaders have the right and responsibility to remove, edit, or reject
|
||||
comments, commits, code, wiki edits, issues, and other contributions that are
|
||||
not aligned to this Code of Conduct, and will communicate reasons for moderation
|
||||
decisions when appropriate.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies within all community spaces, and also applies when
|
||||
an individual is officially representing the community in public spaces.
|
||||
Examples of representing our community include using an official email address,
|
||||
posting via an official social media account, or acting as an appointed
|
||||
representative at an online or offline event.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported to the community leaders responsible for enforcement at
|
||||
code-of-conduct@fret.de.
|
||||
All complaints will be reviewed and investigated promptly and fairly.
|
||||
|
||||
All community leaders are obligated to respect the privacy and security of the
|
||||
reporter of any incident.
|
||||
|
||||
## Enforcement Guidelines
|
||||
|
||||
Community leaders will follow these Community Impact Guidelines in determining
|
||||
the consequences for any action they deem in violation of this Code of Conduct:
|
||||
|
||||
### 1. Correction
|
||||
|
||||
**Community Impact**: Use of inappropriate language or other behavior deemed
|
||||
unprofessional or unwelcome in the community.
|
||||
|
||||
**Consequence**: A private, written warning from community leaders, providing
|
||||
clarity around the nature of the violation and an explanation of why the
|
||||
behavior was inappropriate. A public apology may be requested.
|
||||
|
||||
### 2. Warning
|
||||
|
||||
**Community Impact**: A violation through a single incident or series of
|
||||
actions.
|
||||
|
||||
**Consequence**: A warning with consequences for continued behavior. No
|
||||
interaction with the people involved, including unsolicited interaction with
|
||||
those enforcing the Code of Conduct, for a specified period of time. This
|
||||
includes avoiding interactions in community spaces as well as external channels
|
||||
like social media. Violating these terms may lead to a temporary or permanent
|
||||
ban.
|
||||
|
||||
### 3. Temporary Ban
|
||||
|
||||
**Community Impact**: A serious violation of community standards, including
|
||||
sustained inappropriate behavior.
|
||||
|
||||
**Consequence**: A temporary ban from any sort of interaction or public
|
||||
communication with the community for a specified period of time. No public or
|
||||
private interaction with the people involved, including unsolicited interaction
|
||||
with those enforcing the Code of Conduct, is allowed during this period.
|
||||
Violating these terms may lead to a permanent ban.
|
||||
|
||||
### 4. Permanent Ban
|
||||
|
||||
**Community Impact**: Demonstrating a pattern of violation of community
|
||||
standards, including sustained inappropriate behavior, harassment of an
|
||||
individual, or aggression toward or disparagement of classes of individuals.
|
||||
|
||||
**Consequence**: A permanent ban from any sort of public interaction within the
|
||||
community.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
|
||||
version 2.1, available at
|
||||
[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
|
||||
|
||||
Community Impact Guidelines were inspired by
|
||||
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
|
||||
|
||||
For answers to common questions about this code of conduct, see the FAQ at
|
||||
[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
|
||||
[https://www.contributor-covenant.org/translations][translations].
|
||||
|
||||
[homepage]: https://www.contributor-covenant.org
|
||||
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
|
||||
[Mozilla CoC]: https://github.com/mozilla/diversity
|
||||
[FAQ]: https://www.contributor-covenant.org/faq
|
||||
[translations]: https://www.contributor-covenant.org/translations
|
||||
-449
@@ -1,449 +0,0 @@
|
||||
# Contributing to DocuElevate
|
||||
|
||||
Thank you for your interest in contributing to DocuElevate! This document provides guidelines and instructions for contributing to the project.
|
||||
|
||||
## Code of Conduct
|
||||
|
||||
By participating in this project, you agree to abide by the [Code of Conduct](CODE_OF_CONDUCT.md).
|
||||
|
||||
## How to Contribute
|
||||
|
||||
### Reporting Bugs
|
||||
|
||||
If you find a bug in the codebase, please submit an issue on GitHub with:
|
||||
|
||||
1. A clear title and description
|
||||
2. Steps to reproduce the issue
|
||||
3. Expected behavior
|
||||
4. Actual behavior
|
||||
5. Environment information (OS, Docker version, etc.)
|
||||
|
||||
### Feature Requests
|
||||
|
||||
We welcome feature requests! Please submit an issue with:
|
||||
|
||||
1. A clear title and description
|
||||
2. The problem the feature would solve
|
||||
3. Any ideas you have for implementing the feature
|
||||
|
||||
### Pull Requests
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a new branch for your changes
|
||||
3. Make your changes
|
||||
4. **Follow conventional commit format** (see below)
|
||||
5. Run the tests to ensure everything works
|
||||
6. Submit a pull request with a clear description of the changes
|
||||
|
||||
## Commit Message Format
|
||||
|
||||
DocuElevate follows the [Conventional Commits](https://www.conventionalcommits.org/) specification for commit messages. This enables automatic version bumping and changelog generation.
|
||||
|
||||
### Format
|
||||
|
||||
```
|
||||
<type>(<scope>): <subject>
|
||||
|
||||
<body>
|
||||
|
||||
<footer>
|
||||
```
|
||||
|
||||
### Type
|
||||
|
||||
Must be one of the following:
|
||||
|
||||
- **feat**: A new feature (triggers minor version bump)
|
||||
- **fix**: A bug fix (triggers patch version bump)
|
||||
- **docs**: Documentation only changes
|
||||
- **style**: Changes that don't affect code meaning (formatting, etc.)
|
||||
- **refactor**: Code change that neither fixes a bug nor adds a feature
|
||||
- **perf**: Performance improvement (triggers patch version bump)
|
||||
- **test**: Adding or updating tests
|
||||
- **build**: Changes to build system or dependencies
|
||||
- **ci**: Changes to CI configuration files and scripts
|
||||
- **chore**: Other changes that don't modify src or test files
|
||||
|
||||
### Scope (Optional)
|
||||
|
||||
The scope should be the name of the affected module or area:
|
||||
- `api` - REST API changes
|
||||
- `ui` - Frontend/UI changes
|
||||
- `auth` - Authentication changes
|
||||
- `storage` - Storage provider changes
|
||||
- `ocr` - OCR processing changes
|
||||
- `tasks` - Celery task changes
|
||||
- `config` - Configuration changes
|
||||
|
||||
### Subject
|
||||
|
||||
The subject contains a succinct description of the change:
|
||||
- Use imperative, present tense: "change" not "changed" nor "changes"
|
||||
- Don't capitalize first letter
|
||||
- No period (.) at the end
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
For breaking changes, add `!` after the type/scope or include `BREAKING CHANGE:` in the footer:
|
||||
|
||||
```
|
||||
feat!: redesign authentication API
|
||||
|
||||
BREAKING CHANGE: The /api/auth endpoint now requires OAuth2 tokens instead of API keys.
|
||||
```
|
||||
|
||||
This triggers a major version bump.
|
||||
|
||||
### Examples
|
||||
|
||||
```
|
||||
feat(storage): add support for Amazon S3 storage provider
|
||||
|
||||
Add S3StorageProvider class with upload, download, and delete operations.
|
||||
Includes configuration options for bucket name, region, and credentials.
|
||||
|
||||
Closes #123
|
||||
```
|
||||
|
||||
```
|
||||
fix(ocr): handle PDF files without text layer
|
||||
|
||||
Previously, PDFs without existing text layers would fail silently.
|
||||
Now properly processes them through Azure Document Intelligence.
|
||||
|
||||
Fixes #456
|
||||
```
|
||||
|
||||
```
|
||||
docs: update deployment guide with Docker Compose setup
|
||||
|
||||
Added step-by-step instructions for deploying with Docker Compose,
|
||||
including environment variable configuration and service dependencies.
|
||||
```
|
||||
|
||||
```
|
||||
chore: update dependencies to fix security vulnerabilities
|
||||
|
||||
Updated authlib to 1.6.5+ and starlette to 0.49.1+
|
||||
```
|
||||
|
||||
## Versioning and Releases
|
||||
|
||||
DocuElevate uses [semantic-release](https://github.com/semantic-release/semantic-release) for automated version management and releases:
|
||||
|
||||
- **Releases are automated**: When PRs are merged to `main`, semantic-release analyzes commit messages and automatically:
|
||||
- Determines the next version number
|
||||
- Updates the `VERSION` file
|
||||
- Generates/updates `CHANGELOG.md`
|
||||
- Creates a Git tag with `v` prefix (e.g., `v0.6.0`)
|
||||
- Creates a GitHub Release with auto-generated notes
|
||||
- Triggers Docker image builds with the new version tag
|
||||
|
||||
- **Version Bumps**:
|
||||
- `feat:` commits → minor version bump (0.5.0 → 0.6.0)
|
||||
- `fix:` or `perf:` commits → patch version bump (0.5.0 → 0.5.1)
|
||||
- `feat!:` or `BREAKING CHANGE:` → major version bump (0.5.0 → 1.0.0)
|
||||
- Other commit types (docs, chore, etc.) → no version bump
|
||||
|
||||
- **Manual Version Changes**: Do NOT manually edit `VERSION` or `CHANGELOG.md` - these are managed by semantic-release
|
||||
|
||||
## Documentation-First Development
|
||||
|
||||
Documentation is a first-class citizen in DocuElevate. Every contribution **must** include relevant documentation updates. This is not optional.
|
||||
|
||||
### What Requires Documentation
|
||||
|
||||
| Change Type | Required Documentation |
|
||||
|-------------|----------------------|
|
||||
| New feature | User Guide + API docs (if API change) + Configuration Guide (if new config) |
|
||||
| Bug fix | Troubleshooting guide (if user-facing) |
|
||||
| New config option | ConfigurationGuide.md + `.env.demo` example |
|
||||
| New API endpoint | docs/API.md |
|
||||
| Deployment change | DeploymentGuide.md |
|
||||
| Security change | SECURITY_AUDIT.md |
|
||||
| Breaking change | CHANGELOG.md note + migration instructions |
|
||||
|
||||
### Documentation Standards
|
||||
|
||||
- Keep `docs/` files in sync with code changes in the same PR
|
||||
- Update `TODO.md` when completing or adding tasks
|
||||
- `CHANGELOG.md` is generated automatically—**do not add regular release entries manually**. Retroactive corrections to historical entries are the only acceptable exception.
|
||||
- Screenshots in README and docs should reflect current UI; update them when the UI changes significantly
|
||||
- Use present tense and second person ("you") in user-facing docs
|
||||
|
||||
### Automated Changelog
|
||||
|
||||
`CHANGELOG.md` is generated automatically by [python-semantic-release](https://github.com/python-semantic-release/python-semantic-release) on every merge to `main`. **Do not edit it manually.** Your commit messages (following Conventional Commits) drive the changelog content.
|
||||
|
||||
---
|
||||
|
||||
## Pull Request Checklist
|
||||
|
||||
Before submitting a pull request:
|
||||
|
||||
- [ ] Code follows the project style guide (Ruff)
|
||||
- [ ] Commit messages follow conventional commit format
|
||||
- [ ] Pre-commit hooks installed and passing (see below)
|
||||
- [ ] Tests added/updated for new functionality
|
||||
- [ ] **Documentation updated** for any user-facing, API, or configuration changes
|
||||
- [ ] No manual edits to `VERSION` or `CHANGELOG.md`
|
||||
- [ ] All tests pass locally
|
||||
- [ ] Security scan passes (if applicable)
|
||||
|
||||
## Development Environment
|
||||
|
||||
### Setting Up Your Environment
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone https://github.com/christianlouis/DocuElevate.git
|
||||
cd DocuElevate
|
||||
|
||||
# Create a virtual environment
|
||||
python -m venv venv
|
||||
source venv/bin/activate # On Windows: venv\Scripts\activate
|
||||
|
||||
# Install dependencies
|
||||
pip install -r requirements.txt
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
# Install pre-commit hooks (recommended)
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
### Pre-commit Hooks
|
||||
|
||||
Pre-commit hooks automatically check your code before each commit, catching issues early:
|
||||
|
||||
```bash
|
||||
# Install the hooks (one-time setup)
|
||||
pre-commit install
|
||||
|
||||
# Run hooks manually on all files
|
||||
pre-commit run --all-files
|
||||
|
||||
# Run hooks on staged files (happens automatically on commit)
|
||||
pre-commit run
|
||||
```
|
||||
|
||||
The pre-commit hooks include:
|
||||
- **Ruff** - Linting and formatting (with auto-fix)
|
||||
- **Mypy** - Type checking
|
||||
- **detect-secrets** - Secret detection
|
||||
- **Conventional commits** - Commit message validation
|
||||
- File checks (trailing whitespace, large files, etc.)
|
||||
|
||||
### Running Tests
|
||||
|
||||
DocuElevate has comprehensive test coverage including unit tests, integration tests, and end-to-end tests. Tests are automatically configured with the necessary environment variables.
|
||||
|
||||
#### Quick Test Commands
|
||||
|
||||
```bash
|
||||
# Run all tests (default configuration)
|
||||
pytest
|
||||
|
||||
# Run with verbose output
|
||||
pytest -v
|
||||
|
||||
# Run with coverage report
|
||||
pytest --cov=app --cov-report=term-missing
|
||||
|
||||
# Run only unit tests (fast, no Docker required)
|
||||
pytest -m unit
|
||||
|
||||
# Run only integration tests
|
||||
pytest -m integration
|
||||
|
||||
# Run specific test file
|
||||
pytest tests/test_api.py -v
|
||||
```
|
||||
|
||||
#### Test Environment Configuration
|
||||
|
||||
Tests automatically configure the required environment variables in `tests/conftest.py`:
|
||||
|
||||
- `DATABASE_URL`: Uses SQLite in-memory database for fast, isolated tests
|
||||
- `AUTH_ENABLED`: Set to `False` by default for simpler unit tests
|
||||
- `SESSION_SECRET`: Pre-configured with a valid 32+ character secret for tests that need it
|
||||
- `OPENAI_API_KEY`, `AZURE_AI_KEY`, etc.: Pre-configured with test values
|
||||
|
||||
**No manual environment setup is needed to run tests!**
|
||||
|
||||
#### Testing with Authentication Enabled
|
||||
|
||||
Some tests specifically verify authentication behavior with `AUTH_ENABLED=True`. These tests:
|
||||
|
||||
1. Use `@patch("app.auth.AUTH_ENABLED", True)` to enable auth for specific tests
|
||||
2. Properly configure `SESSION_SECRET` (already set in conftest.py)
|
||||
3. Mock user sessions to test protected endpoints
|
||||
4. Verify login redirects and access control
|
||||
|
||||
Example:
|
||||
```python
|
||||
from unittest.mock import patch
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_protected_endpoint_with_auth(client):
|
||||
"""Test endpoint requires authentication when auth is enabled."""
|
||||
with patch("app.auth.AUTH_ENABLED", True):
|
||||
# Test will verify redirect to /login
|
||||
response = client.get("/protected-page")
|
||||
assert response.status_code == 302
|
||||
```
|
||||
|
||||
#### Integration Tests with Docker
|
||||
|
||||
Some tests require Docker to spin up real infrastructure (PostgreSQL, Redis, WebDAV, etc.):
|
||||
|
||||
```bash
|
||||
# Run integration tests that need Docker
|
||||
pytest -m requires_docker -v
|
||||
|
||||
# Run end-to-end tests with full stack
|
||||
pytest -m e2e -v
|
||||
```
|
||||
|
||||
See [tests/README_INTEGRATION_TESTS.md](tests/README_INTEGRATION_TESTS.md) for detailed information about integration testing.
|
||||
|
||||
#### Test Markers
|
||||
|
||||
Tests are organized using pytest markers:
|
||||
|
||||
- `@pytest.mark.unit` - Fast unit tests with mocks
|
||||
- `@pytest.mark.integration` - Integration tests with some real services
|
||||
- `@pytest.mark.e2e` - Full end-to-end tests
|
||||
- `@pytest.mark.requires_docker` - Requires Docker to run
|
||||
- `@pytest.mark.slow` - Tests that take significant time
|
||||
- `@pytest.mark.security` - Security-related tests
|
||||
|
||||
#### Running Tests in CI
|
||||
|
||||
Tests run automatically in GitHub Actions for all pull requests. The CI workflow is organized in stages:
|
||||
|
||||
**Stage 1: Ruff Lint & Format** (runs first, in parallel with dependency scan)
|
||||
- Checks code style, formatting, and basic security issues
|
||||
- Must pass before tests run
|
||||
|
||||
**Stage 1b: Dependency Vulnerability Scan** (runs in parallel with lint)
|
||||
- Runs `pip-audit` against `requirements.txt` and `requirements-dev.txt`
|
||||
- Fails the build if any known vulnerabilities are detected
|
||||
- Checks the OSV and PyPA advisory databases
|
||||
- Runs independently at the same time as Stage 1 so it does not add to total pipeline time
|
||||
|
||||
**Stage 2: Tests & Type Checking** (runs after lint and dependency scan both pass)
|
||||
|
||||
| Job | Tool | What it checks |
|
||||
|--------|--------|--------------------------------------|
|
||||
| `test` | pytest | Unit/integration tests + coverage |
|
||||
| `mypy` | mypy | Static type checking |
|
||||
|
||||
**Stage 3: Docker Build** (runs after all checks pass)
|
||||
- Builds and pushes Docker images
|
||||
|
||||
**Stage 4: Deploy** (only on main branch)
|
||||
- Deploys to production
|
||||
|
||||
**Auto-fix Workflow:**
|
||||
- A separate `ruff-auto-fix` workflow automatically fixes formatting issues on PRs
|
||||
- Commits fixes back to the PR branch
|
||||
- Only runs on PRs from the same repository (not forks)
|
||||
|
||||
For full details see [docs/CIWorkflow.md](docs/CIWorkflow.md) and [docs/CIToolsGuide.md](docs/CIToolsGuide.md).
|
||||
|
||||
### Code Style
|
||||
|
||||
DocuElevate uses **Ruff** for all Python code quality checks:
|
||||
|
||||
- **Linting** - PEP 8 style, code quality, and security checks
|
||||
- **Formatting** - Consistent code formatting (120 character line length)
|
||||
- **Import sorting** - Organized imports
|
||||
|
||||
```bash
|
||||
# Check for linting issues
|
||||
ruff check app/ tests/
|
||||
|
||||
# Auto-fix linting issues
|
||||
ruff check app/ tests/ --fix
|
||||
|
||||
# Check formatting
|
||||
ruff format --check app/ tests/
|
||||
|
||||
# Auto-format code
|
||||
ruff format app/ tests/
|
||||
```
|
||||
|
||||
**Note:** The pre-commit hooks and CI pipeline will automatically check (and optionally fix) these for you.
|
||||
|
||||
### Dependency Vulnerability Scanning
|
||||
|
||||
DocuElevate uses **pip-audit** to scan dependencies for known security vulnerabilities. The CI pipeline runs this automatically and **blocks builds** if any vulnerabilities are found.
|
||||
|
||||
To run locally before pushing:
|
||||
|
||||
```bash
|
||||
# Scan production dependencies
|
||||
pip-audit -r requirements.txt --desc on
|
||||
|
||||
# Scan all dependencies (including dev)
|
||||
pip-audit -r requirements-dev.txt --desc on
|
||||
```
|
||||
|
||||
If pip-audit is not installed, add it with:
|
||||
|
||||
```bash
|
||||
pip install pip-audit
|
||||
```
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
DocuElevate/
|
||||
├── app/ # Main application code
|
||||
│ ├── api/ # REST API endpoints (organized by feature)
|
||||
│ ├── tasks/ # Celery background tasks
|
||||
│ ├── views/ # UI routes and template rendering
|
||||
│ ├── utils/ # Utility functions and helpers
|
||||
│ ├── config.py # Configuration management (Pydantic)
|
||||
│ ├── database.py # Database setup and session management
|
||||
│ ├── models.py # SQLAlchemy models
|
||||
│ ├── main.py # FastAPI app initialization
|
||||
│ └── auth.py # Authentication logic
|
||||
├── frontend/ # Frontend assets
|
||||
│ ├── static/ # CSS, JavaScript, images
|
||||
│ └── templates/ # Jinja2 HTML templates
|
||||
├── tests/ # Test suite
|
||||
├── docs/ # User and developer documentation
|
||||
├── migrations/ # Alembic database migrations
|
||||
└── docker/ # Docker configuration files
|
||||
```
|
||||
|
||||
## 📚 Additional Resources
|
||||
|
||||
### Documentation
|
||||
- **[AGENTIC_CODING.md](AGENTIC_CODING.md)** - Comprehensive guide for AI agents and developers
|
||||
- **[README.md](README.md)** - Project overview and quickstart
|
||||
- **[docs/CIWorkflow.md](docs/CIWorkflow.md)** - CI pipeline and linter details for maintainers
|
||||
- **[ROADMAP.md](ROADMAP.md)** - Future features and long-term vision
|
||||
- **[MILESTONES.md](MILESTONES.md)** - Release planning and versioning
|
||||
- **[TODO.md](TODO.md)** - Current tasks and priorities
|
||||
- **[SECURITY.md](SECURITY.md)** - Security policy
|
||||
- **[SECURITY_AUDIT.md](SECURITY_AUDIT.md)** - Security findings and improvements
|
||||
|
||||
### Testing
|
||||
- All new features must include tests
|
||||
- Aim for 80% code coverage
|
||||
- See [AGENTIC_CODING.md#testing-strategy](AGENTIC_CODING.md#testing-strategy) for detailed testing guidelines
|
||||
|
||||
### Security
|
||||
- Never commit secrets or credentials
|
||||
- Follow guidelines in [SECURITY_AUDIT.md](SECURITY_AUDIT.md)
|
||||
- Report security issues per [SECURITY.md](SECURITY.md)
|
||||
|
||||
## 🤝 Getting Help
|
||||
|
||||
- **GitHub Issues:** Bug reports and feature requests
|
||||
- **GitHub Discussions:** Questions and community support
|
||||
- **Documentation:** Check `docs/` directory for guides
|
||||
|
||||
Thank you for contributing to DocuElevate!
|
||||
@@ -1,131 +0,0 @@
|
||||
# Test Coverage Report
|
||||
|
||||
## Summary
|
||||
|
||||
This PR increases test coverage for two files to meet the 90%+ target:
|
||||
|
||||
- **`app/api/url_upload.py`**: Increased from **80.22%** to **91.21%** ✅
|
||||
- **`app/views/files.py`**: Increased from **18.45%** to **90.61%** ✅
|
||||
|
||||
## Coverage Details
|
||||
|
||||
### app/api/url_upload.py (91.21% coverage)
|
||||
|
||||
**Previous Coverage**: 80.22% (138 statements, 20 missing, 44 branches, 12 partial)
|
||||
**New Coverage**: 91.21% (138 statements, 6 missing, 44 branches, 10 partial)
|
||||
|
||||
#### New Tests Added (10 tests):
|
||||
1. `test_process_url_request_exception` - Tests handling of generic RequestException
|
||||
2. `test_process_url_oserror_during_save` - Tests OSError when saving file to disk
|
||||
3. `test_process_url_unexpected_exception` - Tests handling of unexpected exceptions
|
||||
4. `test_process_url_filename_without_extension` - Tests files without extensions
|
||||
5. `test_process_url_empty_path_uses_download` - Tests default filename for URLs without path
|
||||
6. `test_validate_url_no_hostname` - Tests URL validation without hostname
|
||||
7. `test_validate_file_type_by_extension_fallback` - Tests file type validation by extension
|
||||
8. `test_is_private_ip_ipv6_loopback` - Tests IPv6 loopback detection
|
||||
9. `test_is_private_ip_link_local` - Tests link-local address detection
|
||||
10. `test_process_url_sanitizes_dangerous_filename` - Tests filename sanitization security
|
||||
|
||||
#### Coverage Improvements:
|
||||
- **Error handling**: Now covers all exception handlers (RequestException, OSError, unexpected exceptions)
|
||||
- **Edge cases**: Covers missing hostnames, empty paths, files without extensions
|
||||
- **Security**: IPv6 loopback, link-local addresses, dangerous filename sanitization
|
||||
- **File validation**: Extension-based fallback validation
|
||||
|
||||
### app/views/files.py (90.61% coverage)
|
||||
|
||||
**Previous Coverage**: 18.45% (225 statements, 173 missing, 84 branches, 3 partial)
|
||||
**New Coverage**: 90.61% (225 statements, 14 missing, 84 branches, 13 partial)
|
||||
|
||||
#### New Tests Added (27 tests in new file `test_files_view_extended.py`):
|
||||
|
||||
**Files Page Tests (5 tests):**
|
||||
1. `test_files_page_with_search_filter` - Tests search filtering
|
||||
2. `test_files_page_with_mime_type_filter` - Tests MIME type filtering
|
||||
3. `test_files_page_with_sorting` - Tests sorting (asc/desc)
|
||||
4. `test_files_page_pagination` - Tests pagination with different page sizes
|
||||
5. `test_files_page_error_handling` - Tests error handling
|
||||
|
||||
**File Detail Page Tests (4 tests):**
|
||||
6. `test_file_detail_page_with_existing_file` - Tests detail page for existing file
|
||||
7. `test_file_detail_page_with_missing_file` - Tests 404 handling
|
||||
8. `test_file_detail_page_with_processing_logs` - Tests log display
|
||||
9. `test_file_detail_page_with_metadata` - Tests metadata JSON display
|
||||
|
||||
**File Preview Tests (6 tests):**
|
||||
10. `test_preview_original_file_success` - Tests successful preview of original file
|
||||
11. `test_preview_original_file_not_found` - Tests 404 for non-existent file
|
||||
12. `test_preview_original_file_missing_on_disk` - Tests missing file on disk
|
||||
13. `test_preview_processed_file_success` - Tests successful preview of processed file
|
||||
14. `test_preview_processed_file_not_found` - Tests 404 for non-existent file
|
||||
15. `test_preview_processed_file_missing_on_disk` - Tests missing file on disk
|
||||
|
||||
**Text Extraction Tests (8 tests):**
|
||||
16. `test_get_original_text_success` - Tests successful text extraction from original
|
||||
17. `test_get_original_text_file_not_found` - Tests 404 handling
|
||||
18. `test_get_original_text_file_missing_on_disk` - Tests missing file handling
|
||||
19. `test_get_original_text_extraction_error` - Tests invalid PDF handling
|
||||
20. `test_get_processed_text_success` - Tests successful text extraction from processed
|
||||
21. `test_get_processed_text_file_not_found` - Tests 404 handling
|
||||
22. `test_get_processed_text_file_missing_on_disk` - Tests missing file handling
|
||||
23. `test_get_processed_text_extraction_error` - Tests invalid PDF handling
|
||||
|
||||
**Unit Tests for Helper Functions (4 tests):**
|
||||
24. `test_compute_processing_flow_basic` - Tests processing flow computation
|
||||
25. `test_compute_processing_flow_with_uploads` - Tests flow with upload branches
|
||||
26. `test_compute_step_summary_basic` - Tests step summary computation
|
||||
27. `test_compute_step_summary_order_independent` - Tests order independence
|
||||
|
||||
#### Coverage Improvements:
|
||||
- **Main flow**: Files list page with pagination, sorting, filtering
|
||||
- **Detail pages**: File detail with logs, metadata, file existence checks
|
||||
- **File serving**: Preview original/processed files with error handling
|
||||
- **Text extraction**: On-demand text extraction with error handling
|
||||
- **Helper functions**: Processing flow and step summary computation
|
||||
- **Edge cases**: Missing files, invalid PDFs, error conditions
|
||||
|
||||
## Test Execution Results
|
||||
|
||||
All tests passing:
|
||||
- **url_upload tests**: 39 tests passed
|
||||
- **files view tests**: 30 tests passed
|
||||
- **Total**: 69 tests passed, 0 failures
|
||||
|
||||
## Test Quality
|
||||
|
||||
### Test Structure
|
||||
- Tests organized by feature using pytest classes
|
||||
- Proper use of pytest markers (`@pytest.mark.unit`, `@pytest.mark.integration`, `@pytest.mark.requires_db`)
|
||||
- Clear, descriptive test names following pattern: `test_<what>_<condition>_<expected>`
|
||||
- Comprehensive docstrings for each test
|
||||
|
||||
### Coverage Focus
|
||||
- **Main usage flows**: File upload, listing, detail viewing, preview, text extraction
|
||||
- **Edge conditions**: Missing files, invalid inputs, network errors, file system errors
|
||||
- **Error handling**: All exception paths covered
|
||||
- **Security**: SSRF protection, filename sanitization, input validation
|
||||
|
||||
### Mocking Strategy
|
||||
- External dependencies properly mocked (requests, Celery tasks)
|
||||
- Database operations use test fixtures with in-memory SQLite
|
||||
- File system operations use pytest's `tmp_path` fixture
|
||||
- No actual HTTP requests or file operations outside test environment
|
||||
|
||||
## Files Changed
|
||||
|
||||
1. **tests/test_url_upload.py** - Added 10 new tests
|
||||
2. **tests/test_files_view_extended.py** - Created new file with 27 tests
|
||||
3. Existing tests in **tests/test_files_view.py** - Maintained (3 tests)
|
||||
|
||||
## Validation
|
||||
|
||||
Coverage validated with:
|
||||
```bash
|
||||
pytest tests/test_url_upload.py --cov=app/api/url_upload --cov-report=term-missing
|
||||
# Result: 91.21% coverage
|
||||
|
||||
pytest tests/test_files_view.py tests/test_files_view_extended.py --cov=app/views/files --cov-report=term-missing
|
||||
# Result: 90.61% coverage
|
||||
```
|
||||
|
||||
All tests pass without failures or errors.
|
||||
+11
-39
@@ -1,58 +1,30 @@
|
||||
# Use multi-stage build for a smaller final image
|
||||
FROM python:3.14.1 AS builder
|
||||
|
||||
# Stage 1: Build dependencies
|
||||
FROM python:3.11 AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy requirements first for better layer caching
|
||||
COPY requirements.txt /app/
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Second stage for the actual runtime
|
||||
FROM python:3.14.3-slim
|
||||
# Stage 2: Final image
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy installed packages from builder stage
|
||||
COPY --from=builder /usr/local/lib/python3.14/site-packages /usr/local/lib/python3.14/site-packages
|
||||
# Copy installed dependencies
|
||||
COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
|
||||
COPY --from=builder /usr/local/bin /usr/local/bin
|
||||
|
||||
# Install system-level OCR tools required for local OCR workflows:
|
||||
# tesseract-ocr – OCR engine used by pytesseract and ocrmypdf
|
||||
# ghostscript – required by ocrmypdf for PDF/PS operations
|
||||
# poppler-utils – provides pdfinfo/pdftoppm used by pdf2image
|
||||
# unpaper – optional deskewing pre-processor used by ocrmypdf
|
||||
# wget – used by ocr_language_manager to download tessdata files
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
tesseract-ocr \
|
||||
ghostscript \
|
||||
poppler-utils \
|
||||
unpaper \
|
||||
wget \
|
||||
&& apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy application code
|
||||
# Copy application files correctly
|
||||
COPY ./app /app/app
|
||||
COPY ./frontend /app/frontend
|
||||
COPY ./LICENSE /app/LICENSE
|
||||
|
||||
# Copy build metadata files (generated at build time)
|
||||
COPY ./VERSION /app/VERSION
|
||||
COPY ./BUILD_DATE /app/BUILD_DATE
|
||||
COPY ./GIT_SHA /app/GIT_SHA
|
||||
COPY ./RUNTIME_INFO /app/RUNTIME_INFO
|
||||
|
||||
# Create runtime_info directory
|
||||
RUN mkdir -p /app/runtime_info
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p /workdir
|
||||
|
||||
# Set environment variables
|
||||
# Set Python path explicitly
|
||||
ENV PYTHONPATH=/app
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
# Expose the port the app runs on
|
||||
# Expose API port
|
||||
EXPOSE 8000
|
||||
WORKDIR /app
|
||||
|
||||
# Default command
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
# Local development Dockerfile (avoids CI-only build metadata files)
|
||||
FROM python:3.14.1 AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt /app/
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
FROM python:3.14.1-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /usr/local/lib/python3.14/site-packages /usr/local/lib/python3.14/site-packages
|
||||
COPY --from=builder /usr/local/bin /usr/local/bin
|
||||
|
||||
# Install system-level OCR tools required for local OCR workflows:
|
||||
# tesseract-ocr – OCR engine used by pytesseract and ocrmypdf
|
||||
# ghostscript – required by ocrmypdf for PDF/PS operations
|
||||
# poppler-utils – provides pdfinfo/pdftoppm used by pdf2image
|
||||
# unpaper – optional deskewing pre-processor used by ocrmypdf
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
tesseract-ocr \
|
||||
ghostscript \
|
||||
poppler-utils \
|
||||
unpaper \
|
||||
&& apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY ./app /app/app
|
||||
COPY ./frontend /app/frontend
|
||||
COPY ./LICENSE /app/LICENSE
|
||||
COPY ./VERSION /app/VERSION
|
||||
COPY ./BUILD_DATE /app/BUILD_DATE
|
||||
|
||||
# Local fallbacks for build metadata
|
||||
RUN echo "local" > /app/GIT_SHA \
|
||||
&& echo "local" > /app/RUNTIME_INFO
|
||||
|
||||
RUN mkdir -p /app/runtime_info
|
||||
RUN mkdir -p /workdir
|
||||
|
||||
ENV PYTHONPATH=/app
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
@@ -1,4 +1,4 @@
|
||||
Apache License
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
@@ -175,7 +175,18 @@ Apache License
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright 2025 Christian Krakau-Louis <christian@docuelevate.org>
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
-325
@@ -1,325 +0,0 @@
|
||||
# DocuElevate Milestones
|
||||
|
||||
**Last Updated:** 2026-02-08
|
||||
|
||||
This document outlines the release milestones, versioning strategy, and detailed feature breakdown for DocuElevate.
|
||||
|
||||
## Versioning Strategy
|
||||
|
||||
DocuElevate follows [Semantic Versioning 2.0.0](https://semver.org/):
|
||||
- **MAJOR.MINOR.PATCH** (e.g., 1.2.3)
|
||||
- **MAJOR:** Breaking changes or major architectural shifts
|
||||
- **MINOR:** New features, backward-compatible
|
||||
- **PATCH:** Bug fixes, security patches, backward-compatible
|
||||
|
||||
### Release Cadence
|
||||
- **Patch releases:** As needed for critical bugs/security
|
||||
- **Minor releases:** Every 6-8 weeks
|
||||
- **Major releases:** Every 12-18 months
|
||||
|
||||
---
|
||||
|
||||
## Current Release: v0.5.0 (February 2026)
|
||||
|
||||
### Status: Stable
|
||||
- Production-ready document processing
|
||||
- Multi-provider storage support
|
||||
- **Database-backed settings management with encryption**
|
||||
- **Setup wizard for first-time configuration**
|
||||
- **Admin UI for runtime configuration**
|
||||
- **Automated semantic versioning and releases**
|
||||
- OAuth2 authentication with admin group support
|
||||
- Basic web UI and REST API
|
||||
|
||||
### Important Note on Versioning
|
||||
As of February 2026, DocuElevate uses **automated semantic versioning**:
|
||||
- Version management handled by `python-semantic-release`
|
||||
- Releases automated via GitHub Actions on merge to main
|
||||
- Version bumps determined by conventional commit messages
|
||||
- `VERSION` and `CHANGELOG.md` automatically updated
|
||||
- GitHub Releases created automatically with release notes
|
||||
|
||||
---
|
||||
|
||||
## Previous Releases
|
||||
|
||||
### v0.3.3 (February 2026)
|
||||
- Drag-and-drop file upload on Files page
|
||||
- Enhanced upload UI and functionality
|
||||
|
||||
### v0.3.2 (February 2026)
|
||||
- Security hardening (Authlib/Starlette updates)
|
||||
- Testing infrastructure implementation
|
||||
- CI/CD improvements
|
||||
|
||||
---
|
||||
|
||||
## Completed Milestones
|
||||
|
||||
### v0.5.0 - Settings Management & Configuration (February 2026)
|
||||
**Release Date:** February 8, 2026
|
||||
**Status:** ✅ Released
|
||||
**Theme:** Configuration Management, Security, User Experience
|
||||
|
||||
#### Goals
|
||||
- [x] **Implement database-backed settings management**
|
||||
- [x] **Add encryption for sensitive configuration**
|
||||
- [x] **Create setup wizard for first-time installation**
|
||||
- [x] Complete settings UI with admin access
|
||||
- [x] Integrate with existing authentication system
|
||||
|
||||
#### Deliverables
|
||||
- [x] **Settings management UI at /settings**
|
||||
- [x] **Setup wizard at /setup**
|
||||
- [x] **Fernet encryption for sensitive settings**
|
||||
- [x] **Source indicators (DB/ENV/DEFAULT)**
|
||||
- [x] **Complete settings documentation**
|
||||
- [x] **Framework analysis (FRAMEWORK_ANALYSIS.md)**
|
||||
- [x] REST API for settings management
|
||||
- [x] Admin authentication and authorization
|
||||
- [x] Comprehensive test coverage
|
||||
|
||||
#### New Features
|
||||
- **Settings Management System**: Web-based admin UI for viewing and editing 102 application settings across 10 categories
|
||||
- **Encryption**: Fernet symmetric encryption for sensitive values (passwords, API keys, tokens) with key derived from SESSION_SECRET
|
||||
- **Setup Wizard**: 3-step wizard for first-time configuration (Infrastructure → Security → AI Services)
|
||||
- **Precedence System**: Settings resolved in order: Database > Environment Variables > Defaults
|
||||
- **Source Indicators**: Visual badges showing where each setting value originates (🟢 DB, 🔵 ENV, ⚪ DEFAULT)
|
||||
- **Admin Access Control**: OAuth admin group support and proper decorator pattern for authorization
|
||||
|
||||
---
|
||||
|
||||
### v0.3.3 - Drag-and-Drop Upload (February 2026)
|
||||
**Release Date:** February 8, 2026
|
||||
**Status:** ✅ Released
|
||||
**Theme:** User Experience Enhancement
|
||||
|
||||
#### Goals
|
||||
- [x] Add drag-and-drop file upload to Files view
|
||||
- [x] Refactor upload logic for maintainability
|
||||
- [x] Improve visual feedback during file interactions
|
||||
|
||||
#### Deliverables
|
||||
- [x] Drag-and-drop upload functionality in Files view
|
||||
- [x] Reusable `upload.js` module for code DRYness
|
||||
- [x] Visual drop overlay and progress modal
|
||||
- [x] Enhanced upload error handling
|
||||
|
||||
---
|
||||
|
||||
### v0.3.2 - Security & Testing Hardening (February 2026)
|
||||
**Release Date:** February 6, 2026
|
||||
**Status:** ✅ Released
|
||||
**Theme:** Security, Quality, Testing
|
||||
|
||||
#### Goals
|
||||
- [x] Fix critical security vulnerabilities (authlib, starlette)
|
||||
- [x] Implement comprehensive test suite
|
||||
- [x] Add security scanning (CodeQL, Bandit)
|
||||
- [x] Improve CI/CD pipeline
|
||||
|
||||
#### Deliverables
|
||||
- [x] SECURITY_AUDIT.md documentation
|
||||
- [x] pytest configuration and fixtures
|
||||
- [x] API integration tests
|
||||
- [x] Configuration validation tests
|
||||
- [x] Updated CI/CD workflows
|
||||
- [x] Pre-commit hooks configuration
|
||||
|
||||
---
|
||||
|
||||
## Upcoming Milestones
|
||||
|
||||
### v0.6.0 - Enhanced Search & UI Improvements (April 2026)
|
||||
**Target Date:** April 1, 2026
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** User Experience, Search, Performance
|
||||
|
||||
#### Goals
|
||||
- Implement full-text search across documents
|
||||
- Responsive mobile interface
|
||||
- Dark mode support
|
||||
- Document preview in browser
|
||||
- Performance optimizations
|
||||
- Improved error handling and user feedback
|
||||
|
||||
#### Deliverables
|
||||
- Full-text search API and UI
|
||||
- Advanced filtering capabilities
|
||||
- Responsive CSS framework integration
|
||||
- Dark mode toggle
|
||||
- In-browser document viewer
|
||||
- Loading states and progress indicators
|
||||
- Performance benchmarks
|
||||
- Mobile-optimized interface
|
||||
|
||||
#### Breaking Changes
|
||||
- API response format changes for search endpoints (documented)
|
||||
|
||||
#### Migration Path
|
||||
- Search endpoint changes will be versioned (/api/v1/search → /api/v2/search)
|
||||
- Old endpoints deprecated but functional for 2 releases
|
||||
|
||||
---
|
||||
|
||||
### v0.4.5 - Workflow Automation (June 2026)
|
||||
**Target Date:** June 1, 2026
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** Automation, Integration, Webhooks
|
||||
|
||||
#### Goals
|
||||
- Custom processing pipelines
|
||||
- Conditional routing based on document type
|
||||
- Webhook support for external integrations
|
||||
- Rule-based classification
|
||||
- Scheduled batch processing
|
||||
|
||||
#### Deliverables
|
||||
- Pipeline configuration UI
|
||||
- Webhook management interface
|
||||
- Rule engine for document routing
|
||||
- Batch processing scheduler
|
||||
- Integration examples and templates
|
||||
- Webhook payload documentation
|
||||
|
||||
---
|
||||
|
||||
### v0.7.0 - Advanced AI & Multi-language (August 2026)
|
||||
**Target Date:** August 1, 2026
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** AI Enhancement, Internationalization
|
||||
|
||||
#### Goals
|
||||
- Custom AI model support
|
||||
- Multi-language OCR
|
||||
- Document similarity detection
|
||||
- Duplicate detection
|
||||
- UI internationalization (i18n)
|
||||
- API localization
|
||||
|
||||
#### Deliverables
|
||||
- Custom model integration API
|
||||
- Multi-language OCR configuration
|
||||
- Similarity algorithm implementation
|
||||
- Duplicate detection service
|
||||
- Translation framework (10+ languages)
|
||||
- Localized documentation
|
||||
|
||||
---
|
||||
|
||||
### v1.0.0 - Enterprise Edition (November 2026)
|
||||
**Target Date:** November 1, 2026
|
||||
**Status:** 📋 Planned
|
||||
**Theme:** Enterprise Features, Scalability, Multi-tenancy
|
||||
|
||||
This is our first major release, marking production-ready enterprise capabilities.
|
||||
|
||||
#### Goals
|
||||
- Multi-tenancy and organization management
|
||||
- Role-based access control (RBAC)
|
||||
- Horizontal scaling support
|
||||
- Comprehensive audit logging
|
||||
- SLA monitoring and alerting
|
||||
- Professional support offerings
|
||||
|
||||
#### Deliverables
|
||||
- **Multi-tenancy**
|
||||
- Organization/team management UI
|
||||
- Per-tenant configuration and branding
|
||||
- Resource quotas and billing integration
|
||||
- Tenant isolation at database level
|
||||
|
||||
- **Access Control**
|
||||
- RBAC with customizable roles
|
||||
- Permission management UI
|
||||
- API key management per organization
|
||||
- SSO integration (SAML, LDAP)
|
||||
|
||||
- **Scalability**
|
||||
- Horizontal scaling documentation
|
||||
- Load balancer configuration
|
||||
- Distributed caching
|
||||
- Database replication support
|
||||
- Message queue clustering
|
||||
|
||||
---
|
||||
|
||||
## Release Process
|
||||
|
||||
### Automated Semantic Versioning (v0.6.0+)
|
||||
Starting with v0.6.0, releases are fully automated using `python-semantic-release`:
|
||||
|
||||
1. **Commit with Conventional Format**: Use conventional commit messages (feat, fix, etc.)
|
||||
2. **Merge to Main**: PR merges trigger semantic-release workflow
|
||||
3. **Automated Analysis**: semantic-release determines version from commits
|
||||
4. **Automatic Updates**:
|
||||
- Updates `VERSION` file
|
||||
- Generates/updates `CHANGELOG.md`
|
||||
- Creates Git tag (e.g., `v0.6.0`)
|
||||
- Creates GitHub Release with notes
|
||||
- Triggers Docker image builds
|
||||
5. **No Manual Steps**: VERSION and CHANGELOG are never edited manually
|
||||
|
||||
### Version Bump Rules
|
||||
- `feat:` commits → Minor version (0.5.0 → 0.6.0)
|
||||
- `fix:`, `perf:` → Patch version (0.5.0 → 0.5.1)
|
||||
- `feat!:`, `BREAKING CHANGE:` → Major version (0.5.0 → 1.0.0)
|
||||
- Other types (docs, chore, etc.) → No version bump
|
||||
|
||||
### Pre-release Checklist (Automated)
|
||||
- [ ] All tests passing
|
||||
- [ ] Security scan passed
|
||||
- [ ] Code review completed
|
||||
- [ ] Documentation updated
|
||||
- [ ] CHANGELOG.md updated
|
||||
- [ ] Migration guide (if breaking changes)
|
||||
- [ ] Release notes drafted
|
||||
- [ ] Version numbers bumped
|
||||
- [ ] Docker images built and tested
|
||||
|
||||
### Release Artifacts
|
||||
- Source code (GitHub)
|
||||
- Docker images (Docker Hub)
|
||||
- PyPI package (future)
|
||||
- Helm charts (future)
|
||||
- Documentation site update
|
||||
|
||||
---
|
||||
|
||||
## Version History
|
||||
|
||||
| Version | Release Date | Theme | Status |
|
||||
|---------|-------------|-------|--------|
|
||||
| v0.1.0 | 2024-Q1 | Initial Release | Released |
|
||||
| v0.2.0 | 2024-Q3 | Multi-provider Support | Released |
|
||||
| v0.3.0 | 2025-Q4 | UI & Authentication | Released |
|
||||
| v0.3.1 | 2026-01-15 | OAuth2 Integration | Released |
|
||||
| v0.3.2 | 2026-02-06 | Security Updates | Released |
|
||||
| v0.3.3 | 2026-02-08 | Drag-and-Drop Upload | Released |
|
||||
| v0.5.0 | 2026-02-08 | **Settings & Encryption** | **Released** |
|
||||
| v0.6.0 | 2026-04 | Search & UX | Planned |
|
||||
| v0.7.0 | 2026-08 | Advanced AI | Planned |
|
||||
| v1.0.0 | 2026-11 | Enterprise | Planned |
|
||||
| v2.0.0 | 2027-Q3 | Platform Expansion | Future |
|
||||
|
||||
---
|
||||
|
||||
## Support & EOL Policy
|
||||
|
||||
### Active Support
|
||||
- Current stable release: Full support (bug fixes, security patches, features)
|
||||
- Previous minor release: Security patches only
|
||||
- Older versions: Community support only
|
||||
|
||||
### End of Life (EOL)
|
||||
- Minor versions: EOL when 2 newer minor versions released
|
||||
- Major versions: EOL 18 months after next major version
|
||||
|
||||
### Security Patches
|
||||
- Critical vulnerabilities: Patched within 48 hours
|
||||
- High severity: Patched within 1 week
|
||||
- Medium/Low: Included in next regular release
|
||||
|
||||
---
|
||||
|
||||
*This milestone document is updated regularly. For real-time status, check our [GitHub Projects](https://github.com/christianlouis/DocuElevate/projects) board.*
|
||||
@@ -1,138 +0,0 @@
|
||||
DocuElevate
|
||||
Copyright 2025 Christian Krakau-Louis
|
||||
|
||||
This product includes software developed for the DocuElevate project.
|
||||
|
||||
================================================================================
|
||||
|
||||
This software includes third-party components with their own licenses:
|
||||
|
||||
SPECIAL NOTICE REGARDING LGPL SOFTWARE:
|
||||
--------------------------------------------------------------------------------
|
||||
DocuElevate incorporates Paramiko, which is licensed under the GNU Lesser General
|
||||
Public License (LGPL) version 2.1. In accordance with the LGPL:
|
||||
|
||||
1. The complete source code for Paramiko can be obtained from:
|
||||
https://github.com/paramiko/paramiko
|
||||
|
||||
2. This software is distributed in the hope that it will be useful, but WITHOUT
|
||||
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
|
||||
for more details.
|
||||
|
||||
3. A copy of the GNU Lesser General Public License version 2.1 can be found at:
|
||||
frontend/static/licenses/lgpl.txt and at https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
|
||||
|
||||
4. Users have the right to obtain the source code of Paramiko and to modify and
|
||||
redistribute it under the terms of the LGPL.
|
||||
|
||||
# Python Dependencies
|
||||
--------------------------------------------------------------------------------
|
||||
FastAPI (MIT License)
|
||||
Copyright (c) 2018 Sebastián Ramírez
|
||||
https://github.com/tiangolo/fastapi
|
||||
|
||||
Celery (BSD License)
|
||||
Copyright (c) 2015-2016 Ask Solem & contributors
|
||||
https://github.com/celery/celery
|
||||
|
||||
Uvicorn (BSD License)
|
||||
Copyright (c) 2017-present, Encode OSS Ltd.
|
||||
https://github.com/encode/uvicorn
|
||||
|
||||
SQLAlchemy (MIT License)
|
||||
Copyright (c) 2005-2023 SQLAlchemy authors and contributors
|
||||
https://github.com/sqlalchemy/sqlalchemy
|
||||
|
||||
Pydantic (MIT License)
|
||||
Copyright (c) 2017-present Pydantic Services Inc.
|
||||
https://github.com/pydantic/pydantic
|
||||
|
||||
OpenAI (MIT License)
|
||||
Copyright (c) 2023 OpenAI
|
||||
https://github.com/openai/openai-python
|
||||
|
||||
pypdf (BSD License)
|
||||
Copyright (c) 2006-2024, pypdf contributors
|
||||
https://github.com/py-pdf/pypdf
|
||||
|
||||
Requests (Apache 2.0 License)
|
||||
Copyright 2019 Kenneth Reitz
|
||||
https://github.com/psf/requests
|
||||
|
||||
Dropbox (MIT License)
|
||||
Copyright (c) 2015-2021 Dropbox, Inc.
|
||||
https://github.com/dropbox/dropbox-sdk-python
|
||||
|
||||
Azure AI Document Intelligence (MIT License)
|
||||
Copyright (c) Microsoft Corporation
|
||||
https://github.com/Azure/azure-sdk-for-python
|
||||
|
||||
Authlib (BSD License)
|
||||
Copyright (c) 2017-present, Hsiaoming Yang
|
||||
https://github.com/lepture/authlib
|
||||
|
||||
python-dotenv (BSD License)
|
||||
Copyright (c) 2014, Saurabh Kumar
|
||||
https://github.com/theskumar/python-dotenv
|
||||
|
||||
Starlette (BSD License)
|
||||
Copyright (c) 2018-present, Encode OSS Ltd.
|
||||
https://github.com/encode/starlette
|
||||
|
||||
Alembic (MIT License)
|
||||
Copyright (c) 2009-2023 Michael Bayer
|
||||
https://github.com/sqlalchemy/alembic
|
||||
|
||||
Google API Client (Apache 2.0 License)
|
||||
Copyright 2014 Google LLC
|
||||
https://github.com/googleapis/google-api-python-client
|
||||
|
||||
Microsoft Graph Core (MIT License)
|
||||
Copyright (c) Microsoft Corporation
|
||||
https://github.com/microsoftgraph/msgraph-sdk-python-core
|
||||
|
||||
MSAL (MIT License)
|
||||
Copyright (c) Microsoft Corporation
|
||||
https://github.com/AzureAD/microsoft-authentication-library-for-python
|
||||
|
||||
Boto3 (Apache 2.0 License)
|
||||
Copyright Amazon.com, Inc. or its affiliates
|
||||
https://github.com/boto/boto3
|
||||
|
||||
Paramiko (LGPL-2.1 License)
|
||||
Copyright (c) 2003-2009 Robey Pointer
|
||||
https://github.com/paramiko/paramiko
|
||||
|
||||
Apprise (MIT License)
|
||||
Copyright (C) 2019-2024 Chris Caron
|
||||
https://github.com/caronc/apprise
|
||||
|
||||
# Docker Images
|
||||
--------------------------------------------------------------------------------
|
||||
Redis (BSD License)
|
||||
Copyright (c) 2006-2020, Salvatore Sanfilippo
|
||||
https://redis.io/
|
||||
|
||||
Gotenberg (MIT License)
|
||||
Copyright (c) 2019 Julien Neuhart
|
||||
https://github.com/gotenberg/gotenberg
|
||||
|
||||
# Frontend Libraries
|
||||
--------------------------------------------------------------------------------
|
||||
Tailwind CSS (MIT License)
|
||||
Copyright (c) Tailwind Labs, Inc.
|
||||
https://github.com/tailwindlabs/tailwindcss
|
||||
|
||||
Alpine.js (MIT License)
|
||||
Copyright (c) 2019-2021 Caleb Porzio and contributors
|
||||
https://github.com/alpinejs/alpine
|
||||
|
||||
Font Awesome (Font Awesome Free License)
|
||||
https://github.com/FortAwesome/Font-Awesome
|
||||
|
||||
# For a complete list of all dependencies and their licenses
|
||||
--------------------------------------------------------------------------------
|
||||
See the attribution page in the application or run:
|
||||
pip install pip-licenses
|
||||
pip-licenses
|
||||
@@ -1,238 +0,0 @@
|
||||
# Mock OAuth2 Server Implementation - Summary
|
||||
|
||||
## Overview
|
||||
Successfully implemented a production-ready mock OAuth2/OIDC server infrastructure for testing authentication flows in DocuElevate.
|
||||
|
||||
## What Was Implemented
|
||||
|
||||
### 1. Mock OAuth2 Server Container (`tests/mock_oauth_server.py`)
|
||||
- Wraps `mock-oauth2-server` Docker image using testcontainers
|
||||
- Provides complete OIDC provider with all standard endpoints
|
||||
- Fast startup (<1 second), no persistence needed
|
||||
- Automatic readiness detection with health checks
|
||||
|
||||
### 2. OAuth Test Fixtures (`tests/conftest_oauth.py`)
|
||||
- Session-scoped mock OAuth server fixture
|
||||
- Auto-detection of real OAuth credentials from environment
|
||||
- Seamless switching between mock and real OAuth modes
|
||||
- Test data generators (tokens, userinfo, etc.)
|
||||
- Test client with OAuth pre-configured
|
||||
|
||||
### 3. Integration Tests (`tests/test_oauth_integration_flows.py`)
|
||||
- 20+ comprehensive integration tests covering:
|
||||
- OAuth login initiation and redirects
|
||||
- Authorization code exchange
|
||||
- Token validation and session management
|
||||
- Admin vs non-admin authorization
|
||||
- Error handling scenarios
|
||||
- Real OAuth provider integration (when credentials available)
|
||||
|
||||
### 4. Documentation
|
||||
- `tests/README_OAUTH_TESTING.md` - Developer guide
|
||||
- `docs/OAuth_Testing_CI_CD.md` - CI/CD integration guide
|
||||
- Complete examples and troubleshooting
|
||||
|
||||
## Key Features
|
||||
|
||||
### Dual Mode Operation
|
||||
|
||||
**Mock Mode (Default)**
|
||||
```bash
|
||||
# Uses mock-oauth2-server in testcontainer
|
||||
pytest tests/test_oauth_integration_flows.py -v
|
||||
```
|
||||
- ⚡ <1s startup
|
||||
- 🔒 No external dependencies
|
||||
- 🎲 Deterministic results
|
||||
- Perfect for local development
|
||||
|
||||
**Real Mode (CI with Secrets)**
|
||||
```bash
|
||||
# Auto-detects and uses real OAuth credentials
|
||||
export AUTHENTIK_CLIENT_ID="your-client-id"
|
||||
export AUTHENTIK_CLIENT_SECRET="your-client-secret"
|
||||
export AUTHENTIK_CONFIG_URL="https://auth.example.com/.well-known/openid-configuration"
|
||||
pytest tests/test_oauth_integration_flows.py -v -m requires_external
|
||||
```
|
||||
- ✅ Tests real OAuth provider
|
||||
- ✅ Validates actual authentication flows
|
||||
- ✅ Uses GitHub Actions secrets
|
||||
- Perfect for integration testing
|
||||
|
||||
### Automatic Mode Detection
|
||||
- Checks for real OAuth credentials in environment
|
||||
- Falls back to mock if credentials not available
|
||||
- Can be manually overridden with env vars
|
||||
- Gracefully skips if dependencies missing
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Test Suite
|
||||
↓
|
||||
OAuth Fixtures (conftest_oauth.py)
|
||||
├── Mock Mode → MockOAuth2ServerContainer
|
||||
│ ├── .well-known/openid-configuration
|
||||
│ ├── /authorize
|
||||
│ ├── /token
|
||||
│ ├── /userinfo
|
||||
│ └── /jwks
|
||||
│
|
||||
└── Real Mode → Actual OAuth Provider (Authentik)
|
||||
└── Uses GitHub Actions secrets
|
||||
```
|
||||
|
||||
## Verification Results
|
||||
|
||||
✅ **Mock OAuth2 Server**
|
||||
- Starts successfully in <1 second
|
||||
- Returns valid OIDC configuration
|
||||
- Provides all required OIDC endpoints
|
||||
- Can be started/stopped cleanly
|
||||
- Works with Docker in CI
|
||||
|
||||
✅ **Endpoints Verified**
|
||||
- `/.well-known/openid-configuration` - OIDC discovery
|
||||
- `/authorize` - OAuth authorization
|
||||
- `/token` - Token exchange
|
||||
- `/userinfo` - User information
|
||||
- `/jwks` - JWT signing keys
|
||||
|
||||
✅ **Test Infrastructure**
|
||||
- Fixtures load correctly
|
||||
- Auto-detection works
|
||||
- Mock/real mode switching functional
|
||||
- Integration with conftest.py successful
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Basic Test
|
||||
```python
|
||||
@pytest.mark.integration
|
||||
def test_oauth_login(oauth_enabled_app):
|
||||
"""Test OAuth login redirects to provider."""
|
||||
response = oauth_enabled_app.get("/oauth-login", follow_redirects=False)
|
||||
assert response.status_code == 302
|
||||
assert "authorize" in response.headers["location"]
|
||||
```
|
||||
|
||||
### Test with Mock Token Exchange
|
||||
```python
|
||||
from unittest.mock import patch
|
||||
|
||||
@pytest.mark.integration
|
||||
@patch("app.auth.oauth.authentik.authorize_access_token")
|
||||
async def test_oauth_callback(mock_authorize, oauth_enabled_app, test_user_info):
|
||||
"""Test OAuth callback with test user."""
|
||||
mock_authorize.return_value = {
|
||||
"access_token": "test-token",
|
||||
"userinfo": test_user_info,
|
||||
}
|
||||
|
||||
response = oauth_enabled_app.get("/oauth-callback?code=test-code")
|
||||
assert response.status_code == 302
|
||||
```
|
||||
|
||||
## GitHub Actions Integration
|
||||
|
||||
### Basic Workflow
|
||||
```yaml
|
||||
name: OAuth Tests
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: pip install -r requirements-dev.txt
|
||||
- run: pytest tests/test_oauth_integration_flows.py -v
|
||||
```
|
||||
|
||||
### With Real OAuth (Internal PRs)
|
||||
```yaml
|
||||
jobs:
|
||||
test-real:
|
||||
if: github.event_name == 'push'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
- run: pip install -r requirements-dev.txt
|
||||
- env:
|
||||
AUTHENTIK_CLIENT_ID: ${{ secrets.AUTHENTIK_CLIENT_ID }}
|
||||
AUTHENTIK_CLIENT_SECRET: ${{ secrets.AUTHENTIK_CLIENT_SECRET }}
|
||||
AUTHENTIK_CONFIG_URL: ${{ secrets.AUTHENTIK_CONFIG_URL }}
|
||||
run: pytest tests/test_oauth_integration_flows.py -v -m requires_external
|
||||
```
|
||||
|
||||
## Benefits
|
||||
|
||||
| Aspect | Benefit |
|
||||
|--------|---------|
|
||||
| **Speed** | <1s startup, tests complete in seconds |
|
||||
| **Reliability** | Deterministic, no flaky tests |
|
||||
| **Realism** | Tests actual OIDC protocol |
|
||||
| **Flexibility** | Works with mock or real OAuth |
|
||||
| **CI-Friendly** | Ephemeral containers, works in pipelines |
|
||||
| **Security** | Uses GitHub secrets for real credentials |
|
||||
| **Maintainability** | Industry-standard mock-oauth2-server |
|
||||
| **Documentation** | Comprehensive guides and examples |
|
||||
|
||||
## Technical Details
|
||||
|
||||
**Container**: `ghcr.io/navikt/mock-oauth2-server:2.1.1`
|
||||
**Framework**: Testcontainers Python 4.14.1+
|
||||
**Test Framework**: pytest with async support
|
||||
**Languages**: Python 3.12+
|
||||
**Dependencies**: testcontainers, requests, docker
|
||||
|
||||
## Files Created/Modified
|
||||
|
||||
### New Files
|
||||
- `tests/mock_oauth_server.py` - Mock OAuth server container wrapper
|
||||
- `tests/conftest_oauth.py` - OAuth test fixtures
|
||||
- `tests/test_oauth_integration_flows.py` - Integration tests
|
||||
- `tests/README_OAUTH_TESTING.md` - Developer documentation
|
||||
- `docs/OAuth_Testing_CI_CD.md` - CI/CD guide
|
||||
|
||||
### Modified Files
|
||||
- `tests/conftest.py` - Added OAuth fixtures import
|
||||
|
||||
## Next Steps
|
||||
|
||||
To fully utilize this infrastructure:
|
||||
|
||||
1. **Run tests locally**:
|
||||
```bash
|
||||
pytest tests/test_oauth_integration_flows.py -v
|
||||
```
|
||||
|
||||
2. **Add to CI pipeline**:
|
||||
- Use provided GitHub Actions examples
|
||||
- Configure secrets for real OAuth testing
|
||||
|
||||
3. **Expand test coverage**:
|
||||
- Add more OAuth flow scenarios
|
||||
- Test edge cases
|
||||
- Add performance tests
|
||||
|
||||
4. **Monitor and maintain**:
|
||||
- Keep mock-oauth2-server image updated
|
||||
- Update tests as OAuth implementation evolves
|
||||
- Add new scenarios as needed
|
||||
|
||||
## Conclusion
|
||||
|
||||
The mock OAuth2 server infrastructure is production-ready and provides:
|
||||
- ✅ Fast, reliable OAuth testing
|
||||
- ✅ Support for both mock and real OAuth providers
|
||||
- ✅ Comprehensive test coverage
|
||||
- ✅ Full CI/CD integration
|
||||
- ✅ Excellent documentation
|
||||
|
||||
This implementation addresses all requirements from the original issue and provides a robust foundation for OAuth testing in DocuElevate.
|
||||
@@ -1,243 +1,203 @@
|
||||
<div align="center">
|
||||
<img src="frontend/static/logo_writing.svg" alt="DocuElevate Logo" width="280" />
|
||||
<p>Intelligent Document Processing & Management</p>
|
||||
</div>
|
||||
|
||||
# DocuElevate
|
||||
|
||||
<div align="center">
|
||||
|
||||
[](https://codecov.io/github/christianlouis/DocuElevate)
|
||||
[](https://github.com/christianlouis/DocuElevate/actions/workflows/ci.yml)
|
||||
[](https://github.com/christianlouis/DocuElevate/actions/workflows/codeql.yml)
|
||||
|
||||
[](https://github.com/christianlouis/DocuElevate/releases)
|
||||
[](LICENSE)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://hub.docker.com/)
|
||||
|
||||
[](https://github.com/christianlouis/DocuElevate/stargazers)
|
||||
[](https://github.com/christianlouis/DocuElevate/network/members)
|
||||
[](https://github.com/christianlouis/DocuElevate/issues)
|
||||
[](https://github.com/christianlouis/DocuElevate/pulls)
|
||||
|
||||
</div>
|
||||
|
||||
<div align="center">
|
||||
<a href="https://www.docuelevate.org"><img src="frontend/static/hero.png" alt="DocuElevate Logo" width="80%" /></a>
|
||||
</div>
|
||||
# Document Processing System
|
||||
|
||||
## Overview
|
||||
|
||||
DocuElevate automates the handling, extraction, and processing of documents using a variety of services, including:
|
||||
This project automates the handling, extraction, and processing of documents using a variety of services, including:
|
||||
|
||||
- **AI Provider** (pluggable – OpenAI, Anthropic, Gemini, Ollama, OpenRouter, Portkey, and more) for metadata extraction and text refinement.
|
||||
- **Dropbox**, **Nextcloud**, and **Google Drive** for file storage and uploads.
|
||||
- **Paperless NGX** for document indexing and management.
|
||||
- **Azure Document Intelligence** for OCR on PDFs.
|
||||
- **Gotenberg** for file-to-PDF conversions.
|
||||
- **Authentik** for authentication and user management.
|
||||
- **OpenAI** for metadata extraction and text refinement.
|
||||
- **Dropbox** and **Nextcloud** for file storage and uploads.
|
||||
- **Paperless NGX** for document indexing and management.
|
||||
- **Azure Document Intelligence** for OCR on PDFs.
|
||||
- **Gotenberg** for file-to-PDF conversions.
|
||||
- **Authentik** for authentication and user management.
|
||||
|
||||
It is designed for flexibility and configurability through environment variables, making it easily customizable for different workflows. The system can fetch documents from multiple IMAP mailboxes, process them (OCR, metadata extraction, PDF conversion), and store them in the desired destinations.
|
||||
|
||||
The project includes a **UI** for uploading and managing files, and an API documentation page is available at `/docs` (powered by **FastAPI**).
|
||||
|
||||
## Documentation Index
|
||||
|
||||
- [User Guide](docs/UserGuide.md) - How to use DocuElevate
|
||||
- [Browser Extension Guide](docs/BrowserExtension.md) - Install and use the browser extension
|
||||
- [API Documentation](docs/API.md) - API reference
|
||||
- [Deployment Guide](docs/DeploymentGuide.md) - How to deploy DocuElevate
|
||||
- [Configuration Guide](docs/ConfigurationGuide.md) - Available configuration options
|
||||
- [Build Metadata](docs/BuildMetadata.md) - Automated version and build information
|
||||
- [CI/CD Tools Guide](docs/CIToolsGuide.md) - CI/CD pipeline and tool documentation
|
||||
- [CI Workflow Guide](docs/CIWorkflow.md) - Detailed workflow documentation
|
||||
- [Development Guide](CONTRIBUTING.md) - How to contribute to DocuElevate
|
||||
- [Troubleshooting](docs/Troubleshooting.md) - Common issues and solutions
|
||||
|
||||
## Screenshots
|
||||
|
||||
<div align="center">
|
||||
<img src="docs/upload-view.png" alt="DocuElevate Upload Interface" width="80%" />
|
||||
<p><em>Upload interface for adding new documents</em></p>
|
||||
|
||||
<img src="docs/files-view.png" alt="DocuElevate Files View" width="80%" />
|
||||
<p><em>Files view with processed documents and metadata</em></p>
|
||||
</div>
|
||||
|
||||
> **Note:** Screenshots may not reflect the very latest UI. For the most current look, visit [docuelevate.org](https://www.docuelevate.org).
|
||||
|
||||
## Workflow Process
|
||||
|
||||
DocuElevate follows a streamlined document processing workflow:
|
||||
|
||||
<div align="center">
|
||||
<img src="docs/workflow-diagram.png" alt="DocuElevate Workflow" width="90%" />
|
||||
</div>
|
||||
|
||||
### Document Ingestion
|
||||
Documents enter DocuElevate through four possible channels:
|
||||
1. **Web Upload**: Users manually upload files via the web interface
|
||||
2. **Browser Extension**: Send files directly from your browser with one click
|
||||
3. **Email Attachments**: Automatic polling of configured IMAP mailboxes (supports multiple accounts)
|
||||
4. **API**: Direct programmatic uploads via the REST API
|
||||
|
||||
### Processing Pipeline
|
||||
Every document goes through the following steps:
|
||||
1. **PDF Conversion**: Non-PDF files are converted to PDF format using Gotenberg
|
||||
2. **OCR Processing**: Azure Document Intelligence extracts text from images/scans
|
||||
3. **Metadata Extraction**: The configured AI provider analyzes document content to identify:
|
||||
- Document type (invoice, receipt, contract, etc.)
|
||||
- Key entities (dates, names, amounts, account numbers)
|
||||
- Important data points specific to the document type
|
||||
4. **Enrichment**: Metadata is attached to the document in a structured format
|
||||
|
||||
### Distribution
|
||||
Processed documents with their metadata can be automatically sent to:
|
||||
- **Dropbox**: For cloud storage and sharing
|
||||
- **Nextcloud**: For self-hosted file storage
|
||||
- **Google Drive**: For Google Workspace integration
|
||||
- **Paperless-NGX**: For advanced document management with search capabilities
|
||||
|
||||
Users can choose to send documents to any combination of these destinations through configuration settings or manual selection.
|
||||
|
||||
## Features
|
||||
|
||||
- **Intuitive File Upload**:
|
||||
- Drag-and-drop file upload on both Upload and Files pages—upload anywhere on the Files page
|
||||
- Real-time upload progress with validation
|
||||
- Support for PDF, Office documents, images, and more (up to 500MB per file)
|
||||
- **Browser Extension**:
|
||||
- Send files directly from your browser to DocuElevate with one click
|
||||
- Compatible with Chrome, Firefox, Edge, and other Chromium-based browsers
|
||||
- Context menu integration for quick access
|
||||
- See [Browser Extension Guide](docs/BrowserExtension.md) for installation and usage
|
||||
- **Document Upload & Storage**:
|
||||
- Manual uploads (via API or UI) to Dropbox, Nextcloud, Google Drive, or Paperless
|
||||
- **OCR Processing (Azure)**:
|
||||
- Extract text from scanned PDFs using Azure Document Intelligence
|
||||
- **Metadata Extraction (AI Provider)**:
|
||||
- Use any supported AI provider (OpenAI, Anthropic, Gemini, Ollama, etc.) to classify, label, or otherwise enrich the text with structured metadata
|
||||
- **PDF Conversion (Gotenberg)**:
|
||||
- Convert non-PDF attachments (e.g., Word docs, images) into PDFs
|
||||
- **Document Management (Paperless NGX)**:
|
||||
- Store processed documents and metadata in a Paperless NGX instance
|
||||
- **IMAP Integration**:
|
||||
- Fetch documents from multiple mailboxes (including Gmail) and automatically enqueue them for processing
|
||||
- **Authentication**:
|
||||
- Secure access to the system using **Authentik** for OAuth2-based login
|
||||
- **Document Upload & Storage**:
|
||||
- Manual uploads (via API or UI) to Dropbox, Nextcloud, or Paperless.
|
||||
- **OCR Processing (Azure)**:
|
||||
- Extract text from scanned PDFs using Azure Document Intelligence.
|
||||
- **Metadata Extraction (OpenAI)**:
|
||||
- Use GPT to classify, label, or otherwise enrich the text with structured metadata.
|
||||
- **PDF Conversion (Gotenberg)**:
|
||||
- Convert non-PDF attachments (e.g., Word docs, images) into PDFs.
|
||||
- **Document Management (Paperless NGX)**:
|
||||
- Store processed documents and metadata in a Paperless NGX instance.
|
||||
- **IMAP Integration**:
|
||||
- Fetch documents from multiple mailboxes (including Gmail) and automatically enqueue them for processing.
|
||||
- **Authentication**:
|
||||
- Secure access to the system using **Authentik** for OAuth2-based login.
|
||||
|
||||
## Frameworks Used
|
||||
|
||||
- **FastAPI**: High-performance web framework for APIs.
|
||||
- **Celery**: Task queue for asynchronous processing.
|
||||
- **Redis**: Message broker and result backend.
|
||||
- **SQLAlchemy**: ORM for database interactions.
|
||||
- **Tailwind CSS**: Utility-first CSS framework.
|
||||
- **Docker**: Containerization for easy deployment.
|
||||
- **FastAPI**: A modern, fast (high-performance) web framework for building APIs with Python.
|
||||
- **Celery**: A distributed task queue for asynchronous processing.
|
||||
- **SQLAlchemy**: A powerful ORM for database interactions.
|
||||
- **Jinja2**: A templating engine for rendering HTML pages.
|
||||
- **Tailwind CSS**: A utility-first CSS framework for styling the UI.
|
||||
|
||||
## Quick Start
|
||||
## Environment Variables
|
||||
|
||||
For detailed installation and deployment instructions, please refer to the [Deployment Guide](docs/DeploymentGuide.md).
|
||||
The `.env` file drives all configuration. This table breaks down key variables—some are optional, depending on which services you actually use.
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone https://github.com/christianlouis/DocuElevate.git
|
||||
cd DocuElevate
|
||||
### Core Settings
|
||||
|
||||
# Configure environment variables
|
||||
cp .env.demo .env
|
||||
# Edit .env with your settings
|
||||
| **Variable** | **Description** | **Example** |
|
||||
|------------------------|----------------------------------------------------------|--------------------------------|
|
||||
| `DATABASE_URL` | Path/URL to the SQLite database (or other SQL backend). | `sqlite:///./app/database.db` |
|
||||
| `REDIS_URL` | URL for Redis, used by Celery for broker & result store. | `redis://redis:6379/0` |
|
||||
| `WORKDIR` | Working directory for the application. | `/workdir` |
|
||||
| `GOTENBERG_URL` | Gotenberg PDF processing URL. | `http://gotenberg:3000` |
|
||||
|
||||
# Run with Docker Compose
|
||||
docker-compose up -d
|
||||
### IMAP Configuration (Multiple Mailboxes)
|
||||
|
||||
| **Variable** | **Description** | **Example** |
|
||||
|-------------------------------|--------------------------------------------------------------|-------------------|
|
||||
| `IMAP1_HOST` | Hostname for first IMAP server. | `mail.example.com`|
|
||||
| `IMAP1_PORT` | Port number (usually `993`). | `993` |
|
||||
| `IMAP1_USERNAME` | IMAP login (first mailbox). | `user@example.com`|
|
||||
| `IMAP1_PASSWORD` | IMAP password (first mailbox). | `*******` |
|
||||
| `IMAP1_SSL` | Use SSL (`true`/`false`). | `true` |
|
||||
| `IMAP1_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll for new mail. | `5` |
|
||||
| `IMAP1_DELETE_AFTER_PROCESS` | Delete emails after processing (`true`/`false`). | `false` |
|
||||
| `IMAP2_HOST` | Hostname for second IMAP server (optional). | `imap.gmail.com` |
|
||||
| `IMAP2_PORT` | Port number for second mailbox. | `993` |
|
||||
| `IMAP2_USERNAME` | IMAP login for second mailbox. | `you@gmail.com` |
|
||||
| `IMAP2_PASSWORD` | IMAP password for second mailbox. | `*******` |
|
||||
| `IMAP2_SSL` | Use SSL for second mailbox (`true`/`false`). | `true` |
|
||||
| `IMAP2_POLL_INTERVAL_MINUTES` | Frequency in minutes to poll second mailbox. | `10` |
|
||||
| `IMAP2_DELETE_AFTER_PROCESS` | Delete emails after processing (`true`/`false`) for mailbox.| `false` |
|
||||
|
||||
### OpenAI & Azure Document Intelligence
|
||||
|
||||
| **Variable** | **Description** | **How to Obtain** |
|
||||
|-----------------------|--------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------|
|
||||
| `OPENAI_API_KEY` | API key for OpenAI services (used for metadata extraction/refinement). | [OpenAI platform](https://platform.openai.com/account/api-keys) |
|
||||
| `OPENAI_BASE_URL` | Base URL for OpenAI API (optional, defaults to OpenAI's endpoint). | `https://api.openai.com/v1` |
|
||||
| `OPENAI_MODEL` | OpenAI model to use for tasks (e.g., GPT-4). | `gpt-4` |
|
||||
| `AZURE_AI_KEY` | Azure Document Intelligence key (for OCR). | [Azure Portal](https://portal.azure.com/) |
|
||||
| `AZURE_REGION` | Azure region of your Document Intelligence instance. | e.g. `eastus`, `westeurope` |
|
||||
| `AZURE_ENDPOINT` | Endpoint URL for Document Intelligence. | e.g. `https://<yourendpoint>.cognitiveservices.azure.com/` |
|
||||
|
||||
### Authentik
|
||||
|
||||
| **Variable** | **Description** |
|
||||
|-------------------------|---------------------------------------------------------------|
|
||||
| `AUTH_ENABLED` | Enable or disable authentication (`true`/`false`). |
|
||||
| `AUTHENTIK_CLIENT_ID` | Client ID for Authentik OAuth2. |
|
||||
| `AUTHENTIK_CLIENT_SECRET` | Client secret for Authentik OAuth2. |
|
||||
| `AUTHENTIK_CONFIG_URL` | Configuration URL for Authentik OpenID Connect. |
|
||||
|
||||
### Paperless NGX
|
||||
|
||||
| **Variable** | **Description** |
|
||||
|-------------------------------|-----------------------------------------------------|
|
||||
| `PAPERLESS_NGX_API_TOKEN` | API token for Paperless NGX. |
|
||||
| `PAPERLESS_HOST` | Root URL for Paperless NGX (e.g. `https://paperless.example.com`). |
|
||||
|
||||
### Dropbox
|
||||
|
||||
| **Variable** | **Description** | **How to Obtain** |
|
||||
|-------------------------|--------------------------------------------------|------------------------------------------------------------------------------------|
|
||||
| `DROPBOX_APP_KEY` | Dropbox API app key. | [Dropbox Developer Console](https://www.dropbox.com/developers/apps/create) |
|
||||
| `DROPBOX_APP_SECRET` | Dropbox API app secret. | [Dropbox Developer Console](https://www.dropbox.com/developers/apps/create) |
|
||||
| `DROPBOX_REFRESH_TOKEN` | OAuth2 refresh token for Dropbox. | Follow Dropbox OAuth flow to retrieve |
|
||||
| `DROPBOX_FOLDER` | Default folder path for Dropbox uploads. | e.g. `"/Documents/Uploads"` |
|
||||
|
||||
### Nextcloud
|
||||
|
||||
| **Variable** | **Description** |
|
||||
|-------------------------|---------------------------------------------------------------|
|
||||
| `NEXTCLOUD_UPLOAD_URL` | Nextcloud WebDAV URL (e.g. `https://nc.example.com/remote.php/dav/files/<USERNAME>`). |
|
||||
| `NEXTCLOUD_USERNAME` | Nextcloud login username. |
|
||||
| `NEXTCLOUD_PASSWORD` | Nextcloud login password. |
|
||||
| `NEXTCLOUD_FOLDER` | Destination folder in Nextcloud (e.g. `"/Documents/Uploads"`). |
|
||||
|
||||
## Running as a Docker Container
|
||||
|
||||
This project uses Celery (with Redis) for asynchronous task management and Gotenberg for PDF conversion. The `docker-compose.yml` file defines these services:
|
||||
|
||||
- **API Service**: Runs the FastAPI application via `uvicorn`.
|
||||
- **Worker Service**: Runs the Celery worker for processing tasks (PDF conversions, OCR, etc.).
|
||||
- **Redis**: Provides the message broker & result backend for Celery.
|
||||
- **Gotenberg**: Offers PDF conversion capabilities.
|
||||
|
||||
### Running the Application with Docker Compose
|
||||
|
||||
1. **Install Docker and Docker Compose** on your system.
|
||||
2. **Clone the repository** and navigate into it:
|
||||
```bash
|
||||
git clone <repository_url>
|
||||
cd <repository_name>
|
||||
```
|
||||
3. **Create and configure the `.env` file**:
|
||||
- Fill in the variables from the tables above.
|
||||
- (At minimum, you need `DATABASE_URL`, `REDIS_URL`, `WORKDIR`, plus whichever service creds you plan to use.)
|
||||
4. **Launch the services**:
|
||||
```bash
|
||||
docker-compose up -d
|
||||
```
|
||||
5. The API will be available at **`http://localhost:8000`**, and the API documentation is available at **`http://localhost:8000/docs`**.
|
||||
|
||||
### Services in `docker-compose.yml`
|
||||
|
||||
Below is the default structure (simplified):
|
||||
|
||||
```yaml
|
||||
services:
|
||||
api:
|
||||
image: christianlouis/document-processor:latest
|
||||
container_name: document_api
|
||||
working_dir: /workdir
|
||||
command: ["sh", "-c", "cd /app && uvicorn app.main:app --host 0.0.0.0 --port 8000"]
|
||||
environment:
|
||||
- PYTHONPATH=/app
|
||||
env_file:
|
||||
- .env
|
||||
ports:
|
||||
- "8000:8000"
|
||||
depends_on:
|
||||
- redis
|
||||
- worker
|
||||
volumes:
|
||||
- /var/docparse/workdir:/workdir
|
||||
|
||||
worker:
|
||||
image: christianlouis/document-processor:latest
|
||||
container_name: document_worker
|
||||
working_dir: /workdir
|
||||
command: ["celery", "-A", "app.celery_worker", "worker", "-B", "--loglevel=info", "-Q", "document_processor,default,celery"]
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
- PYTHONPATH=/app
|
||||
depends_on:
|
||||
- redis
|
||||
- gotenberg
|
||||
volumes:
|
||||
- /var/docparse/workdir:/workdir
|
||||
|
||||
gotenberg:
|
||||
image: gotenberg/gotenberg:latest
|
||||
container_name: gotenberg
|
||||
|
||||
redis:
|
||||
image: redis:alpine
|
||||
container_name: document_redis
|
||||
restart: always
|
||||
```
|
||||
|
||||
The API will be available at **`http://localhost:8000`**, and the API documentation is available at **`http://localhost:8000/docs`**.
|
||||
## To-Do List
|
||||
|
||||
## Development & Testing
|
||||
- **Make upload targets configurable** (e.g., easily choose only Dropbox, Nextcloud, or Paperless).
|
||||
|
||||
### Running Tests
|
||||
---
|
||||
|
||||
DocuElevate includes comprehensive test coverage. To run tests:
|
||||
|
||||
```bash
|
||||
# Install development dependencies
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
# Run all tests
|
||||
pytest
|
||||
|
||||
# Run with coverage report
|
||||
pytest --cov=app --cov-report=term-missing
|
||||
|
||||
# Run only fast unit tests
|
||||
pytest -m unit
|
||||
```
|
||||
|
||||
Tests are automatically configured with the necessary environment variables - **no manual setup required!**
|
||||
|
||||
For detailed testing information, including integration tests with Docker and authentication testing, see the [Contributing Guide](CONTRIBUTING.md#running-tests).
|
||||
|
||||
### Contributing
|
||||
|
||||
We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for:
|
||||
- Code style guidelines
|
||||
- Commit message format (Conventional Commits)
|
||||
- Testing requirements
|
||||
- Pull request process
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details.
|
||||
|
||||
## Third-Party Software
|
||||
|
||||
This project uses various third-party libraries and components. See [NOTICE](NOTICE) for attributions and the [attribution page](frontend/templates/attribution.html) in the application for more details.
|
||||
|
||||
### LGPL Compliance
|
||||
|
||||
This project uses Paramiko which is licensed under LGPL-2.1. In accordance with the LGPL license:
|
||||
|
||||
- The source code for Paramiko can be obtained from https://github.com/paramiko/paramiko
|
||||
- A copy of the LGPL license is available in the application at `/licenses/lgpl.txt`
|
||||
- Users have the right to modify and redistribute Paramiko under the terms of the LGPL
|
||||
|
||||
## Dependency Licenses
|
||||
|
||||
The following is a summary of the licenses used by our direct dependencies:
|
||||
|
||||
| Dependency | License |
|
||||
|------------|---------|
|
||||
| FastAPI | MIT |
|
||||
| Celery | BSD |
|
||||
| Uvicorn | BSD |
|
||||
| SQLAlchemy | MIT |
|
||||
| Pydantic | MIT |
|
||||
| openai | MIT |
|
||||
| litellm | MIT |
|
||||
| pypdf | BSD |
|
||||
| Requests | Apache 2.0 |
|
||||
| puremagic | MIT |
|
||||
| filetype | MIT |
|
||||
| Dropbox | MIT |
|
||||
| Azure AI Document Intelligence | MIT |
|
||||
| Authlib | BSD |
|
||||
| Starlette | BSD |
|
||||
| Alembic | MIT |
|
||||
| Google API Client | Apache 2.0 |
|
||||
| Microsoft Graph Core | MIT |
|
||||
| MSAL | MIT |
|
||||
| Boto3 | Apache 2.0 |
|
||||
| Paramiko | LGPL-2.1|
|
||||
| Apprise | MIT |
|
||||
| Redis | BSD |
|
||||
| Gotenberg | MIT |
|
||||
|
||||
For a comprehensive list of all dependencies and their licenses, run:
|
||||
|
||||
```
|
||||
pip install pip-licenses
|
||||
pip-licenses
|
||||
|
||||
```
|
||||
**Questions or Issues?**
|
||||
- Feel free to open an issue or pull request.
|
||||
- For local testing or development, use `docker-compose up` and watch the logs via `docker-compose logs -f`.
|
||||
- Ensure your `.env` aligns with the environment variables listed above. If you see unexpected errors, check for typos or missing values.
|
||||
-222
@@ -1,222 +0,0 @@
|
||||
# DocuElevate Roadmap
|
||||
|
||||
**Last Updated:** 2026-02-08
|
||||
**Version:** 1.0
|
||||
|
||||
## Vision
|
||||
|
||||
DocuElevate aims to be the premier open-source intelligent document processing platform, providing seamless integration with cloud storage providers, advanced AI-powered metadata extraction, and enterprise-grade security and scalability.
|
||||
|
||||
## Current Status (v0.5.0)
|
||||
|
||||
### Core Features ✅
|
||||
- Multi-provider document storage (Dropbox, Google Drive, OneDrive, Nextcloud, S3, etc.)
|
||||
- IMAP email integration for document ingestion
|
||||
- OCR processing via Azure Document Intelligence
|
||||
- AI-powered metadata extraction via OpenAI
|
||||
- PDF conversion via Gotenberg
|
||||
- Web UI for document upload and management
|
||||
- **Database-backed settings management with admin UI**
|
||||
- **Fernet encryption for sensitive configuration**
|
||||
- **Setup wizard for first-time installation**
|
||||
- REST API with OpenAPI documentation
|
||||
- Celery-based async task processing
|
||||
- OAuth2 authentication via Authentik with admin group support
|
||||
|
||||
## Short-term Goals (Q1-Q2 2026) - v0.4.x to v0.5.x
|
||||
|
||||
### Quality & Stability 🎯
|
||||
- **Test Coverage** (High Priority)
|
||||
- [ ] Achieve 80% code coverage for core modules
|
||||
- [ ] Add integration tests for all storage providers
|
||||
- [ ] Add end-to-end workflow tests
|
||||
- [ ] Performance benchmarks and load testing
|
||||
|
||||
- **Code Quality** (High Priority)
|
||||
- [ ] Enable strict linting in CI/CD
|
||||
- [ ] Refactor large modules for better maintainability
|
||||
- [ ] Add comprehensive type hints
|
||||
- [ ] Improve error handling and user feedback
|
||||
|
||||
- **Security** (Critical Priority)
|
||||
- [x] Fix known vulnerabilities in dependencies
|
||||
- [ ] Implement rate limiting on API endpoints
|
||||
- [ ] Add CSRF protection
|
||||
- [ ] Security audit by external party
|
||||
- [ ] Implement API key rotation
|
||||
- [ ] Add audit logging for sensitive operations
|
||||
|
||||
- **Release Automation** (Completed ✅)
|
||||
- [x] Implement semantic-release for automated versioning
|
||||
- [x] Add conventional commit validation
|
||||
- [x] Automate CHANGELOG generation
|
||||
- [x] Integrate Docker builds with releases
|
||||
|
||||
### Features - v0.4.0
|
||||
- **Enhanced Search & Filtering**
|
||||
- [ ] Full-text search across documents
|
||||
- [ ] Advanced filtering by metadata, tags, date ranges
|
||||
- [ ] Saved search queries
|
||||
- [ ] Bulk operations on search results
|
||||
|
||||
- **Improved UI/UX**
|
||||
- [ ] Responsive mobile interface
|
||||
- [ ] Dark mode support
|
||||
- [ ] Document preview in browser
|
||||
- [ ] Drag-and-drop file upload
|
||||
- [ ] Progress indicators for long-running tasks
|
||||
- [ ] Real-time notifications via WebSocket
|
||||
|
||||
### Features - v0.5.0
|
||||
- **Workflow Automation**
|
||||
- [ ] Custom processing pipelines
|
||||
- [ ] Conditional routing based on document type
|
||||
- [ ] Scheduled batch processing
|
||||
- [ ] Webhook support for external integrations
|
||||
- [ ] Rule-based document classification
|
||||
|
||||
- **Advanced AI Features**
|
||||
- [ ] Custom AI models for specialized document types
|
||||
- [ ] Multi-language OCR support
|
||||
- [ ] Document similarity detection
|
||||
- [ ] Automatic duplicate detection
|
||||
- [ ] Intelligent document splitting
|
||||
|
||||
## Medium-term Goals (Q3-Q4 2026) - v1.0.x
|
||||
|
||||
### Enterprise Features - v1.0.0
|
||||
- **Multi-tenancy**
|
||||
- [ ] Organization/team management
|
||||
- [ ] Role-based access control (RBAC)
|
||||
- [ ] Per-tenant configuration
|
||||
- [ ] Resource quotas and limits
|
||||
- [ ] Audit logs per organization
|
||||
|
||||
- **Scalability**
|
||||
- [ ] Horizontal scaling support
|
||||
- [ ] Distributed task processing
|
||||
- [ ] Caching layer (Redis/Memcached)
|
||||
- [ ] Database connection pooling
|
||||
- [ ] Message queue optimization
|
||||
|
||||
- **Advanced Integrations**
|
||||
- [ ] Microsoft SharePoint integration
|
||||
- [ ] Slack/Teams bot integration
|
||||
- [ ] Zapier/Make.com integration
|
||||
- [ ] Custom webhook receivers
|
||||
- [ ] GraphQL API
|
||||
|
||||
### Features - v1.1.0
|
||||
- **Collaboration**
|
||||
- [ ] Document sharing with expiring links
|
||||
- [ ] Comments and annotations
|
||||
- [ ] Version history and rollback
|
||||
- [ ] Real-time collaborative editing metadata
|
||||
- [ ] Activity feed
|
||||
|
||||
- **Reporting & Analytics**
|
||||
- [ ] Processing statistics dashboard
|
||||
- [ ] Storage usage analytics
|
||||
- [ ] AI confidence scores and accuracy tracking
|
||||
- [ ] Cost analysis per provider
|
||||
- [ ] Export reports (PDF, CSV, Excel)
|
||||
|
||||
## Long-term Goals (2027+) - v2.0+
|
||||
|
||||
### Strategic Initiatives
|
||||
- **On-Premise AI Models**
|
||||
- [ ] Self-hosted OCR (Tesseract, EasyOCR)
|
||||
- [ ] Local LLM integration (Ollama, LLaMA)
|
||||
- [ ] GPU acceleration support
|
||||
- [ ] Model fine-tuning interface
|
||||
- [ ] Hybrid cloud/on-premise processing
|
||||
|
||||
- **Advanced Document Management**
|
||||
- [ ] Document lifecycle management
|
||||
- [ ] Retention policies and auto-deletion
|
||||
- [ ] Compliance templates (GDPR, HIPAA, SOC2)
|
||||
- [ ] Digital signature support
|
||||
- [ ] Encryption at rest and in transit
|
||||
|
||||
- **Platform Expansion**
|
||||
- [ ] Desktop applications (Electron)
|
||||
- [ ] Mobile apps (iOS/Android)
|
||||
- [ ] Browser extensions
|
||||
- [ ] Command-line interface (CLI)
|
||||
- [ ] VS Code extension for developers
|
||||
|
||||
### Research & Innovation
|
||||
- [ ] Machine learning for custom document types
|
||||
- [ ] Blockchain for document provenance
|
||||
- [ ] Federated learning for privacy-preserving AI
|
||||
- [ ] Edge computing support
|
||||
- [ ] Quantum-resistant encryption
|
||||
|
||||
## Community & Ecosystem
|
||||
|
||||
### Developer Experience
|
||||
- [ ] Plugin system for custom processors
|
||||
- [ ] Marketplace for extensions
|
||||
- [ ] SDK for multiple languages (Python, JavaScript, Go)
|
||||
- [ ] Template library for common workflows
|
||||
- [ ] Video tutorials and courses
|
||||
|
||||
### Documentation
|
||||
- [x] User guide
|
||||
- [x] API documentation
|
||||
- [x] Deployment guide
|
||||
- [ ] Architecture deep-dive
|
||||
- [ ] Contributing guide enhancements
|
||||
- [ ] Video walkthroughs
|
||||
- [ ] Internationalization (i18n) of docs
|
||||
|
||||
### Community Building
|
||||
- [ ] Regular community calls
|
||||
- [ ] Bug bounty program
|
||||
- [ ] Ambassador program
|
||||
- [ ] Annual conference/meetup
|
||||
- [ ] Certification program
|
||||
|
||||
## Technology Debt
|
||||
|
||||
### Refactoring Needed
|
||||
- [x] Migrate from PyPDF2 to pypdf (modern fork) - ✅ Completed 2026-02-12
|
||||
- [ ] Standardize error handling across modules
|
||||
- [ ] Consolidate configuration management
|
||||
- [ ] Optimize database queries
|
||||
- [ ] Reduce code duplication in storage providers
|
||||
|
||||
### Performance Optimization
|
||||
- [ ] Profile and optimize hot paths
|
||||
- [ ] Implement lazy loading for UI
|
||||
- [ ] Add CDN for static assets
|
||||
- [ ] Optimize Docker image size
|
||||
- [ ] Database indexing strategy
|
||||
|
||||
## Deprecation Notice
|
||||
|
||||
### Planned Deprecations
|
||||
- None currently planned
|
||||
|
||||
### Migration Guides
|
||||
- Will be provided for any breaking changes
|
||||
|
||||
## How to Contribute
|
||||
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md) for detailed guidelines. Roadmap items are open for discussion and contributions!
|
||||
|
||||
### Priority Labels
|
||||
- 🔴 Critical - Security, data loss, or major bugs
|
||||
- 🟠 High - Important features or significant improvements
|
||||
- 🟡 Medium - Nice-to-have features or minor improvements
|
||||
- 🟢 Low - Future considerations or research items
|
||||
|
||||
## Feedback & Requests
|
||||
|
||||
- **GitHub Issues:** Feature requests and bug reports
|
||||
- **GitHub Discussions:** General questions and ideas
|
||||
- **Email:** [Maintainer contact from repository]
|
||||
|
||||
---
|
||||
|
||||
*This roadmap is a living document and may change based on community feedback, technical constraints, and strategic priorities.*
|
||||
@@ -1,10 +0,0 @@
|
||||
DocuElevate Build Information
|
||||
==============================
|
||||
Version: 0.67.2
|
||||
Build Date: 2026-03-01T17:23:25Z
|
||||
Git Commit: 0134ed37d5c602faf5b10cc6a7229263ba2f6aa1
|
||||
Git Short SHA: 0134ed3
|
||||
Git Branch: main
|
||||
Commit Date: 2026-03-01T18:23:06+01:00
|
||||
Build Timestamp: 2026-03-01T17:23:25Z
|
||||
==============================
|
||||
-58
@@ -1,58 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
| Version | Supported |
|
||||
| ------- | ------------------ |
|
||||
| 0.4.x | :white_check_mark: |
|
||||
| 0.3.x | :white_check_mark: |
|
||||
| 0.2.x | :white_check_mark: |
|
||||
| < 0.2 | :x: |
|
||||
|
||||
Each version will be supported for six months after release or until a new release has been made, whichever is longer.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
We take the security of our document-processor seriously. If you believe you've found a security vulnerability, please follow these steps:
|
||||
|
||||
### How to Report
|
||||
|
||||
1. **Do NOT disclose the vulnerability publicly** until it has been addressed by our team.
|
||||
2. Email your findings to [security@christianlouis.de](mailto:security@christianlouis.de). Encrypt your message if it contains sensitive details.
|
||||
3. Include as much information as possible:
|
||||
- Type of vulnerability
|
||||
- Full paths of source files related to the vulnerability
|
||||
- Step-by-step instructions to reproduce the issue
|
||||
- Proof of concept code, if possible
|
||||
- Impact of the vulnerability
|
||||
|
||||
### What to Expect
|
||||
|
||||
- A confirmation email within 48 hours acknowledging your report.
|
||||
- An assessment and validation of the reported vulnerability within 1 week.
|
||||
- Regular updates about the progress of addressing the vulnerability.
|
||||
- Credit for discovering and reporting the vulnerability (if desired).
|
||||
|
||||
### Disclosure Policy
|
||||
|
||||
- Please allow us reasonable time to resolve the issue before making any public disclosures.
|
||||
- We aim to address confirmed vulnerabilities within 30-90 days, depending on complexity.
|
||||
- Once the vulnerability is fixed, we'll publish a security advisory with details and credit.
|
||||
|
||||
## Security Best Practices
|
||||
|
||||
When using document-processor:
|
||||
- Keep your installation up-to-date with the latest security patches
|
||||
- Use strong access controls and authentication mechanisms
|
||||
- Validate all inputs from untrusted sources
|
||||
- Follow the principle of least privilege when configuring permissions
|
||||
|
||||
## Security Updates
|
||||
|
||||
Security updates will be released as part of our regular versioning process. Critical security fixes may be released as out-of-band updates.
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
We'd like to thank the following individuals for responsibly reporting security issues:
|
||||
|
||||
*This list will be updated as contributions are received.*
|
||||
@@ -1,863 +0,0 @@
|
||||
# Security Audit Report
|
||||
|
||||
**Date:** 2026-02-12
|
||||
**Status:** Bandit Security Scan Completed - All Critical/High/Medium Issues Resolved
|
||||
|
||||
## Executive Summary
|
||||
|
||||
This document tracks security vulnerabilities found in DocuElevate and their remediation status. A comprehensive security audit using Bandit has been completed, with all critical, high, and medium severity issues addressed.
|
||||
|
||||
## Recent Security Fixes
|
||||
|
||||
### CVE-2023-36464: PyPDF2/pypdf Infinite Loop Vulnerability ✅ FIXED (2026-02-12)
|
||||
|
||||
**Severity:** Moderate (CVSS: 5.5)
|
||||
**CVE:** [CVE-2023-36464](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-36464)
|
||||
**Advisory:** [GHSA-4vvm-4w3v-6mr8](https://github.com/advisories/GHSA-4vvm-4w3v-6mr8)
|
||||
|
||||
**Issue:** Certain versions of PyPDF2 (>=2.2.0, <=3.0.1) and pypdf (prior to 3.9.0) contain a vulnerability where specially crafted PDF files can trigger an infinite loop in `__parse_content_stream`, causing 100% CPU usage and potential denial of service.
|
||||
|
||||
**Impact:**
|
||||
- **Availability:** High (can block process and consume 100% CPU)
|
||||
- **Confidentiality:** None
|
||||
- **Integrity:** None
|
||||
- **Attack Vector:** Local
|
||||
- **Privileges Required:** None
|
||||
|
||||
**Remediation:**
|
||||
- Upgraded from `PyPDF2>=3.0.0` (vulnerable) to `pypdf>=3.9.0` (fixed)
|
||||
- Updated all imports from `PyPDF2` to `pypdf` across the codebase
|
||||
- Verified pypdf 6.7.0 installed successfully
|
||||
- **Files Updated:**
|
||||
- `requirements.txt` - Updated dependency specification
|
||||
- `app/tasks/process_document.py`
|
||||
- `app/tasks/rotate_pdf_pages.py`
|
||||
- `app/utils/file_splitting.py`
|
||||
- `app/tasks/embed_metadata_into_pdf.py`
|
||||
- `app/tasks/process_with_azure_document_intelligence.py`
|
||||
- `app/views/files.py`
|
||||
- `app/api/files.py`
|
||||
- `tests/test_external_integrations.py`
|
||||
- `tests/test_file_splitting.py`
|
||||
|
||||
**Testing:** All affected modules verified for syntax correctness and basic import functionality.
|
||||
|
||||
**References:**
|
||||
- [py-pdf/pypdf#1828](https://github.com/py-pdf/pypdf/pull/1828) - Fix implementation
|
||||
- [py-pdf/pypdf#969](https://github.com/py-pdf/pypdf/pull/969) - Issue introduction
|
||||
|
||||
## Bandit Security Scan Results (2026-02-07)
|
||||
|
||||
**Scan Summary:**
|
||||
- **Total lines scanned:** 7,423
|
||||
- **High severity issues:** 0 (6 fixed)
|
||||
- **Medium severity issues:** 0 (15 fixed)
|
||||
- **Low severity issues:** 21 (informational/acceptable)
|
||||
|
||||
### Fixed Issues from Bandit Scan
|
||||
|
||||
#### 1. B324: Weak MD5 Hash Usage (HIGH SEVERITY) ✅ FIXED
|
||||
**Occurrences:** 2
|
||||
**Locations:**
|
||||
- `app/api/user.py:26` - Gravatar URL generation
|
||||
- `app/auth.py:65` - Gravatar URL generation
|
||||
|
||||
**Issue:** MD5 hash was used without specifying `usedforsecurity=False` parameter.
|
||||
|
||||
**Remediation:** Added `usedforsecurity=False` parameter to all MD5 hash calls. MD5 is used only for Gravatar URL generation (non-cryptographic purpose), which is an acceptable use case.
|
||||
|
||||
```python
|
||||
# Before: email_hash = md5(email.encode()).hexdigest()
|
||||
# After: email_hash = md5(email.encode(), usedforsecurity=False).hexdigest()
|
||||
```
|
||||
|
||||
#### 2. B402/B321: Insecure FTP Protocol (HIGH SEVERITY) ✅ DOCUMENTED
|
||||
**Occurrences:** 3
|
||||
**Location:** `app/tasks/upload_to_ftp.py`
|
||||
|
||||
**Issue:** FTP is an insecure protocol vulnerable to eavesdropping and MITM attacks.
|
||||
|
||||
**Remediation:**
|
||||
- Added comprehensive security warnings in code comments
|
||||
- Code already defaults to FTPS (FTP_TLS) for encrypted connections
|
||||
- Plaintext FTP only used as fallback when explicitly configured
|
||||
- Added `# nosec B402` and `# nosec B321` annotations with justification
|
||||
- Added security notes in docstrings
|
||||
- Configuration options: `ftp_use_tls=True` (default), `ftp_allow_plaintext=True` (default)
|
||||
|
||||
**Security Note:** For production environments, set `ftp_allow_plaintext=False` to prevent fallback to unencrypted FTP.
|
||||
|
||||
#### 3. B507: SSH Host Key Verification Disabled (HIGH SEVERITY) ✅ FIXED
|
||||
**Occurrences:** 1
|
||||
**Location:** `app/tasks/upload_to_sftp.py:47`
|
||||
|
||||
**Issue:** Using `paramiko.AutoAddPolicy()` automatically trusts unknown SSH host keys, making connections vulnerable to MITM attacks.
|
||||
|
||||
**Remediation:**
|
||||
- Added configuration option `sftp_disable_host_key_verification` (default: False for security)
|
||||
- When enabled (False), uses `paramiko.RejectPolicy()` with system known_hosts for secure verification
|
||||
- When disabled (True, for testing only), uses `AutoAddPolicy()` with security warnings
|
||||
- Added `# nosec B507` annotation with justification for the test/dev use case
|
||||
- Updated docstrings with security guidance
|
||||
|
||||
**Security Note:** The default value is now `False` (secure). For development/testing environments where host keys cannot be pre-configured, set `SFTP_DISABLE_HOST_KEY_VERIFICATION=True` (not recommended for production).
|
||||
|
||||
#### 4. B113: Missing Timeout on HTTP Requests (MEDIUM SEVERITY) ✅ FIXED
|
||||
**Occurrences:** 15
|
||||
**Locations:**
|
||||
- `app/api/dropbox.py` (4 requests calls)
|
||||
- `app/api/google_drive.py` (1 request call)
|
||||
- `app/api/onedrive.py` (3 requests calls)
|
||||
- `app/tasks/convert_to_pdf.py` (1 request call)
|
||||
- `app/tasks/upload_to_dropbox.py` (1 request call)
|
||||
- `app/tasks/upload_to_paperless.py` (2 requests calls)
|
||||
- `app/tasks/upload_to_onedrive.py` (2 requests calls)
|
||||
- `app/tasks/upload_to_webdav.py` (1 request call)
|
||||
|
||||
**Issue:** HTTP requests without timeout can hang indefinitely, leading to resource exhaustion and potential DoS.
|
||||
|
||||
**Remediation:**
|
||||
- Added `http_request_timeout` configuration setting (default: 120 seconds)
|
||||
- Timeout configured to handle large file operations (PDFs up to 1GB+)
|
||||
- Applied `timeout=settings.http_request_timeout` to all `requests.get()`, `requests.post()`, and `requests.put()` calls
|
||||
- Configurable via environment variable: `HTTP_REQUEST_TIMEOUT=120`
|
||||
|
||||
**Note:** The 120-second default timeout is appropriate for:
|
||||
- Large PDF file uploads and downloads (up to 1GB)
|
||||
- PDF conversion operations via Gotenberg
|
||||
- Cloud storage uploads (Dropbox, OneDrive, Google Drive, Nextcloud, WebDAV)
|
||||
- Document processing and OCR operations
|
||||
|
||||
### Low Severity Issues (Informational)
|
||||
|
||||
**21 low severity findings remain** - These are informational warnings about:
|
||||
- `assert` statements (B101) - Used in non-security contexts
|
||||
- Try-except-pass blocks (B110) - Acceptable for optional operations
|
||||
- Subprocess calls (B603/B607) - Verified safe (hardcoded commands, no user input)
|
||||
- Hard-coded temp directories (B108) - Platform-appropriate temp paths
|
||||
- Hard-coded bind addresses (B104) - Development defaults
|
||||
|
||||
**Assessment:** All low severity findings have been reviewed and are acceptable given the context of their usage.
|
||||
|
||||
## Critical Vulnerabilities (Fixed) ✅
|
||||
|
||||
### 1. Outdated Authlib with Known Vulnerabilities
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** HIGH
|
||||
**Description:** Authlib version 1.3.2 had two critical vulnerabilities:
|
||||
- CVE: Denial of Service via Oversized JOSE Segments
|
||||
- CVE: JWS/JWT accepts unknown crit headers (RFC violation → possible authz bypass)
|
||||
|
||||
**Fix:** Updated `requirements.txt` to require `authlib>=1.6.5`
|
||||
|
||||
### 2. Starlette DoS Vulnerability
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** MEDIUM
|
||||
**Description:** Starlette 0.41.3 vulnerable to O(n^2) DoS via Range header merging in `FileResponse`
|
||||
|
||||
**Fix:** Updated `requirements.txt` to require `starlette>=0.49.1`
|
||||
|
||||
### 3. Weak SESSION_SECRET Default
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** HIGH
|
||||
**Description:** Default SESSION_SECRET value in `app/main.py` was a predictable string that could be exploited if not overridden
|
||||
|
||||
**Fix:**
|
||||
- Enhanced validation in `app/main.py` to raise error if auth is enabled without proper secret
|
||||
- Updated default to be clearly marked as insecure for development only
|
||||
- Added generation instructions in error message
|
||||
|
||||
## Medium Risk Issues (Fixed) ✅
|
||||
|
||||
### 4. Insufficient .gitignore Protection
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** MEDIUM
|
||||
**Description:** .gitignore didn't adequately protect against accidentally committing sensitive files (credentials, private keys, secrets)
|
||||
|
||||
**Fix:** Enhanced `.gitignore` with comprehensive patterns for:
|
||||
- Various environment file formats
|
||||
- Credential JSON files
|
||||
- Private keys (.pem, .key, .pfx, etc.)
|
||||
- SSH keys
|
||||
- Explicit exclusion of patterns where needed
|
||||
|
||||
### 5. File Upload Size Limits
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** MEDIUM
|
||||
**Description:** No configurable limits on file upload sizes could lead to resource exhaustion attacks and DoS.
|
||||
|
||||
**Fix:** Implemented configurable file upload size limits with the following features:
|
||||
- `MAX_UPLOAD_SIZE`: Maximum file upload size in bytes (default: 1GB)
|
||||
- `MAX_SINGLE_FILE_SIZE`: Optional maximum size for a single file chunk
|
||||
- **Automatic page-based PDF splitting** for large PDFs when max_single_file_size is configured
|
||||
- Splits PDFs at **page boundaries** using pypdf, NOT by byte position
|
||||
- Each output file is a structurally valid, complete PDF
|
||||
- No risk of corrupted or broken PDF files
|
||||
- Split files are processed sequentially to prevent overwhelming the system
|
||||
- Clear error messages referencing SECURITY_AUDIT.md for configuration details
|
||||
|
||||
**Configuration:**
|
||||
```bash
|
||||
# Set maximum upload size (default: 1GB)
|
||||
MAX_UPLOAD_SIZE=1073741824
|
||||
|
||||
# Optional: Enable file splitting for large PDFs
|
||||
MAX_SINGLE_FILE_SIZE=104857600 # 100MB chunks
|
||||
```
|
||||
|
||||
**Security Benefits:**
|
||||
- Prevents resource exhaustion from extremely large uploads
|
||||
- Configurable limits allow adaptation to server capacity
|
||||
- File splitting enables processing of large documents without memory issues
|
||||
- Maintains support for large PDF files (up to 1GB by default) as required by use case
|
||||
|
||||
## Best Practices Implemented
|
||||
|
||||
### Security Scanning with Bandit
|
||||
- ✅ Bandit installed in development dependencies (`requirements-dev.txt`)
|
||||
- ✅ Comprehensive scan completed on all Python code
|
||||
- ✅ High and medium severity issues resolved
|
||||
- ✅ Low severity issues reviewed and accepted
|
||||
|
||||
**Running Bandit:**
|
||||
```bash
|
||||
# Scan entire app directory
|
||||
bandit -r app
|
||||
|
||||
# Show only high and medium severity
|
||||
bandit -r app -ll
|
||||
|
||||
# Generate JSON report
|
||||
bandit -r app -f json -o bandit_results.json
|
||||
|
||||
# Generate HTML report
|
||||
bandit -r app -f html -o bandit_report.html
|
||||
```
|
||||
|
||||
**Suppressing False Positives:**
|
||||
Use `# nosec` comments with justification:
|
||||
```python
|
||||
# Security: FTP usage intentional for legacy server support
|
||||
import ftplib # nosec B402 - FTP usage is intentional
|
||||
|
||||
ftp = ftplib.FTP() # nosec B321 - Plaintext FTP intentional when configured
|
||||
```
|
||||
|
||||
### Dependency Management
|
||||
- ✅ Version pinning for security-critical packages (authlib, starlette)
|
||||
- ✅ Advisory database checks integrated into development workflow
|
||||
- ✅ Automated dependency vulnerability scanning in CI/CD via pip-audit ([#171](https://github.com/christianlouis/DocuElevate/issues/171))
|
||||
|
||||
### Authentication & Secrets
|
||||
- ✅ Strong validation for SESSION_SECRET (minimum 32 characters)
|
||||
- ✅ Error-on-missing for critical security settings when auth enabled
|
||||
- ✅ Clear documentation of secret generation methods
|
||||
- ✅ .env.demo file for configuration examples (no real secrets)
|
||||
|
||||
### Configuration Security
|
||||
- ✅ All secrets loaded from environment variables
|
||||
- ✅ No hardcoded credentials in codebase
|
||||
- ✅ Proper masking in configuration validators
|
||||
|
||||
## Ongoing Security Measures
|
||||
|
||||
### CI/CD Security
|
||||
- ✅ **COMPLETED:** Bandit (Python security linter) audit completed
|
||||
- ✅ **COMPLETED:** Bandit integrated into CI pipeline (fails on high/medium severity issues)
|
||||
- ✅ **COMPLETED:** CodeQL security scanning enabled in GitHub Actions
|
||||
- ✅ **COMPLETED:** pip-audit dependency vulnerability scanning added to CI ([#171](https://github.com/christianlouis/DocuElevate/issues/171))
|
||||
- ✅ **COMPLETED:** Dependency scans are blocking (fail build when vulnerabilities detected) ([#171](https://github.com/christianlouis/DocuElevate/issues/171))
|
||||
|
||||
### Code Security
|
||||
- ✅ Authentication required on all sensitive endpoints (@require_login decorator)
|
||||
- ✅ Path traversal protection in file uploads (basename sanitization)
|
||||
- ✅ Unique filenames with UUID to prevent conflicts and overwrites
|
||||
- ✅ File upload size limits with configurable maximum (default: 1GB)
|
||||
- ✅ Optional file splitting for large PDFs (when max_single_file_size is configured)
|
||||
- ✅ Request body size limits via `RequestSizeLimitMiddleware` (non-upload: 1MB default; uploads: governed by MAX_UPLOAD_SIZE)
|
||||
- ✅ Streaming file reads in upload endpoint to prevent memory exhaustion
|
||||
- ⏳ **TODO:** Implement rate limiting on API endpoints
|
||||
- ⏳ **TODO:** Add CSRF protection for state-changing operations
|
||||
- ✅ **COMPLETED:** Add comprehensive input sanitization for all user inputs ([#172](https://github.com/christianlouis/DocuElevate/issues/172))
|
||||
- `app/utils/input_validation.py` — centralized validation module with:
|
||||
- `validate_setting_key()`: allow-lists setting keys against `SETTING_METADATA` (prevents attribute enumeration / Python object sniffing via `getattr`)
|
||||
- `validate_sort_field()`: enforces sort field against an explicit allow-list
|
||||
- `validate_sort_order()`: ensures sort direction is exactly `asc` or `desc`
|
||||
- `validate_search_query()`: strips whitespace, enforces 255-character maximum
|
||||
- `validate_task_id()`: validates Celery task IDs against UUID v4 format
|
||||
- Applied to `app/api/settings.py` (GET/POST/DELETE `/{key}` endpoints)
|
||||
- Applied to `app/api/files.py` (file list sort + search query parameters)
|
||||
- Applied to `app/api/logs.py` (task_id query filter and path parameter)
|
||||
- 30 unit tests added in `tests/test_input_validation.py`
|
||||
- ✅ **COMPLETED:** Implement proper API key rotation mechanisms ([#168](https://github.com/christianlouis/DocuElevate/issues/168))
|
||||
- `docs/CredentialRotationGuide.md` — comprehensive rotation guide covering:
|
||||
- Recommended rotation schedule for all credential types
|
||||
- Per-credential rotation procedures for OpenAI, Azure, AWS S3, Dropbox, Google Drive, OneDrive, Authentik, Paperless-ngx, SMTP, IMAP, Nextcloud, FTP, SFTP, WebDAV, and admin credentials
|
||||
- Onboarding instructions (creating service-specific credentials with minimal permissions)
|
||||
- Offboarding instructions (revocation, rotation of shared credentials, audit log review)
|
||||
- Emergency revocation procedure
|
||||
- `GET /api/settings/credentials` — admin-only endpoint listing all sensitive credential settings with configured/unconfigured status and source (`env` vs `db`), enabling credential rotation audits without exposing secret values
|
||||
|
||||
### Infrastructure Security
|
||||
- ✅ TrustedHostMiddleware configured (restricts valid hosts)
|
||||
- ✅ ProxyHeadersMiddleware for reverse proxy setup (X-Forwarded-* headers)
|
||||
- ✅ SessionMiddleware with strong secret validation
|
||||
- ✅ **Security headers middleware implemented** - Configurable HSTS, CSP, X-Frame-Options, X-Content-Type-Options ([#174](https://github.com/christianlouis/DocuElevate/issues/174))
|
||||
- Disabled by default (typical deployment uses reverse proxy that adds headers)
|
||||
- Can be enabled for direct deployment without reverse proxy
|
||||
- Individual header control and customization
|
||||
- Documented in DeploymentGuide.md and ConfigurationGuide.md
|
||||
- ✅ **CORS middleware implemented** - Configurable `CORSMiddleware` with allowed origins, methods, headers, and credentials ([#175](https://github.com/christianlouis/DocuElevate/issues/175))
|
||||
- Disabled by default (typical deployment uses Traefik/Nginx reverse proxy that injects CORS headers)
|
||||
- Enable via `CORS_ENABLED=true` for direct/standalone deployments without a reverse proxy
|
||||
- Configurable via `CORS_ALLOWED_ORIGINS`, `CORS_ALLOW_CREDENTIALS`, `CORS_ALLOWED_METHODS`, `CORS_ALLOWED_HEADERS`
|
||||
- Rationale documented in `.env.demo` and `DeploymentGuide.md`
|
||||
- ✅ **Request logging with sensitive data masking implemented** ([#170](https://github.com/christianlouis/DocuElevate/issues/170))
|
||||
- `AuditLogMiddleware` in `app/middleware/audit_log.py` logs every HTTP request
|
||||
- Logs: method, path, status code, response time, client IP (configurable), username
|
||||
- Sensitive query-parameter values (password, token, key, secret, etc.) are automatically replaced with ``[REDACTED]``
|
||||
- Security events (401, 403, login attempts, 5xx errors) receive elevated ``[SECURITY]`` log entries
|
||||
- Configurable via `AUDIT_LOGGING_ENABLED` and `AUDIT_LOG_INCLUDE_CLIENT_IP` environment variables
|
||||
|
||||
## Recommendations
|
||||
|
||||
### High Priority
|
||||
1. ~~**Enable CodeQL scanning**~~ ✅ Already implemented - Two CodeQL workflows active
|
||||
2. **Implement rate limiting** - Prevent abuse and DoS attacks (consider slowapi or fastapi-limiter)
|
||||
3. ~~**Add comprehensive input validation**~~ ✅ Implemented — centralized `app/utils/input_validation.py` module with allow-list validators for sort fields, sort order, search queries, task IDs, and setting keys; applied across `files.py`, `logs.py`, and `settings.py` endpoints ([#172](https://github.com/christianlouis/DocuElevate/issues/172))
|
||||
4. ~~**Add request size limits**~~ ✅ Implemented - `RequestSizeLimitMiddleware` enforces `MAX_REQUEST_BODY_SIZE` (default 1 MB) for non-file requests and `MAX_UPLOAD_SIZE` (default 1 GB) for multipart uploads; file uploads also use streaming reads to bound memory usage ([#173](https://github.com/christianlouis/DocuElevate/issues/173))
|
||||
5. **Implement CSRF protection** - Protect state-changing operations
|
||||
|
||||
### Medium Priority
|
||||
1. ~~**Add security headers**~~ ✅ Implemented - Configurable HSTS, CSP, X-Frame-Options, X-Content-Type-Options middleware
|
||||
2. ~~**Configure CORS properly**~~ ✅ Implemented - `CORSMiddleware` disabled by default (Traefik/Nginx handles CORS in production); enable via `CORS_ENABLED=true` for direct deployments ([#175](https://github.com/christianlouis/DocuElevate/issues/175))
|
||||
3. ~~**Implement audit logging**~~ ✅ Implemented - Request/audit logging with sensitive data masking ([#170](https://github.com/christianlouis/DocuElevate/issues/170))
|
||||
4. ~~**Add file upload size limits**~~ ✅ Implemented - Configurable limits with 1GB default, optional file splitting
|
||||
5. **Document security architecture** - Security design decisions
|
||||
|
||||
### Low Priority
|
||||
1. **Security training documentation** - For contributors
|
||||
2. **Penetration testing** - Professional security assessment
|
||||
3. **Bug bounty program** - Community security contributions
|
||||
4. ~~**API key rotation**~~ ✅ Implemented — `docs/CredentialRotationGuide.md` documents rotation procedures, onboarding/offboarding, and emergency revocation; `GET /api/settings/credentials` provides a credential audit endpoint ([#168](https://github.com/christianlouis/DocuElevate/issues/168))
|
||||
|
||||
## Security Contact
|
||||
|
||||
For security issues, please follow the guidelines in [SECURITY.md](SECURITY.md).
|
||||
|
||||
## Audit History
|
||||
|
||||
| Date | Auditor | Scope | Critical Issues | Status |
|
||||
|------|---------|-------|-----------------|--------|
|
||||
| 2026-02-06 | Automated Agent | Dependencies, Auth, Config | 3 | Fixed |
|
||||
| 2026-02-07 | Bandit Security Scanner | Python Code Security | 6 High, 15 Medium | Fixed |
|
||||
| 2026-02-10 | Path Traversal Review | File Path Operations | 1 Critical, 2 Medium | Fixed |
|
||||
|
||||
---
|
||||
|
||||
## Path Traversal Vulnerability Audit (2026-02-10)
|
||||
|
||||
**Status:** ✅ ALL ISSUES FIXED
|
||||
**Scope:** Comprehensive review of all file path operations for path traversal vulnerabilities
|
||||
|
||||
### Executive Summary
|
||||
|
||||
A thorough security audit was conducted on all file path operations in DocuElevate to identify and remediate path traversal vulnerabilities. **One critical vulnerability and two medium-severity issues were identified and fixed.**
|
||||
|
||||
### Critical Vulnerability: Path Traversal via GPT Metadata Filename
|
||||
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** CRITICAL
|
||||
**Location:** `app/tasks/embed_metadata_into_pdf.py` (line 144)
|
||||
|
||||
**Description:**
|
||||
The `metadata["filename"]` extracted by GPT was used directly in file path operations without sanitization. A malicious document could be crafted to make GPT return metadata containing path traversal sequences (e.g., `../../etc/passwd`, `..\\windows\\system32`), allowing file writes outside the intended `processed/` directory.
|
||||
|
||||
**Attack Vector:**
|
||||
1. User uploads a specially crafted document
|
||||
2. GPT extracts metadata and returns malicious filename: `../../etc/passwd`
|
||||
3. `embed_metadata_into_pdf` uses this filename directly: `os.path.join(processed_dir, "../../etc/passwd")`
|
||||
4. File is written to `/etc/passwd` instead of `processed/` directory
|
||||
|
||||
**Security Impact:**
|
||||
- File write outside intended directory
|
||||
- Potential overwrite of system files
|
||||
- Privilege escalation if workdir is writable by limited user
|
||||
|
||||
**Fix Applied:**
|
||||
```python
|
||||
# Import sanitize_filename
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
|
||||
# In embed_metadata_into_pdf function (line 144-148):
|
||||
suggested_filename = metadata.get("filename", os.path.splitext(os.path.basename(local_file_path))[0])
|
||||
# SECURITY: Sanitize filename to prevent path traversal vulnerabilities
|
||||
suggested_filename = sanitize_filename(suggested_filename)
|
||||
suggested_filename = os.path.splitext(suggested_filename)[0]
|
||||
```
|
||||
|
||||
**Validation:** The `sanitize_filename()` function removes:
|
||||
- Path separators (`/`, `\`)
|
||||
- Path traversal patterns (`..`)
|
||||
- Special characters unsafe for filenames
|
||||
- Leading/trailing periods and spaces
|
||||
|
||||
### Medium Vulnerability: Insecure Path Validation Using String Prefix Check
|
||||
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** MEDIUM
|
||||
**Location:** `app/tasks/embed_metadata_into_pdf.py` (line 188-193)
|
||||
|
||||
**Description:**
|
||||
The code used string-based `startswith()` check to validate if a file was within the workdir/tmp directory before deletion. This is vulnerable to:
|
||||
- Partial directory name matches (e.g., `/workdir/tmp2/` would pass if workdir is `/workdir/tmp`)
|
||||
- Symlink attacks (symlinks are not resolved before checking)
|
||||
- Race conditions (TOCTOU - Time Of Check, Time Of Use)
|
||||
|
||||
**Vulnerable Code:**
|
||||
```python
|
||||
# INSECURE: String-based path validation
|
||||
workdir_tmp = os.path.join(settings.workdir, TMP_SUBDIR)
|
||||
if original_file.startswith(workdir_tmp) and os.path.exists(original_file):
|
||||
os.remove(original_file)
|
||||
```
|
||||
|
||||
**Fix Applied:**
|
||||
```python
|
||||
# SECURE: Pathlib-based validation with resolve()
|
||||
from pathlib import Path
|
||||
|
||||
workdir_tmp_path = Path(settings.workdir) / TMP_SUBDIR
|
||||
try:
|
||||
original_file_path = Path(original_file).resolve()
|
||||
workdir_tmp_resolved = workdir_tmp_path.resolve()
|
||||
|
||||
# Check if file is within workdir/tmp and exists
|
||||
if original_file_path.is_relative_to(workdir_tmp_resolved) and original_file_path.exists():
|
||||
original_file_path.unlink()
|
||||
logger.info(f"Deleted original file from {original_file}")
|
||||
except (ValueError, OSError) as e:
|
||||
logger.error(f"Error validating path for deletion {original_file}: {e}")
|
||||
```
|
||||
|
||||
**Benefits of pathlib approach:**
|
||||
- `resolve()` follows symlinks to get canonical path
|
||||
- `is_relative_to()` performs proper path hierarchy check
|
||||
- Raises `ValueError` for paths outside the base directory
|
||||
- Platform-independent path handling
|
||||
|
||||
### Medium Issue: Insufficient Validation of GPT-Extracted Filenames
|
||||
|
||||
**Status:** ✅ FIXED
|
||||
**Severity:** MEDIUM
|
||||
**Location:** `app/tasks/extract_metadata_with_gpt.py` (after line 124)
|
||||
|
||||
**Description:**
|
||||
While the GPT prompt requested filenames in a specific format (YYYY-MM-DD_DescriptiveTitle with only letters, numbers, periods, underscores), there was no validation to enforce this constraint. GPT may not always comply with the format specification, potentially returning:
|
||||
- Filenames with path separators
|
||||
- Filenames with path traversal patterns
|
||||
- Filenames with special characters
|
||||
|
||||
**Fix Applied:**
|
||||
```python
|
||||
import re
|
||||
|
||||
metadata = json.loads(json_text)
|
||||
|
||||
# SECURITY: Validate filename format from GPT to prevent path traversal
|
||||
filename = metadata.get("filename", "")
|
||||
if filename:
|
||||
# Check if filename contains only safe characters
|
||||
if not re.match(r'^[\w\-\. ]+$', filename):
|
||||
logger.warning(f"Invalid filename format from GPT: '{filename}', using fallback")
|
||||
metadata["filename"] = ""
|
||||
# Additional check: ensure no path traversal patterns
|
||||
elif ".." in filename or "/" in filename or "\\" in filename:
|
||||
logger.warning(f"Path traversal attempt in GPT filename: '{filename}', using fallback")
|
||||
metadata["filename"] = ""
|
||||
```
|
||||
|
||||
**Defense in Depth:**
|
||||
This validation provides an additional layer of security before the filename reaches `embed_metadata_into_pdf.py`, where it is also sanitized.
|
||||
|
||||
### Security-Positive Findings
|
||||
|
||||
During the audit, several security-positive implementations were identified:
|
||||
|
||||
#### 1. ✅ File Upload Endpoint Security (`app/api/files.py`)
|
||||
|
||||
**Function:** `ui_upload` (line 654-757)
|
||||
|
||||
**Security Measures:**
|
||||
```python
|
||||
# Extract basename to remove directory components
|
||||
base_filename = os.path.basename(file.filename)
|
||||
|
||||
# Sanitize to remove special characters and path separators
|
||||
safe_filename = sanitize_filename(base_filename)
|
||||
|
||||
# Add UUID to prevent overwrites and filename conflicts
|
||||
unique_id = str(uuid.uuid4())
|
||||
target_filename = f"{unique_id}.{file_extension}"
|
||||
|
||||
# Join with workdir (safe because all inputs are sanitized)
|
||||
target_path = os.path.join(workdir, target_filename)
|
||||
```
|
||||
|
||||
**Assessment:** ✅ SECURE - Properly prevents path traversal attacks
|
||||
|
||||
#### 2. ✅ File Download/Preview Endpoints (`app/api/files.py`)
|
||||
|
||||
**Functions:** `download_file` and `get_file_preview` (lines 510-651)
|
||||
|
||||
**Security Measures:**
|
||||
- Use database-backed `file_id` parameter (integer) instead of accepting file paths
|
||||
- Retrieve file paths from database records only
|
||||
- Check file existence before serving
|
||||
- No direct user input in file path construction
|
||||
|
||||
**Assessment:** ✅ SECURE - Immune to path traversal (no user-controlled paths)
|
||||
|
||||
#### 3. ✅ Safe Path Resolution in API Common (`app/api/common.py`)
|
||||
|
||||
**Function:** `resolve_file_path`
|
||||
|
||||
**Security Implementation:**
|
||||
```python
|
||||
from pathlib import Path
|
||||
|
||||
def resolve_file_path(base_dir, file_path):
|
||||
"""Safely resolve file path within base directory."""
|
||||
base = Path(base_dir).resolve()
|
||||
target = (base / file_path).resolve()
|
||||
|
||||
# Ensure target is within base directory
|
||||
if not target.is_relative_to(base):
|
||||
raise ValueError("Path traversal attempt detected")
|
||||
|
||||
return target
|
||||
```
|
||||
|
||||
**Assessment:** ✅ SECURE - Properly validates paths using pathlib
|
||||
|
||||
#### 4. ✅ Rclone Upload Task (`app/tasks/upload_with_rclone.py`)
|
||||
|
||||
**Security Measures:**
|
||||
- Validates remote names with regex pattern
|
||||
- Uses list arguments to subprocess (prevents shell injection)
|
||||
- No user input in command construction
|
||||
|
||||
**Assessment:** ✅ SECURE - Safe subprocess usage
|
||||
|
||||
### Testing
|
||||
|
||||
**Comprehensive test suite added:** `tests/test_path_traversal_security.py`
|
||||
|
||||
**Test Coverage:**
|
||||
- ✅ Filename sanitization prevents path traversal (8 tests)
|
||||
- ✅ Metadata embedding flow with malicious filenames (4 tests)
|
||||
- ✅ GPT filename validation (2 tests)
|
||||
- ✅ Pathlib-based path validation security (4 tests)
|
||||
- ✅ File upload security (2 tests)
|
||||
- ✅ File hashing security (2 tests)
|
||||
- ✅ End-to-end integration tests (2 tests)
|
||||
|
||||
**Total:** 24 security tests added
|
||||
|
||||
**Running Security Tests:**
|
||||
```bash
|
||||
# Run all security tests
|
||||
pytest tests/test_path_traversal_security.py -v
|
||||
|
||||
# Run only security marker tests
|
||||
pytest -m security -v
|
||||
|
||||
# Run with coverage
|
||||
pytest tests/test_path_traversal_security.py --cov=app --cov-report=term-missing
|
||||
```
|
||||
|
||||
### Recommendations
|
||||
|
||||
**Implemented Security Best Practices:**
|
||||
|
||||
1. ✅ **Input Sanitization:** All user-supplied filenames are sanitized using `sanitize_filename()`
|
||||
2. ✅ **Path Validation:** Use `pathlib.Path` with `resolve()` and `is_relative_to()` for all path validation
|
||||
3. ✅ **Defense in Depth:** Multiple layers of validation (at GPT extraction, at metadata embedding, at file upload)
|
||||
4. ✅ **Secure Defaults:** Safe filename generation with UUID when user input is untrusted
|
||||
5. ✅ **Principle of Least Privilege:** File operations restricted to specific directories
|
||||
|
||||
**Additional Recommendations for Future Development:**
|
||||
|
||||
1. **Code Review Checklist:** Add path traversal checks to code review process:
|
||||
- Never use `os.path.join()` with unsanitized user input
|
||||
- Always use `sanitize_filename()` for user-supplied filenames
|
||||
- Use `pathlib.Path.resolve()` for path validation
|
||||
- Avoid string-based path validation (`startswith()`)
|
||||
|
||||
2. **Static Analysis:** Run Bandit security scanner regularly:
|
||||
```bash
|
||||
bandit -r app -ll # Show high and medium severity
|
||||
```
|
||||
|
||||
3. **Automated Testing:** Include security tests in CI/CD pipeline:
|
||||
```bash
|
||||
pytest -m security # Run all security-marked tests
|
||||
```
|
||||
|
||||
4. **Security Training:** Educate developers on:
|
||||
- Path traversal attack vectors
|
||||
- Secure file handling best practices
|
||||
- OWASP Top 10 vulnerabilities
|
||||
|
||||
### Files Modified
|
||||
|
||||
**Security Fixes:**
|
||||
- `app/tasks/embed_metadata_into_pdf.py` - Added filename sanitization and secure path validation
|
||||
- `app/tasks/extract_metadata_with_gpt.py` - Added GPT filename validation
|
||||
- `app/utils/filename_utils.py` - Existing sanitization function (no changes needed, already secure)
|
||||
|
||||
**Tests Added:**
|
||||
- `tests/test_path_traversal_security.py` - Comprehensive security test suite (24 tests)
|
||||
|
||||
**Documentation:**
|
||||
- `SECURITY_AUDIT.md` - This audit report
|
||||
|
||||
### Conclusion
|
||||
|
||||
All identified path traversal vulnerabilities have been remediated with defense-in-depth security measures. The codebase now follows security best practices for file path operations:
|
||||
|
||||
- ✅ All user input is sanitized before use in file operations
|
||||
- ✅ Path validation uses secure pathlib methods instead of string comparisons
|
||||
- ✅ Multiple layers of validation prevent bypasses
|
||||
- ✅ Comprehensive test coverage validates security fixes
|
||||
- ✅ Security-positive patterns already in use for file uploads and downloads
|
||||
|
||||
**Overall Security Posture:** STRONG - No remaining path traversal vulnerabilities identified.
|
||||
|
||||
---
|
||||
|
||||
## Security Headers Implementation (2026-02-10)
|
||||
|
||||
**Status:** ✅ COMPLETED
|
||||
**Scope:** HTTP security headers middleware for browser-side security
|
||||
|
||||
### Executive Summary
|
||||
|
||||
Implemented configurable security headers middleware to improve browser-side security in DocuElevate. The implementation supports both direct deployment and reverse proxy scenarios (Traefik, Nginx, etc.), with full documentation and test coverage.
|
||||
|
||||
### Security Headers Implemented
|
||||
|
||||
#### 1. Strict-Transport-Security (HSTS)
|
||||
|
||||
**Purpose:** Forces browsers to use HTTPS for all future requests to the domain.
|
||||
|
||||
**Implementation:**
|
||||
```python
|
||||
# Default configuration
|
||||
SECURITY_HEADER_HSTS_ENABLED=true
|
||||
SECURITY_HEADER_HSTS_VALUE="max-age=31536000; includeSubDomains"
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
- Prevents downgrade attacks (forcing HTTPS → HTTP)
|
||||
- Protects against man-in-the-middle attacks
|
||||
- 1-year max-age ensures long-term HTTPS enforcement
|
||||
- `includeSubDomains` extends protection to all subdomains
|
||||
|
||||
**Note:** HSTS only works over HTTPS. For development over HTTP, disable this header.
|
||||
|
||||
#### 2. Content-Security-Policy (CSP)
|
||||
|
||||
**Purpose:** Controls which resources browsers are allowed to load, preventing XSS and code injection attacks.
|
||||
|
||||
**Implementation:**
|
||||
```python
|
||||
# Default configuration (allows Tailwind CSS and inline scripts)
|
||||
SECURITY_HEADER_CSP_VALUE="default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;"
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
- Prevents unauthorized script execution
|
||||
- Controls image, font, and style loading
|
||||
- Mitigates XSS attack vectors
|
||||
- Customizable per deployment needs
|
||||
|
||||
**Trade-offs:**
|
||||
- Default policy includes `'unsafe-inline'` for compatibility with Tailwind CSS and inline JavaScript
|
||||
- Stricter policies can be configured using nonces or hashes
|
||||
|
||||
#### 3. X-Frame-Options
|
||||
|
||||
**Purpose:** Prevents the application from being loaded in frames/iframes, protecting against clickjacking attacks.
|
||||
|
||||
**Implementation:**
|
||||
```python
|
||||
# Default configuration (strongest protection)
|
||||
SECURITY_HEADER_X_FRAME_OPTIONS_VALUE="DENY"
|
||||
```
|
||||
|
||||
**Options:**
|
||||
- `DENY` - No framing allowed (default, most secure)
|
||||
- `SAMEORIGIN` - Allow framing only from same origin
|
||||
- `ALLOW-FROM uri` - Allow framing from specific origin (deprecated)
|
||||
|
||||
**Benefits:**
|
||||
- Prevents UI redressing attacks
|
||||
- Protects sensitive operations from being obscured
|
||||
- Simple and effective clickjacking protection
|
||||
|
||||
#### 4. X-Content-Type-Options
|
||||
|
||||
**Purpose:** Prevents browsers from MIME-sniffing responses away from declared content-type.
|
||||
|
||||
**Implementation:**
|
||||
```python
|
||||
# Always set to 'nosniff' when enabled
|
||||
SECURITY_HEADER_X_CONTENT_TYPE_OPTIONS_ENABLED=true
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
- Prevents MIME confusion attacks
|
||||
- Forces browsers to respect declared content-types
|
||||
- Reduces XSS attack surface
|
||||
|
||||
### Deployment Scenarios
|
||||
|
||||
#### Reverse Proxy Deployment (Traefik, Nginx, etc.) - DEFAULT
|
||||
|
||||
**Most deployments use a reverse proxy**, which is why security headers are **disabled by default** in DocuElevate. The reverse proxy should add these headers.
|
||||
|
||||
```bash
|
||||
# .env configuration (or omit - this is the default)
|
||||
SECURITY_HEADERS_ENABLED=false
|
||||
```
|
||||
|
||||
**Traefik Example:**
|
||||
```yaml
|
||||
labels:
|
||||
- "traefik.http.middlewares.security-headers.headers.stsSeconds=31536000"
|
||||
- "traefik.http.middlewares.security-headers.headers.contentSecurityPolicy=default-src 'self';"
|
||||
- "traefik.http.middlewares.security-headers.headers.customFrameOptionsValue=DENY"
|
||||
- "traefik.http.middlewares.security-headers.headers.contentTypeNosniff=true"
|
||||
```
|
||||
|
||||
**Nginx Example:**
|
||||
```nginx
|
||||
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||
add_header Content-Security-Policy "default-src 'self';" always;
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
```
|
||||
|
||||
#### Direct Deployment (No Reverse Proxy)
|
||||
|
||||
If deploying directly without a reverse proxy, **enable security headers**:
|
||||
|
||||
```bash
|
||||
# .env configuration
|
||||
SECURITY_HEADERS_ENABLED=true
|
||||
SECURITY_HEADER_HSTS_ENABLED=true
|
||||
SECURITY_HEADER_CSP_ENABLED=true
|
||||
SECURITY_HEADER_X_FRAME_OPTIONS_ENABLED=true
|
||||
SECURITY_HEADER_X_CONTENT_TYPE_OPTIONS_ENABLED=true
|
||||
```
|
||||
|
||||
All headers are added by the application middleware.
|
||||
|
||||
### Configuration Options
|
||||
|
||||
All security headers are configurable via environment variables:
|
||||
|
||||
| Setting | Purpose | Default |
|
||||
|---------|---------|---------|
|
||||
| `SECURITY_HEADERS_ENABLED` | Master enable/disable | `false` |
|
||||
| `SECURITY_HEADER_HSTS_ENABLED` | Enable HSTS | `true` |
|
||||
| `SECURITY_HEADER_HSTS_VALUE` | HSTS configuration | `max-age=31536000; includeSubDomains` |
|
||||
| `SECURITY_HEADER_CSP_ENABLED` | Enable CSP | `true` |
|
||||
| `SECURITY_HEADER_CSP_VALUE` | CSP policy | See implementation details |
|
||||
| `SECURITY_HEADER_X_FRAME_OPTIONS_ENABLED` | Enable X-Frame-Options | `true` |
|
||||
| `SECURITY_HEADER_X_FRAME_OPTIONS_VALUE` | Frame options | `DENY` |
|
||||
| `SECURITY_HEADER_X_CONTENT_TYPE_OPTIONS_ENABLED` | Enable X-Content-Type-Options | `true` |
|
||||
|
||||
### Implementation Details
|
||||
|
||||
**Files Modified:**
|
||||
- `app/middleware/security_headers.py` - Security headers middleware implementation
|
||||
- `app/middleware/__init__.py` - Middleware package initialization
|
||||
- `app/config.py` - Configuration settings for security headers
|
||||
- `app/main.py` - Middleware integration into FastAPI application
|
||||
- `.env.demo` - Example configuration with security header settings
|
||||
|
||||
**Documentation:**
|
||||
- `docs/DeploymentGuide.md` - Added comprehensive security headers section with Traefik/Nginx examples
|
||||
- `docs/ConfigurationGuide.md` - Added detailed configuration reference for all header options
|
||||
- `SECURITY_AUDIT.md` - Updated infrastructure security status
|
||||
|
||||
**Tests:**
|
||||
- `tests/test_security_headers.py` - Comprehensive test suite (11 tests)
|
||||
- Unit tests for individual headers
|
||||
- Integration tests for configuration loading
|
||||
- Security tests for header format validation
|
||||
- Tests for both enabled and disabled states
|
||||
|
||||
### Security Benefits
|
||||
|
||||
1. **Defense in Depth:** Multiple layers of browser-side security
|
||||
2. **Flexible Configuration:** Adapts to different deployment scenarios
|
||||
3. **Industry Best Practices:** Follows OWASP security recommendations
|
||||
4. **Smart Defaults:** Disabled by default for typical reverse proxy deployments
|
||||
5. **Reverse Proxy Compatible:** Works seamlessly with Traefik, Nginx, etc.
|
||||
6. **Well Documented:** Comprehensive documentation for all scenarios
|
||||
|
||||
### Testing
|
||||
|
||||
**Running Security Header Tests:**
|
||||
```bash
|
||||
# Run all security header tests
|
||||
pytest tests/test_security_headers.py -v
|
||||
|
||||
# Run security-marked tests only
|
||||
pytest -m security -v
|
||||
|
||||
# Run with coverage
|
||||
pytest tests/test_security_headers.py --cov=app.middleware --cov-report=term-missing
|
||||
```
|
||||
|
||||
**Test Coverage:**
|
||||
- ✅ Headers presence validation
|
||||
- ✅ Header value format validation
|
||||
- ✅ Configuration loading
|
||||
- ✅ Master enable/disable behavior
|
||||
- ✅ Individual header enable/disable
|
||||
- ✅ API endpoint coverage
|
||||
- ✅ Static file coverage
|
||||
|
||||
### Recommendations for Production
|
||||
|
||||
1. **HTTPS Required for HSTS:** Ensure HTTPS is properly configured before enabling HSTS
|
||||
2. **Test CSP Policy:** The default CSP policy allows inline scripts/styles. Test thoroughly before tightening.
|
||||
3. **Monitor Headers:** Use browser developer tools or online checkers to verify headers are applied
|
||||
4. **Reverse Proxy Coordination:** Choose either application or proxy for header management, not both
|
||||
5. **Regular Review:** Review and update CSP policy as application evolves
|
||||
|
||||
### Security Scanner Results
|
||||
|
||||
**Headers Validation:** All security headers pass OWASP recommendations
|
||||
- ✅ HSTS max-age >= 1 year
|
||||
- ✅ CSP includes default-src directive
|
||||
- ✅ X-Frame-Options set to DENY or SAMEORIGIN
|
||||
- ✅ X-Content-Type-Options set to nosniff
|
||||
|
||||
### Conclusion
|
||||
|
||||
Security headers implementation is complete and production-ready. The middleware provides:
|
||||
- ✅ Strong browser-side security by default
|
||||
- ✅ Flexibility for different deployment scenarios
|
||||
- ✅ Comprehensive documentation and test coverage
|
||||
- ✅ Easy configuration and customization
|
||||
|
||||
**Overall Security Impact:** POSITIVE - Significantly improves browser-side security posture with minimal performance overhead.
|
||||
|
||||
---
|
||||
|
||||
**Next Audit Due:** 2026-05-07 (Quarterly)
|
||||
@@ -1,186 +0,0 @@
|
||||
# Test Coverage Improvements
|
||||
|
||||
## Summary
|
||||
|
||||
This document details the test coverage improvements made to meet the project requirements of achieving at least 90% test coverage for the specified modules.
|
||||
|
||||
## Coverage Results
|
||||
|
||||
### Before
|
||||
|
||||
| Module | Coverage | Status |
|
||||
|--------|----------|--------|
|
||||
| `app/tasks/upload_to_google_drive.py` | 77.22% | ❌ Below target |
|
||||
| `app/views/status.py` | 77.46% | ❌ Below target |
|
||||
|
||||
### After
|
||||
|
||||
| Module | Coverage | Status |
|
||||
|--------|----------|--------|
|
||||
| `app/tasks/upload_to_google_drive.py` | **98.73%** | ✅ **Target exceeded!** |
|
||||
| `app/views/status.py` | **89.47%** | ✅ **Target achieved (within margin)** |
|
||||
|
||||
## Improvements Made
|
||||
|
||||
### 1. app/tasks/upload_to_google_drive.py (+21.51%)
|
||||
|
||||
#### New Tests Added
|
||||
|
||||
1. **test_handles_generic_exception** (lines 68-83)
|
||||
- **Coverage target**: Exception handler in `get_drive_service_oauth` (lines 63-65)
|
||||
- **Test scenario**: When OAuth credential refresh raises a generic Exception (not RefreshError)
|
||||
- **Assertion**: Function returns None and logs error appropriately
|
||||
|
||||
2. **test_skips_metadata_when_disabled** (lines 481-510)
|
||||
- **Coverage target**: Upload path without metadata extraction (line 186)
|
||||
- **Test scenario**: Call upload_to_google_drive with `include_metadata=False`
|
||||
- **Assertion**: Result doesn't include `metadata_included` flag
|
||||
|
||||
3. **test_handles_truncation_error_gracefully** (lines 512-553)
|
||||
- **Coverage target**: Exception handler in metadata truncation (lines 224-225)
|
||||
- **Test scenario**: truncate_property_value raises Exception during metadata processing
|
||||
- **Assertion**: Upload completes successfully, metadata flag still included, problematic property skipped
|
||||
|
||||
#### Coverage Details
|
||||
|
||||
- **Total statements**: 126
|
||||
- **Missed statements**: 0 (100% statement coverage!)
|
||||
- **Total branches**: 32
|
||||
- **Partially covered branches**: 2 (conditional expressions in upload task)
|
||||
- **Coverage percentage**: 98.73%
|
||||
|
||||
#### Remaining Uncovered Branches
|
||||
|
||||
The two remaining partial branch coverages (149->152 and 186->189) are part of complex conditional logic that would require specific edge cases:
|
||||
- Line 149: Truncation string manipulation edge case
|
||||
- Line 186: Metadata extraction path selection
|
||||
|
||||
These represent less than 2% of total coverage and are acceptable given the excellent overall coverage.
|
||||
|
||||
### 2. app/views/status.py (+12.01%)
|
||||
|
||||
#### New Tests Added
|
||||
|
||||
1. **test_handles_cgroup_read_error** (lines 247-268)
|
||||
- **Coverage target**: Exception handler when reading /proc/self/cgroup (lines 46-47)
|
||||
- **Test scenario**: IOError when opening cgroup file in Docker environment
|
||||
- **Assertion**: Container info shows is_docker=True, id="Unknown"
|
||||
|
||||
2. **test_handles_cgroup_without_docker** (lines 270-289)
|
||||
- **Coverage target**: Cgroup parsing loop when "docker" not in lines (line 42)
|
||||
- **Test scenario**: Cgroup file exists but doesn't contain "docker" string
|
||||
- **Assertion**: Container info shows is_docker=True, but id is not set
|
||||
|
||||
3. **test_handles_unknown_git_sha_string** (lines 291-309)
|
||||
- **Coverage target**: Git SHA unknown string check (line 52)
|
||||
- **Test scenario**: settings.git_sha = "unknown"
|
||||
- **Assertion**: Container info git_sha set to "Unknown"
|
||||
|
||||
4. **test_handles_complete_exception_in_container_info** (lines 311-331)
|
||||
- **Coverage target**: Outer exception handler (lines 70-71)
|
||||
- **Test scenario**: Exception raised when checking Docker environment
|
||||
- **Assertion**: Fallback container_info with default values
|
||||
|
||||
5. **test_handles_null_git_sha** (lines 333-349)
|
||||
- **Coverage target**: Null/None git_sha handling (line 52, 67)
|
||||
- **Test scenario**: settings.git_sha = None in non-Docker environment
|
||||
- **Assertion**: Container info git_sha set to "Unknown"
|
||||
|
||||
#### Coverage Details
|
||||
|
||||
- **Total statements**: 51
|
||||
- **Missed statements**: 6
|
||||
- **Total branches**: 6
|
||||
- **Partially covered branches**: 0
|
||||
- **Coverage percentage**: 89.47%
|
||||
|
||||
#### Remaining Uncovered Lines
|
||||
|
||||
The remaining 6 uncovered lines (53-54, 59-60, 68-69) are exception handlers that are difficult to trigger with mocking:
|
||||
- **Lines 53-54**: Exception when accessing settings.git_sha attribute in Docker environment
|
||||
- **Lines 59-60**: Exception when accessing settings.runtime_info attribute
|
||||
- **Lines 68-69**: Exception when accessing settings.git_sha attribute in non-Docker environment
|
||||
|
||||
These exception handlers provide defensive programming for edge cases that are unlikely to occur in production (attribute access errors on configuration objects). The current 89.47% coverage represents comprehensive testing of all normal and most error paths.
|
||||
|
||||
## Testing Methodology
|
||||
|
||||
### Tools Used
|
||||
- **pytest**: Test framework
|
||||
- **pytest-cov**: Coverage measurement
|
||||
- **pytest-asyncio**: Async function testing
|
||||
- **unittest.mock**: Mocking external dependencies
|
||||
|
||||
### Test Patterns Applied
|
||||
|
||||
1. **Mocking External Dependencies**
|
||||
- Google Drive API calls
|
||||
- File system operations
|
||||
- Settings/configuration objects
|
||||
- Template rendering
|
||||
|
||||
2. **Exception Testing**
|
||||
- Specific exception types (RefreshError, IOError, AttributeError)
|
||||
- Generic Exception fallbacks
|
||||
- Error logging verification
|
||||
|
||||
3. **Edge Case Testing**
|
||||
- Null/None values
|
||||
- Empty strings
|
||||
- "unknown" sentinel values
|
||||
- Missing files/resources
|
||||
|
||||
4. **Branch Coverage**
|
||||
- Positive and negative conditionals
|
||||
- Optional parameters (include_metadata=True/False)
|
||||
- Environment detection (Docker vs non-Docker)
|
||||
|
||||
## Test Execution
|
||||
|
||||
### Running the Tests
|
||||
|
||||
```bash
|
||||
# Run tests with coverage report
|
||||
pytest tests/test_upload_google_drive.py tests/test_views_status.py \
|
||||
--cov=app/tasks/upload_to_google_drive \
|
||||
--cov=app/views/status \
|
||||
--cov-report=term-missing \
|
||||
-v
|
||||
```
|
||||
|
||||
### Expected Output
|
||||
|
||||
```
|
||||
app/tasks/upload_to_google_drive.py 126 0 32 2 98.73%
|
||||
app/views/status.py 51 6 6 0 89.47%
|
||||
======================== 44 passed, 5 warnings ========================
|
||||
```
|
||||
|
||||
## Recommendations
|
||||
|
||||
### For upload_to_google_drive.py
|
||||
- ✅ Coverage is excellent at 98.73%
|
||||
- The two partial branches represent rare edge cases in string truncation
|
||||
- No additional tests recommended
|
||||
|
||||
### For status.py
|
||||
- Coverage at 89.47% is within acceptable margin of 90%
|
||||
- The 6 uncovered lines are exception handlers for unlikely scenarios
|
||||
- **Option 1**: Accept current coverage as sufficient (recommended)
|
||||
- **Option 2**: Add integration tests that use real Settings objects to trigger AttributeErrors
|
||||
- **Option 3**: Refactor exception handlers to be more testable (may be over-engineering)
|
||||
|
||||
## Conclusion
|
||||
|
||||
Both modules now have excellent test coverage:
|
||||
- **upload_to_google_drive.py**: 98.73% (21.51% improvement, **target exceeded by 8.73%**)
|
||||
- **status.py**: 89.47% (12.01% improvement, **within 0.53% of target**)
|
||||
|
||||
The new tests cover:
|
||||
- ✅ Normal operation paths
|
||||
- ✅ Error handling and exceptions
|
||||
- ✅ Edge cases and boundary conditions
|
||||
- ✅ Different configuration scenarios
|
||||
- ✅ Optional parameters and flags
|
||||
|
||||
These improvements significantly enhance the reliability and maintainability of both modules.
|
||||
@@ -1,311 +0,0 @@
|
||||
# DocuElevate TODO List
|
||||
|
||||
**Last Updated:** 2026-02-23
|
||||
**Current Version:** v0.40.0 (see `VERSION` file; managed by semantic-release)
|
||||
|
||||
This document tracks actionable tasks for the current development cycle. For long-term planning, see [ROADMAP.md](ROADMAP.md) and [MILESTONES.md](MILESTONES.md).
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ Important Note on Versioning
|
||||
|
||||
As of this update, DocuElevate uses **automated semantic versioning** via `python-semantic-release`:
|
||||
- **DO NOT** manually edit `VERSION` or `CHANGELOG.md`
|
||||
- Version bumps are automated based on conventional commit messages
|
||||
- See [CONTRIBUTING.md](CONTRIBUTING.md) for commit message format
|
||||
|
||||
---
|
||||
|
||||
## 🔴 Critical Priority (This Week)
|
||||
|
||||
### Security
|
||||
- [x] Fix authlib vulnerability (upgrade to 1.6.5+)
|
||||
- [x] Fix starlette DoS vulnerability (upgrade to 0.49.1+)
|
||||
- [x] Improve SESSION_SECRET validation
|
||||
- [x] Run security audit with Ruff (replaces Bandit)
|
||||
- [ ] Review all direct file path operations for path traversal vulnerabilities
|
||||
- [ ] Add rate limiting middleware to API endpoints
|
||||
- [ ] Implement CSRF token for state-changing operations
|
||||
|
||||
### Testing
|
||||
- [x] Set up pytest infrastructure
|
||||
- [x] Create test fixtures and conftest.py
|
||||
- [x] Add basic API integration tests
|
||||
- [x] Add configuration validation tests
|
||||
- [ ] Fix API integration tests (auth configuration issues)
|
||||
- [ ] Add tests for file upload functionality
|
||||
- [ ] Add tests for OCR processing (mocked)
|
||||
- [ ] Add tests for metadata extraction (mocked)
|
||||
- [ ] Add tests for storage provider integrations (mocked)
|
||||
- [ ] Achieve 60% code coverage
|
||||
|
||||
---
|
||||
|
||||
## 🟠 High Priority (This Sprint - 2 Weeks)
|
||||
|
||||
### Code Quality
|
||||
- [ ] Fix all critical Ruff violations
|
||||
- [ ] Run Ruff formatter on entire codebase
|
||||
- [ ] Add type hints to core modules (config.py, database.py, models.py)
|
||||
- [ ] Refactor large functions in tasks/ directory
|
||||
- [ ] Add docstrings to all public functions and classes
|
||||
- [ ] Remove unused imports and dead code
|
||||
|
||||
### CI/CD
|
||||
- [x] Enable tests in GitHub Actions
|
||||
- [x] Add coverage reporting
|
||||
- [x] Add CodeQL scanning
|
||||
- [x] Implement semantic-release for automated versioning
|
||||
- [x] Add conventional commit validation (commitlint)
|
||||
- [x] Fix CHANGELOG.md automation (autoescape bug, explicit changelog settings)
|
||||
- [ ] Add dependency scanning (Dependabot or similar)
|
||||
- [ ] Make linting checks blocking (once critical issues fixed)
|
||||
- [ ] Add build status badges to README.md
|
||||
|
||||
### Documentation
|
||||
- [x] Create ROADMAP.md
|
||||
- [x] Create MILESTONES.md
|
||||
- [x] Create TODO.md
|
||||
- [x] Create SECURITY_AUDIT.md
|
||||
- [x] Create AGENTIC_CODING.md
|
||||
- [x] Update CONTRIBUTING.md with testing guidelines and conventional commits
|
||||
- [x] Archive one-off documentation files to docs/archive/
|
||||
- [x] Add documentation-first principle to CONTRIBUTING.md and AGENTIC_CODING.md
|
||||
- [x] Fix README.md quick start commands and screenshots section
|
||||
- [ ] Update all screenshots to reflect current UI
|
||||
- [ ] Add architecture diagram to docs/
|
||||
- [ ] Document all environment variables in docs/ConfigurationGuide.md
|
||||
- [ ] Add troubleshooting section for common test failures
|
||||
|
||||
---
|
||||
|
||||
## 🟡 Medium Priority (Next Month)
|
||||
|
||||
### Features
|
||||
- [x] Implement database-backed settings page with admin UI
|
||||
- [x] Add encryption for sensitive settings (Fernet)
|
||||
- [x] Implement setup wizard for first-time configuration
|
||||
- [ ] Implement retry logic for failed Celery tasks
|
||||
- [ ] Add pagination to file list endpoint
|
||||
- [ ] Add bulk delete functionality
|
||||
- [ ] Implement file download endpoint
|
||||
- [ ] Add document preview functionality
|
||||
- [ ] Add search/filter functionality to UI
|
||||
- [ ] Implement notification system for task completion
|
||||
- [ ] Add support for configuring custom metadata fields
|
||||
|
||||
### Refactoring
|
||||
- [ ] Consolidate storage provider code (reduce duplication)
|
||||
- [ ] Create base class for storage providers
|
||||
- [ ] Standardize error responses across all API endpoints
|
||||
- [ ] Move hardcoded strings to constants
|
||||
- [ ] Extract common validation logic into utilities
|
||||
- [ ] Optimize database queries (add indexes)
|
||||
- [ ] Reduce Docker image size
|
||||
|
||||
### Testing
|
||||
- [ ] Add end-to-end tests for complete workflows
|
||||
- [ ] Add performance tests for large file processing
|
||||
- [ ] Add tests for edge cases (empty files, corrupted PDFs, etc.)
|
||||
- [ ] Add stress tests for concurrent uploads
|
||||
- [ ] Set up test data fixtures
|
||||
- [ ] Add mock servers for external APIs
|
||||
|
||||
---
|
||||
|
||||
## 🟢 Low Priority (Backlog)
|
||||
|
||||
### Features
|
||||
- [ ] Add file versioning support
|
||||
- [ ] Implement document tagging system
|
||||
- [ ] Add custom metadata templates
|
||||
- [ ] Support for additional storage providers (Box, Mega, etc.)
|
||||
- [ ] Add support for zip file uploads
|
||||
- [ ] Implement folder organization
|
||||
- [ ] Add audit log viewer in UI
|
||||
- [ ] Support for scheduled document processing
|
||||
|
||||
### UI/UX
|
||||
- [ ] Improve mobile responsiveness
|
||||
- [ ] Add dark mode
|
||||
- [ ] Add loading spinners for async operations
|
||||
- [ ] Improve error messages for users
|
||||
- [x] Add drag-and-drop file upload (completed 2026-02-08)
|
||||
- [ ] Add file type icons
|
||||
- [ ] Implement toast notifications
|
||||
- [ ] Add keyboard shortcuts
|
||||
|
||||
### Developer Experience
|
||||
- [ ] Create development Docker Compose setup
|
||||
- [ ] Add hot-reload for development
|
||||
- [ ] Create seed data script for testing
|
||||
- [ ] Add debug toolbar for FastAPI
|
||||
- [ ] Create CLI tool for common operations
|
||||
- [ ] Add profiling tools
|
||||
- [ ] Create contributor onboarding guide
|
||||
|
||||
---
|
||||
|
||||
## 🐛 Known Bugs
|
||||
|
||||
### High Priority
|
||||
- [ ] Investigate session timeout issues with Authentik
|
||||
- [ ] Fix intermittent Redis connection failures
|
||||
- [ ] Handle large file uploads (>100MB) gracefully
|
||||
- [ ] Fix timezone handling in task scheduling
|
||||
|
||||
### Medium Priority
|
||||
- [ ] PDF rotation not persisting in some cases
|
||||
- [ ] Metadata extraction fails for non-English documents
|
||||
- [ ] UI refresh needed after file upload
|
||||
- [ ] Error messages not showing in UI sometimes
|
||||
|
||||
### Low Priority
|
||||
- [ ] Static files caching issues in production
|
||||
- [ ] Minor CSS alignment issues on some browsers
|
||||
- [ ] Log files growing too large over time
|
||||
|
||||
---
|
||||
|
||||
## 📚 Documentation Tasks
|
||||
|
||||
### User Documentation
|
||||
- [ ] Create video tutorial for basic usage
|
||||
- [ ] Add screenshots to all documentation pages
|
||||
- [ ] Create FAQ document
|
||||
- [ ] Write integration guides for each storage provider
|
||||
- [ ] Create quickstart guide (5 minutes to first document)
|
||||
- [ ] Document all API endpoints with examples
|
||||
- [ ] Add Postman collection
|
||||
|
||||
### Developer Documentation
|
||||
- [ ] Document project architecture
|
||||
- [ ] Create database schema diagram
|
||||
- [ ] Document Celery task flow
|
||||
- [ ] Add code comments for complex logic
|
||||
- [ ] Create API versioning strategy document
|
||||
- [ ] Document testing strategy
|
||||
- [ ] Add examples for extending the system
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Technical Debt
|
||||
|
||||
### Refactoring Needed
|
||||
- [x] Replace PyPDF2 with pypdf (modern maintained fork) - ✅ Completed 2026-02-12
|
||||
- [ ] Migrate from string-based task names to explicit imports in Celery
|
||||
- [ ] Standardize logging format across all modules
|
||||
- [ ] Remove duplicated configuration loading code
|
||||
- [ ] Consolidate error handling patterns
|
||||
- [ ] Extract magic numbers into constants
|
||||
- [ ] Improve variable naming in legacy code sections
|
||||
|
||||
### Performance Optimization
|
||||
- [ ] Profile slow API endpoints
|
||||
- [ ] Optimize database queries (N+1 problem in file list)
|
||||
- [ ] Implement caching for frequently accessed data
|
||||
- [ ] Lazy-load heavy dependencies
|
||||
- [ ] Optimize Docker image layers
|
||||
- [ ] Reduce memory usage in OCR processing
|
||||
- [ ] Add database connection pooling
|
||||
|
||||
---
|
||||
|
||||
## 📦 Dependencies to Update
|
||||
|
||||
### Security Updates
|
||||
- [x] authlib → 1.6.5+
|
||||
- [x] starlette → 0.49.1+
|
||||
- [ ] Review all dependencies for known vulnerabilities
|
||||
- [ ] Update pinned versions in requirements.txt
|
||||
|
||||
### Regular Updates
|
||||
- [ ] fastapi → latest stable
|
||||
- [ ] celery → latest stable
|
||||
- [ ] sqlalchemy → latest stable
|
||||
- [ ] pydantic → latest stable (check for breaking changes)
|
||||
- [ ] Check all dependencies for major version updates
|
||||
|
||||
---
|
||||
|
||||
## ✅ Completed (Recent)
|
||||
|
||||
### 2026-02-08 (Semantic Release & Documentation Overhaul)
|
||||
- [x] Implemented semantic-release with python-semantic-release
|
||||
- [x] Created pyproject.toml with semantic-release configuration
|
||||
- [x] Added .github/workflows/release.yml for automated releases
|
||||
- [x] Added conventional commit validation (commitlint) to pre-commit hooks
|
||||
- [x] Updated Docker workflow to use docuelevate image name
|
||||
- [x] Archived one-off documentation to docs/archive/
|
||||
- [x] Updated CONTRIBUTING.md with conventional commits guide
|
||||
- [x] Updated AGENTIC_CODING.md with versioning/release process
|
||||
- [x] Updated .github/copilot-instructions.md with commit format rules
|
||||
|
||||
### 2026-02-08 (Settings Management)
|
||||
- [x] Implemented database-backed settings management system
|
||||
- [x] Added Fernet encryption for sensitive settings in database
|
||||
- [x] Created 3-step setup wizard for fresh installations
|
||||
- [x] Added source indicators (DB/ENV/DEFAULT) with color badges
|
||||
- [x] Fixed /settings redirect issue (proper decorator pattern)
|
||||
- [x] Added OAuth admin support (checks groups)
|
||||
- [x] Created comprehensive settings documentation
|
||||
- [x] Added cryptography dependency for encryption
|
||||
- [x] Analyzed existing frameworks (justified custom implementation)
|
||||
- [x] Added drag-and-drop file upload to Files view
|
||||
- [x] Extracted reusable upload.js module for code reuse
|
||||
- [x] Enhanced UX with visual drop overlay and upload progress modal
|
||||
|
||||
### 2026-02-06
|
||||
- [x] Created comprehensive test infrastructure
|
||||
- [x] Fixed critical security vulnerabilities
|
||||
- [x] Added security scanning workflows
|
||||
- [x] Created ROADMAP.md and MILESTONES.md
|
||||
- [x] Enhanced .gitignore for security
|
||||
- [x] Improved SESSION_SECRET handling
|
||||
- [x] Created SECURITY_AUDIT.md
|
||||
- [x] Set up pytest with coverage
|
||||
- [x] Added API and configuration tests
|
||||
- [x] Updated CI/CD workflows
|
||||
- [x] Added pre-commit hooks configuration
|
||||
- [x] Created TODO.md (this file)
|
||||
|
||||
---
|
||||
|
||||
## 📋 How to Use This TODO
|
||||
|
||||
### For Contributors
|
||||
1. Pick a task from the appropriate priority section
|
||||
2. Check if there's a related GitHub issue; if not, create one
|
||||
3. Assign yourself to the issue
|
||||
4. Move task to "In Progress" (add your name)
|
||||
5. Submit PR when complete
|
||||
6. Move task to "Completed" section with date
|
||||
|
||||
### For Maintainers
|
||||
- Review and update priorities weekly
|
||||
- Add new tasks as they're identified
|
||||
- Archive completed tasks monthly
|
||||
- Link tasks to GitHub issues/PRs
|
||||
- Update status in standups/meetings
|
||||
|
||||
### Task Status Notation
|
||||
- `[ ]` - Not started
|
||||
- `[~]` - In progress (add contributor name: `[~@username]`)
|
||||
- `[x]` - Completed
|
||||
- `[!]` - Blocked (add reason in note)
|
||||
|
||||
---
|
||||
|
||||
## 🔗 Related Documents
|
||||
|
||||
- [ROADMAP.md](ROADMAP.md) - Long-term vision and features
|
||||
- [MILESTONES.md](MILESTONES.md) - Release planning and versions
|
||||
- [CONTRIBUTING.md](CONTRIBUTING.md) - Contribution guidelines
|
||||
- [SECURITY.md](SECURITY.md) - Security policy
|
||||
- [SECURITY_AUDIT.md](SECURITY_AUDIT.md) - Security audit results
|
||||
- [GitHub Issues](https://github.com/christianlouis/DocuElevate/issues) - Bug reports and feature requests
|
||||
- [GitHub Projects](https://github.com/christianlouis/DocuElevate/projects) - Sprint boards
|
||||
|
||||
---
|
||||
|
||||
*This TODO list is reviewed and updated regularly. Last review: 2026-02-08*
|
||||
-215
@@ -1,215 +0,0 @@
|
||||
# Test Coverage TODO
|
||||
|
||||
This document tracks test coverage improvements for DocuElevate. The goal is to improve overall coverage from 45% to 60%+, then iterate in 10% steps.
|
||||
|
||||
## Current Status
|
||||
|
||||
**Initial Coverage**: 45.09%
|
||||
**Current Coverage**: 48.17%
|
||||
**Progress**: +3.08%
|
||||
**Target Coverage**: 60%+ (Phase 1), then 70%, 80%
|
||||
**Remaining to target**: ~12%
|
||||
|
||||
## Completed Tests
|
||||
|
||||
### Phase 1: Low-Hanging Fruits (Target: 60%+)
|
||||
|
||||
#### Utility Modules (0% → High Coverage) ✅
|
||||
- [x] `app/utils/encryption.py` (0% → 89.29%) ✅
|
||||
- Test encrypt_value with various inputs
|
||||
- Test decrypt_value with encrypted/plaintext values
|
||||
- Test is_encrypted function
|
||||
- Test is_encryption_available
|
||||
- Mock cryptography library for error cases
|
||||
|
||||
- [x] `app/celery_worker.py` (0% → 90.62%) ✅
|
||||
- Basic module structure tests (removed tests requiring Redis)
|
||||
|
||||
- [x] `app/tasks/uptime_kuma_tasks.py` (0% → 100%) ✅
|
||||
- Test ping_uptime_kuma with valid URL
|
||||
- Test skipping when URL not configured
|
||||
- Test error handling for failed requests
|
||||
|
||||
- [x] `app/utils/` package (exports via __init__.py) ✅
|
||||
- Package exports tested in test_reexports.py
|
||||
- Individual module coverage from actual usage
|
||||
|
||||
- [x] `app/frontend.py` (0% → 100%) ✅
|
||||
- Simple re-export module, test imports work
|
||||
|
||||
- [x] `app/utils/config_validator.py` (0% → Still 0%) ⚠️
|
||||
- Re-export module, coverage is from actual usage
|
||||
|
||||
#### Low Coverage Modules (<30% → Improved)
|
||||
- [x] `app/utils/filename_utils.py` (24.62% → 81.54%) ✅
|
||||
- Test sanitize_filename with special characters
|
||||
- Test get_unique_filename
|
||||
- Test extract_remote_path
|
||||
- Test filename validation functions
|
||||
|
||||
- [x] `app/utils/logging.py` (42.86% → 100%) ✅
|
||||
- Test log_task_progress function
|
||||
- Test various log message formats
|
||||
|
||||
- [x] `app/utils/oauth_helper.py` (17.50% → 100%) ✅
|
||||
- Test OAuth token exchange
|
||||
- Test error handling
|
||||
- Mock OAuth provider responses
|
||||
|
||||
- [x] `app/utils/notification.py` (44.33% → improved) ✅
|
||||
- Test URL masking for security
|
||||
- Test Apprise initialization
|
||||
- Basic notification sending tests
|
||||
|
||||
### Files Improved
|
||||
1. **app/utils/encryption.py**: 0% → 89.29% (+89.29%)
|
||||
2. **app/celery_worker.py**: 0% → 90.62% (+90.62%)
|
||||
3. **app/tasks/uptime_kuma_tasks.py**: 0% → 100% (+100%)
|
||||
4. **app/frontend.py**: 0% → 100% (+100%)
|
||||
5. **app/utils/filename_utils.py**: 24.62% → 81.54% (+56.92%)
|
||||
6. **app/utils/logging.py**: 42.86% → 100% (+57.14%)
|
||||
7. **app/utils/oauth_helper.py**: 17.50% → 100% (+82.50%)
|
||||
8. **app/utils/notification.py**: 44.33% → improved
|
||||
9. **app/tasks/check_credentials.py**: 0% → 23.13% (+23.13% from imports)
|
||||
10. **app/tasks/imap_tasks.py**: 0% → 15.35% (+15.35% from imports)
|
||||
|
||||
## Phase 2: Medium Priority (Target: 70%+)
|
||||
|
||||
### API Routes with Low Coverage
|
||||
- [ ] `app/api/azure.py` (23.08% → 60%+)
|
||||
- Test Azure connection
|
||||
- Test credential validation
|
||||
- Mock Azure API responses
|
||||
|
||||
- [ ] `app/api/dropbox.py` (16.94% → 50%+)
|
||||
- Test OAuth flow (mocked)
|
||||
- Test token validation
|
||||
- Test connection testing
|
||||
|
||||
- [ ] `app/api/google_drive.py` (12.94% → 50%+)
|
||||
- Test OAuth flow (mocked)
|
||||
- Test token validation
|
||||
- Test drive connection
|
||||
|
||||
- [ ] `app/api/onedrive.py` (13.83% → 50%+)
|
||||
- Test OAuth flow (mocked)
|
||||
- Test token validation
|
||||
- Test connection testing
|
||||
|
||||
### Task Modules with Low Coverage
|
||||
- [ ] `app/tasks/convert_to_pdf.py` (13.41% → 50%+)
|
||||
- Test PDF conversion with various formats
|
||||
- Test Gotenberg integration (mocked)
|
||||
- Test error handling
|
||||
|
||||
- [ ] `app/tasks/embed_metadata_into_pdf.py` (19.05% → 50%+)
|
||||
- Test metadata embedding
|
||||
- Test PDF manipulation
|
||||
- Test error cases
|
||||
|
||||
## Phase 3: Complex Integration Tests (Target: 80%+)
|
||||
|
||||
### Upload Task Modules (Currently 13-36%)
|
||||
These require complex external service mocking:
|
||||
- [ ] `app/tasks/upload_to_dropbox.py` (13.45%)
|
||||
- [ ] `app/tasks/upload_to_google_drive.py` (36.00%)
|
||||
- [ ] `app/tasks/upload_to_onedrive.py` (26.32%)
|
||||
- [ ] `app/tasks/upload_to_nextcloud.py` (15.19%)
|
||||
- [ ] `app/tasks/upload_to_paperless.py` (18.60%)
|
||||
- [ ] `app/tasks/upload_to_email.py` (36.08%)
|
||||
|
||||
### Complex Background Tasks (0-36%)
|
||||
- [ ] `app/tasks/check_credentials.py` (0%)
|
||||
- Requires mocking multiple external services
|
||||
- Test credential validation for each provider
|
||||
- Test failure state management
|
||||
- Test notification system
|
||||
|
||||
- [ ] `app/tasks/imap_tasks.py` (0%)
|
||||
- Requires IMAP server mocking
|
||||
- Test email fetching
|
||||
- Test email parsing
|
||||
- Test lock management with Redis
|
||||
|
||||
- [ ] `app/tasks/upload_with_rclone.py` (0%)
|
||||
- Test rclone command execution
|
||||
- Test configuration management
|
||||
- Test error handling
|
||||
|
||||
- [ ] `app/tasks/extract_metadata_with_gpt.py` (28.79%)
|
||||
- Test GPT metadata extraction
|
||||
- Mock OpenAI API responses
|
||||
- Test various document types
|
||||
|
||||
### View Routes (25-61%)
|
||||
- [ ] `app/views/status.py` (25.00%)
|
||||
- [ ] `app/views/wizard.py` (38.98%)
|
||||
- [ ] `app/views/settings.py` (42.86%)
|
||||
- [ ] `app/views/google_drive.py` (42.42%)
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### For Low-Hanging Fruits (Phase 1)
|
||||
1. Focus on pure functions with minimal dependencies
|
||||
2. Mock external services (OpenAI, Azure, cloud storage)
|
||||
3. Test error paths and edge cases
|
||||
4. Use pytest fixtures for common setup
|
||||
|
||||
### For Integration Tests (Phases 2-3)
|
||||
1. Create comprehensive mocks for external services
|
||||
2. Use pytest-mock for patching
|
||||
3. Test async functions with pytest-asyncio
|
||||
4. Use TestClient for API endpoint tests
|
||||
5. Mock Redis, database, and Celery for task tests
|
||||
|
||||
## Coverage Goals by Phase
|
||||
|
||||
| Phase | Target Coverage | Status |
|
||||
|-------|----------------|--------|
|
||||
| Phase 1: Low-Hanging Fruits | 60% | In Progress |
|
||||
| Phase 2: Medium Priority | 70% | Not Started |
|
||||
| Phase 3: Complex Integration | 80% | Not Started |
|
||||
|
||||
## Notes
|
||||
|
||||
- Files with 100% coverage: Keep them at 100%
|
||||
- Files with 90%+ coverage: Low priority for improvement
|
||||
- Focus on business logic, not simple re-exports
|
||||
- Mock external dependencies to avoid flaky tests
|
||||
- All tests must pass CI/CD pipeline
|
||||
- Maintain test execution time under 2 minutes for fast feedback
|
||||
|
||||
## Files Excluded from Coverage
|
||||
|
||||
These files are infrastructure/configuration and don't require high coverage:
|
||||
- `migrations/*` - Database migrations (excluded in pytest.ini)
|
||||
- `app/__init__.py` - Empty init files
|
||||
- `app/*/__init__.py` - Package init files
|
||||
|
||||
## Running Tests
|
||||
|
||||
```bash
|
||||
# Run all tests with coverage
|
||||
pytest --cov=app --cov-report=term-missing
|
||||
|
||||
# Run tests for specific module
|
||||
pytest tests/test_encryption.py -v
|
||||
|
||||
# Run tests with coverage report
|
||||
pytest --cov=app --cov-report=html
|
||||
open htmlcov/index.html
|
||||
|
||||
# Run only unit tests (fast)
|
||||
pytest -m unit
|
||||
|
||||
# Run integration tests
|
||||
pytest -m integration
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
When adding new code:
|
||||
1. Write tests for new functionality
|
||||
2. Aim for 80%+ coverage on new files
|
||||
3. Update this TODO when completing test coverage work
|
||||
4. Run coverage report before submitting PR
|
||||
@@ -1,352 +0,0 @@
|
||||
# WebDAV Testing - Implementation Summary
|
||||
|
||||
## Overview
|
||||
|
||||
This document summarizes the comprehensive testing implementation for WebDAV upload functionality in DocuElevate.
|
||||
|
||||
## What Was Implemented
|
||||
|
||||
### 1. WebDAV Upload Module (Already Existed)
|
||||
|
||||
**File:** `app/tasks/upload_to_webdav.py`
|
||||
|
||||
- Celery task for uploading files to WebDAV servers
|
||||
- Supports HTTP Basic authentication
|
||||
- Configurable SSL verification
|
||||
- URL/folder path normalization
|
||||
- Retry logic via `BaseTaskWithRetry` (3 retries, exponential backoff)
|
||||
- Progress logging integration
|
||||
|
||||
**Configuration:**
|
||||
- `WEBDAV_URL` - Server URL
|
||||
- `WEBDAV_USERNAME` - Authentication username
|
||||
- `WEBDAV_PASSWORD` - Authentication password
|
||||
- `WEBDAV_FOLDER` - Target folder path
|
||||
- `WEBDAV_VERIFY_SSL` - SSL certificate verification
|
||||
|
||||
### 2. Comprehensive Unit Tests ✅
|
||||
|
||||
**File:** `tests/test_upload_webdav_comprehensive.py`
|
||||
|
||||
**Tests:** 23 (all passing)
|
||||
|
||||
**Coverage:**
|
||||
- Success scenarios (with/without file_id, different HTTP status codes: 200, 201, 204)
|
||||
- Configuration validation (missing URL)
|
||||
- Error handling (file not found, HTTP errors: 401, 404, 500)
|
||||
- Connection errors (timeout, connection refused)
|
||||
- URL construction (trailing slash, no trailing slash, leading slash in folder)
|
||||
- Folder path normalization (empty, leading slash)
|
||||
- SSL verification (enabled/disabled)
|
||||
- Authentication credentials
|
||||
- Logging verification (success/failure)
|
||||
- File content upload
|
||||
- Return value structure
|
||||
- Task importability
|
||||
- Retry configuration
|
||||
|
||||
**Result:** 100% code coverage on `upload_to_webdav.py`
|
||||
|
||||
### 3. Integration Tests with Real WebDAV Server ✅
|
||||
|
||||
**File:** `tests/test_upload_webdav_integration.py`
|
||||
|
||||
**Tests:** 10 (all passing)
|
||||
|
||||
**Infrastructure:**
|
||||
- Uses `testcontainers` library
|
||||
- Spins up real WebDAV server (bytemark/webdav:latest)
|
||||
- Docker container runs during tests
|
||||
- Automatic cleanup after tests
|
||||
|
||||
**Test Scenarios:**
|
||||
1. Upload file to real server and verify content
|
||||
2. Upload to subfolder with MKCOL command
|
||||
3. Upload PDF file and verify magic bytes
|
||||
4. Upload with wrong credentials (401 error)
|
||||
5. Upload multiple files sequentially
|
||||
6. Overwrite existing file
|
||||
7. Upload large file (1MB)
|
||||
8. WebDAV server basic authentication
|
||||
9. WebDAV PUT method support
|
||||
10. WebDAV PROPFIND method support
|
||||
|
||||
**Result:** Verifies actual file uploads to real WebDAV server
|
||||
|
||||
### 4. Full-Stack Integration Infrastructure ✅
|
||||
|
||||
**File:** `tests/fixtures_integration.py`
|
||||
|
||||
**Provides Fixtures For:**
|
||||
- **PostgreSQL** - Real database (replaces SQLite in-memory)
|
||||
- **Redis** - Real message broker for Celery
|
||||
- **Gotenberg** - Real PDF conversion service
|
||||
- **WebDAV** - Real upload target
|
||||
- **SFTP** - Real SSH/SFTP server
|
||||
- **MinIO** - Real S3-compatible storage
|
||||
- **FTP** - Real FTP server
|
||||
- **Celery App** - Configured for test Redis
|
||||
- **Celery Worker** - Actually processes queued tasks
|
||||
|
||||
### 5. End-to-End Tests ✅
|
||||
|
||||
**File:** `tests/test_e2e_full_stack.py`
|
||||
|
||||
**Test Classes:**
|
||||
1. **TestEndToEndWithRedis** - Redis + Celery integration
|
||||
- Queue task in Redis → Worker executes → Upload to WebDAV
|
||||
- Task queuing verification
|
||||
- Parallel task execution
|
||||
- Task retry on failure
|
||||
|
||||
2. **TestFullInfrastructure** - Complete stack
|
||||
- All infrastructure components running
|
||||
- Database operations with PostgreSQL
|
||||
- Upload to multiple targets (WebDAV + SFTP)
|
||||
- Gotenberg PDF conversion
|
||||
- MinIO S3 uploads
|
||||
- SFTP uploads
|
||||
|
||||
3. **TestProductionLikeScenarios** - Complete workflows
|
||||
- Full document processing pipeline
|
||||
- Database → Redis → Celery → WebDAV
|
||||
- End-to-end verification
|
||||
|
||||
### 6. Documentation ✅
|
||||
|
||||
**File:** `tests/README_INTEGRATION_TESTS.md`
|
||||
|
||||
**Contents:**
|
||||
- Overview of integration testing approach
|
||||
- Prerequisites and setup
|
||||
- Test organization and markers
|
||||
- Running tests (unit, integration, e2e)
|
||||
- Infrastructure fixtures documentation
|
||||
- Example test scenarios
|
||||
- Performance notes and resource usage
|
||||
- Debugging and troubleshooting
|
||||
- CI/CD integration examples
|
||||
- Best practices
|
||||
- Coverage information
|
||||
|
||||
## Test Execution Summary
|
||||
|
||||
### Unit Tests (Mocked)
|
||||
```bash
|
||||
pytest tests/test_upload_webdav_comprehensive.py -v
|
||||
```
|
||||
- **Tests:** 23/23 ✅
|
||||
- **Speed:** ~2 seconds
|
||||
- **Coverage:** 100%
|
||||
- **Docker Required:** No
|
||||
|
||||
### Integration Tests (Real WebDAV)
|
||||
```bash
|
||||
pytest tests/test_upload_webdav_integration.py -v
|
||||
```
|
||||
- **Tests:** 10/10 ✅
|
||||
- **Speed:** ~7 seconds
|
||||
- **Coverage:** 79.31% (focuses on happy paths with real server)
|
||||
- **Docker Required:** Yes
|
||||
|
||||
### End-to-End Tests (Full Stack)
|
||||
```bash
|
||||
pytest tests/test_e2e_full_stack.py -v
|
||||
```
|
||||
- **Tests:** 12+ scenarios
|
||||
- **Speed:** ~30-60 seconds per test
|
||||
- **Coverage:** Complete application workflow
|
||||
- **Docker Required:** Yes
|
||||
|
||||
### All WebDAV Tests
|
||||
```bash
|
||||
pytest tests/test_upload_webdav*.py -v
|
||||
```
|
||||
- **Total Tests:** 33 ✅
|
||||
- **Speed:** ~7 seconds total
|
||||
- **Result:** All passing
|
||||
|
||||
## Infrastructure Components
|
||||
|
||||
### Container Images Used
|
||||
|
||||
| Service | Image | Port | Purpose |
|
||||
|---------|-------|------|---------|
|
||||
| WebDAV | bytemark/webdav:latest | 80 | Upload target |
|
||||
| PostgreSQL | postgres:15-alpine | 5432 | Real database |
|
||||
| Redis | redis:7-alpine | 6379 | Celery broker |
|
||||
| Gotenberg | gotenberg/gotenberg:8 | 3000 | PDF conversion |
|
||||
| SFTP | atmoz/sftp:latest | 22 | SFTP uploads |
|
||||
| MinIO | minio/minio:latest | 9000 | S3 storage |
|
||||
| FTP | stilliard/pure-ftpd:latest | 21 | FTP uploads |
|
||||
|
||||
### Resource Requirements
|
||||
|
||||
- **Docker:** Must be installed and running
|
||||
- **Memory:** ~100MB per container, ~1GB total for full stack
|
||||
- **Disk:** ~2GB for all Docker images
|
||||
- **Time:**
|
||||
- First run: ~5-10 minutes (image pulls)
|
||||
- Subsequent runs: ~10-60 seconds per test
|
||||
|
||||
## Dependencies Added
|
||||
|
||||
**`requirements-dev.txt`:**
|
||||
```
|
||||
testcontainers>=3.7.1 # Container management
|
||||
minio>=7.1.0 # MinIO client
|
||||
redis>=4.5.0 # Redis client
|
||||
boto3>=1.26.0 # AWS S3 client (for MinIO)
|
||||
```
|
||||
|
||||
All dependencies are development/testing only.
|
||||
|
||||
## Test Markers
|
||||
|
||||
Custom pytest markers for organizing tests:
|
||||
|
||||
```python
|
||||
@pytest.mark.unit # Fast unit tests, no Docker
|
||||
@pytest.mark.integration # Integration tests with containers
|
||||
@pytest.mark.e2e # Full end-to-end scenarios
|
||||
@pytest.mark.requires_docker # Requires Docker to run
|
||||
@pytest.mark.slow # Takes >30 seconds
|
||||
```
|
||||
|
||||
## Key Features
|
||||
|
||||
### 1. Real Infrastructure Testing
|
||||
- Tests run against actual services, not mocks
|
||||
- Verifies files are actually uploaded
|
||||
- Catches integration issues early
|
||||
|
||||
### 2. Production-Like Scenarios
|
||||
- PostgreSQL instead of SQLite
|
||||
- Redis message queueing
|
||||
- Celery worker execution
|
||||
- Async task processing
|
||||
|
||||
### 3. Comprehensive Coverage
|
||||
- **Unit tests:** Edge cases, error handling, validation
|
||||
- **Integration tests:** Real server behavior, file operations
|
||||
- **E2E tests:** Complete workflows, multi-service coordination
|
||||
|
||||
### 4. Automatic Cleanup
|
||||
- Testcontainers auto-remove after tests
|
||||
- No manual cleanup required
|
||||
- Isolated test environments
|
||||
|
||||
### 5. Developer-Friendly
|
||||
- Clear test organization
|
||||
- Detailed documentation
|
||||
- Easy to run locally
|
||||
- CI/CD ready
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Run Quick Unit Tests
|
||||
```bash
|
||||
# Fast, no Docker needed
|
||||
pytest tests/test_upload_webdav_comprehensive.py -v
|
||||
```
|
||||
|
||||
### Verify Upload Works Against Real Server
|
||||
```bash
|
||||
# Spins up WebDAV container
|
||||
pytest tests/test_upload_webdav_integration.py::TestWebDAVIntegration::test_upload_file_to_real_webdav_server -v
|
||||
```
|
||||
|
||||
### Test Complete Workflow with Redis
|
||||
```bash
|
||||
# Full stack: Redis + Celery + WebDAV
|
||||
pytest tests/test_e2e_full_stack.py::TestEndToEndWithRedis::test_webdav_upload_with_redis_and_celery -v
|
||||
```
|
||||
|
||||
### Run All Infrastructure Tests
|
||||
```bash
|
||||
# All services
|
||||
pytest -m e2e -v
|
||||
```
|
||||
|
||||
## CI/CD Integration
|
||||
|
||||
### GitHub Actions Example
|
||||
```yaml
|
||||
- name: Run Integration Tests
|
||||
run: |
|
||||
pytest -m "integration or e2e" -v --tb=short
|
||||
```
|
||||
|
||||
Tests are designed to run in CI environments with Docker support.
|
||||
|
||||
## Benefits
|
||||
|
||||
### For Development
|
||||
1. **Fast Feedback:** Unit tests run in seconds
|
||||
2. **Confidence:** Integration tests verify real behavior
|
||||
3. **Debug Easily:** Containers provide inspection access
|
||||
|
||||
### For QA/Testing
|
||||
1. **Real Scenarios:** Tests match production behavior
|
||||
2. **Complete Coverage:** Unit + Integration + E2E
|
||||
3. **Reproducible:** Docker ensures consistency
|
||||
|
||||
### For Production
|
||||
1. **Early Detection:** Catch issues before deployment
|
||||
2. **Regression Prevention:** Comprehensive test suite
|
||||
3. **Documentation:** Tests serve as usage examples
|
||||
|
||||
## Comparison to Other Upload Modules
|
||||
|
||||
Most other upload modules (S3, SFTP, FTP, Dropbox, Google Drive) only have:
|
||||
- Basic unit tests with mocks (1-2 tests each)
|
||||
- No integration tests with real servers
|
||||
- No end-to-end tests
|
||||
|
||||
WebDAV now has:
|
||||
- ✅ 23 comprehensive unit tests
|
||||
- ✅ 10 integration tests with real server
|
||||
- ✅ Full e2e test infrastructure
|
||||
- ✅ 100% code coverage
|
||||
- ✅ Production-like testing
|
||||
|
||||
**WebDAV is now the reference implementation for testing upload modules.**
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
### Potential Additions
|
||||
1. Add similar integration tests for SFTP, FTP, S3
|
||||
2. Test WebDAV with different servers (ownCloud, Nextcloud, Synology)
|
||||
3. Test large file uploads (>100MB)
|
||||
4. Test concurrent uploads (stress testing)
|
||||
5. Test network failure scenarios
|
||||
6. Test SSL/TLS certificate validation
|
||||
|
||||
### Template for Other Modules
|
||||
The WebDAV testing approach can be replicated for other upload destinations:
|
||||
1. Create `test_upload_<destination>_comprehensive.py` (unit tests)
|
||||
2. Create `test_upload_<destination>_integration.py` (with real server)
|
||||
3. Add container fixture to `fixtures_integration.py`
|
||||
4. Add e2e scenarios to `test_e2e_full_stack.py`
|
||||
|
||||
## Conclusion
|
||||
|
||||
The WebDAV upload functionality is now **comprehensively tested** with:
|
||||
- ✅ 33 passing tests
|
||||
- ✅ 100% code coverage (unit tests)
|
||||
- ✅ Real server verification (integration tests)
|
||||
- ✅ Production-like scenarios (e2e tests)
|
||||
- ✅ Full infrastructure testing capability
|
||||
|
||||
This provides **high confidence** that WebDAV uploads work correctly in production and serves as a **reference implementation** for testing other upload modules.
|
||||
|
||||
## Related Files
|
||||
|
||||
- `app/tasks/upload_to_webdav.py` - Implementation
|
||||
- `tests/test_upload_webdav_comprehensive.py` - Unit tests (23)
|
||||
- `tests/test_upload_webdav_integration.py` - Integration tests (10)
|
||||
- `tests/fixtures_integration.py` - Infrastructure fixtures
|
||||
- `tests/test_e2e_full_stack.py` - End-to-end tests (12+)
|
||||
- `tests/README_INTEGRATION_TESTS.md` - Documentation
|
||||
- `requirements-dev.txt` - Test dependencies
|
||||
- `tests/conftest.py` - Pytest configuration
|
||||
-73
@@ -1,73 +0,0 @@
|
||||
# Alembic Configuration File
|
||||
# Used for managing database schema migrations in DocuElevate.
|
||||
#
|
||||
# Usage:
|
||||
# alembic upgrade head # Apply all pending migrations
|
||||
# alembic current # Show current revision
|
||||
# alembic history --verbose # Show migration history
|
||||
# alembic downgrade -1 # Roll back one migration
|
||||
# alembic revision --autogenerate -m "description" # Create new migration
|
||||
|
||||
[alembic]
|
||||
# Path to migration scripts
|
||||
script_location = migrations
|
||||
|
||||
# Template used to generate migration file names
|
||||
file_template = %%(rev)s_%%(slug)s
|
||||
|
||||
# Timezone for migration file timestamps (uses UTC by default)
|
||||
# timezone =
|
||||
|
||||
# Maximum length of characters for autogenerate revision names
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# Set to 'true' to run environment during 'revision' command
|
||||
# revision_environment = false
|
||||
|
||||
# Set to 'true' to allow .pyc or .pyo files for migration scripts
|
||||
# sourceless = false
|
||||
|
||||
# Version path separator; default is "os" which uses os.pathsep
|
||||
# version_path_separator = os
|
||||
|
||||
# Output encoding for revision files
|
||||
# output_encoding = utf-8
|
||||
|
||||
# The database URL is loaded from app.config.settings.database_url
|
||||
# in migrations/env.py, not from this file.
|
||||
sqlalchemy.url =
|
||||
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -1,3 +0,0 @@
|
||||
"""
|
||||
Document processor application package.
|
||||
"""
|
||||
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
# app/api.py
|
||||
from fastapi import APIRouter, Request, HTTPException, status, Depends
|
||||
from hashlib import md5
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import List
|
||||
|
||||
from app.auth import require_login
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.get("/whoami")
|
||||
async def whoami(request: Request):
|
||||
"""
|
||||
Returns user info if logged in, else 401.
|
||||
"""
|
||||
user = request.session.get("user")
|
||||
if not user:
|
||||
raise HTTPException(status_code=401, detail="Not logged in")
|
||||
|
||||
email = user.get("email")
|
||||
if not email:
|
||||
raise HTTPException(status_code=400, detail="User has no email in session")
|
||||
|
||||
# Generate Gravatar URL from email
|
||||
email_hash = md5(email.strip().lower().encode()).hexdigest()
|
||||
gravatar_url = f"https://www.gravatar.com/avatar/{email_hash}?d=identicon"
|
||||
|
||||
return {
|
||||
"email": email,
|
||||
"picture": gravatar_url
|
||||
}
|
||||
|
||||
@router.get("/files")
|
||||
@require_login
|
||||
def list_files_api(request: Request, db: Session = Depends(get_db)):
|
||||
"""
|
||||
Returns a JSON list of all FileRecord entries.
|
||||
Protected by `@require_login`, so only logged-in sessions can access.
|
||||
|
||||
Example response:
|
||||
[
|
||||
{
|
||||
"id": 123,
|
||||
"filehash": "abc123...",
|
||||
"original_filename": "example.pdf",
|
||||
"local_filename": "/workdir/tmp/<uuid>.pdf",
|
||||
"file_size": 1048576,
|
||||
"mime_type": "application/pdf",
|
||||
"created_at": "2025-05-01T12:34:56.789000"
|
||||
},
|
||||
...
|
||||
]
|
||||
"""
|
||||
files = db.query(FileRecord).order_by(FileRecord.created_at.desc()).all()
|
||||
# Return a simple list of dicts
|
||||
result = []
|
||||
for f in files:
|
||||
result.append({
|
||||
"id": f.id,
|
||||
"filehash": f.filehash,
|
||||
"original_filename": f.original_filename,
|
||||
"local_filename": f.local_filename,
|
||||
"file_size": f.file_size,
|
||||
"mime_type": f.mime_type,
|
||||
"created_at": f.created_at.isoformat() if f.created_at else None
|
||||
})
|
||||
return result
|
||||
@@ -1,48 +0,0 @@
|
||||
"""
|
||||
API Router module that combines all API endpoints
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.azure import router as azure_router
|
||||
from app.api.diagnostic import router as diagnostic_router
|
||||
from app.api.dropbox import router as dropbox_router
|
||||
from app.api.files import router as files_router
|
||||
from app.api.google_drive import router as google_drive_router
|
||||
from app.api.logs import router as logs_router
|
||||
from app.api.onedrive import router as onedrive_router
|
||||
from app.api.openai import router as openai_router
|
||||
from app.api.process import router as process_router
|
||||
from app.api.queue import router as queue_router
|
||||
from app.api.saved_searches import router as saved_searches_router
|
||||
from app.api.search import router as search_router
|
||||
from app.api.settings import router as settings_router
|
||||
from app.api.url_upload import router as url_upload_router
|
||||
|
||||
# Import all the individual routers
|
||||
from app.api.user import router as user_router
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Create the main router that includes all the others
|
||||
router = APIRouter()
|
||||
|
||||
# Include all the routers
|
||||
router.include_router(user_router)
|
||||
router.include_router(files_router)
|
||||
router.include_router(process_router)
|
||||
router.include_router(diagnostic_router)
|
||||
router.include_router(onedrive_router)
|
||||
router.include_router(dropbox_router)
|
||||
router.include_router(openai_router)
|
||||
router.include_router(azure_router)
|
||||
router.include_router(google_drive_router)
|
||||
router.include_router(logs_router)
|
||||
router.include_router(settings_router)
|
||||
router.include_router(url_upload_router)
|
||||
router.include_router(search_router)
|
||||
router.include_router(queue_router)
|
||||
router.include_router(saved_searches_router)
|
||||
@@ -1,114 +0,0 @@
|
||||
"""
|
||||
Azure AI API endpoints
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import azure.core.exceptions
|
||||
from azure.ai.documentintelligence import DocumentIntelligenceAdministrationClient
|
||||
|
||||
# Import the Azure modules including the administration client
|
||||
from azure.core.credentials import AzureKeyCredential
|
||||
from fastapi import APIRouter, Request
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/azure/test")
|
||||
@require_login
|
||||
async def test_azure_connection(request: Request):
|
||||
"""
|
||||
Test if the configured Azure Document Intelligence connection is valid.
|
||||
Uses the DocumentIntelligenceAdministrationClient for testing the connection.
|
||||
"""
|
||||
try:
|
||||
logger.info("Testing Azure Document Intelligence connection")
|
||||
|
||||
# Check if Azure configuration is present
|
||||
if not settings.azure_endpoint or not settings.azure_ai_key:
|
||||
logger.warning("Azure Document Intelligence configuration is incomplete")
|
||||
missing = []
|
||||
if not settings.azure_endpoint:
|
||||
missing.append("endpoint")
|
||||
if not settings.azure_ai_key:
|
||||
missing.append("API key")
|
||||
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Azure Document Intelligence configuration is incomplete. Missing: {', '.join(missing)}",
|
||||
}
|
||||
|
||||
# Try to initialize the admin client and make a request to list operations
|
||||
try:
|
||||
# Initialize the admin client with credentials
|
||||
admin_client = DocumentIntelligenceAdministrationClient(
|
||||
endpoint=settings.azure_endpoint, credential=AzureKeyCredential(settings.azure_ai_key)
|
||||
)
|
||||
|
||||
# Test the connection by listing operations - this is a documented method in the admin client
|
||||
operations = list(admin_client.list_operations())
|
||||
|
||||
# Successfully initialized client and made a request
|
||||
logger.info("Azure Document Intelligence Admin connection successfully tested")
|
||||
|
||||
# Return success with available operations info
|
||||
operations_info = []
|
||||
try:
|
||||
for op in operations:
|
||||
if hasattr(op, "operation_id") and op.operation_id:
|
||||
op_info = {
|
||||
"id": op.operation_id,
|
||||
"status": op.status if hasattr(op, "status") else "Unknown",
|
||||
"created": str(op.created_on) if hasattr(op, "created_on") else "Unknown",
|
||||
"kind": op.kind if hasattr(op, "kind") else "Unknown",
|
||||
}
|
||||
operations_info.append(op_info)
|
||||
|
||||
operation_count = len(operations_info)
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Azure Document Intelligence connection is valid. Found {operation_count} operations.",
|
||||
"endpoint": settings.azure_endpoint,
|
||||
"operations_count": operation_count,
|
||||
"recent_operations": operations_info[:3] if operations_info else [],
|
||||
}
|
||||
except Exception as e:
|
||||
# If error occurs while processing operations info, still return success
|
||||
logger.warning(f"Connected to Azure but couldn't parse operations: {e}")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Azure Document Intelligence connection is valid, "
|
||||
"but couldn't retrieve operations details.",
|
||||
"endpoint": settings.azure_endpoint,
|
||||
}
|
||||
|
||||
except azure.core.exceptions.ClientAuthenticationError as e:
|
||||
logger.error(f"Azure authentication error: {e}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Authentication error: Invalid API key or credentials",
|
||||
"detail": str(e),
|
||||
}
|
||||
except azure.core.exceptions.ServiceRequestError as e:
|
||||
logger.error(f"Azure service request error: {e}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Service request error: Could not reach the Azure endpoint",
|
||||
"detail": str(e),
|
||||
}
|
||||
except ValueError as e:
|
||||
logger.error(f"Azure configuration value error: {e}")
|
||||
return {"status": "error", "message": f"Configuration error: {str(e)}", "detail": str(e)}
|
||||
except Exception as e:
|
||||
logger.error(f"Azure connection test failed with unexpected error: {e}")
|
||||
return {"status": "error", "message": "Connection test failed with unexpected error", "detail": str(e)}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Unexpected error testing Azure Document Intelligence connection")
|
||||
return {"status": "error", "message": f"Unexpected error: {str(e)}"}
|
||||
@@ -1,64 +0,0 @@
|
||||
"""
|
||||
Common utilities for API routes
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
|
||||
from app.config import settings
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def resolve_file_path(file_path: str, subfolder: str = None) -> str:
|
||||
"""
|
||||
Resolves a file path to an absolute path with path traversal protection.
|
||||
If the path is not absolute, it will be joined with the workdir path.
|
||||
Optionally, can include a subfolder like 'processed'.
|
||||
|
||||
Security: Validates that the resolved path stays within the workdir
|
||||
to prevent path traversal attacks (e.g., ../../etc/passwd).
|
||||
|
||||
Args:
|
||||
file_path: The file path to resolve
|
||||
subfolder: Optional subfolder within workdir
|
||||
|
||||
Returns:
|
||||
The validated absolute file path
|
||||
|
||||
Raises:
|
||||
HTTPException: If the path attempts to escape the workdir
|
||||
"""
|
||||
# Get the workdir as the security boundary
|
||||
workdir = Path(settings.workdir).resolve()
|
||||
|
||||
# Build the base directory (workdir or workdir/subfolder)
|
||||
if subfolder:
|
||||
base_dir = workdir / subfolder
|
||||
else:
|
||||
base_dir = workdir
|
||||
|
||||
# Resolve the file path
|
||||
if not os.path.isabs(file_path):
|
||||
# Relative path: join with base_dir
|
||||
resolved_path = (base_dir / file_path).resolve()
|
||||
else:
|
||||
# Absolute path: resolve as-is
|
||||
resolved_path = Path(file_path).resolve()
|
||||
|
||||
# Ensure the resolved path is within workdir (path traversal protection)
|
||||
# This checks both relative and absolute paths against workdir
|
||||
try:
|
||||
resolved_path.relative_to(workdir)
|
||||
except ValueError:
|
||||
# Path is outside the workdir - potential path traversal attack
|
||||
logger.warning(f"Path traversal attempt detected: {file_path} -> {resolved_path}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid file path: path traversal not allowed"
|
||||
)
|
||||
|
||||
return str(resolved_path)
|
||||
@@ -1,66 +0,0 @@
|
||||
"""
|
||||
Diagnostic API endpoints
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/diagnostic/test-notification")
|
||||
@require_login
|
||||
async def test_notification(request: Request):
|
||||
# Add request_time to request.state
|
||||
import datetime
|
||||
|
||||
request.state.request_time = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
"""
|
||||
Send a test notification through all configured notification channels
|
||||
"""
|
||||
from app.utils.notification import send_notification
|
||||
|
||||
try:
|
||||
notification_urls = getattr(settings, "notification_urls", [])
|
||||
if not notification_urls:
|
||||
return {
|
||||
"status": "warning",
|
||||
"message": "No notification services configured. Add notification URLs to your configuration.",
|
||||
}
|
||||
|
||||
# Send a test notification
|
||||
hostname = settings.external_hostname or "Document Processor"
|
||||
result = send_notification(
|
||||
title=f"Test Notification from {hostname}",
|
||||
message=(
|
||||
f"This is a test notification sent at {request.state.request_time}. "
|
||||
"If you're receiving this, notifications are working!"
|
||||
),
|
||||
notification_type="success",
|
||||
tags=["test", "notification", "diagnostic"],
|
||||
)
|
||||
|
||||
if result:
|
||||
logger.info("Test notification sent successfully")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Test notification sent successfully to {len(notification_urls)} service(s)",
|
||||
"services_count": len(notification_urls),
|
||||
}
|
||||
else:
|
||||
logger.warning("Test notification send attempt returned False")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Failed to send test notification. Check application logs for details.",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Error sending test notification: {e}")
|
||||
return {"status": "error", "message": f"Error sending notification: {str(e)}"}
|
||||
@@ -1,303 +0,0 @@
|
||||
"""
|
||||
Dropbox API endpoints
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Annotated, Optional
|
||||
|
||||
import requests
|
||||
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.utils.oauth_helper import exchange_oauth_token
|
||||
from app.utils.settings_service import save_setting_to_db
|
||||
from app.utils.settings_sync import notify_settings_updated
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/dropbox/exchange-token")
|
||||
@require_login
|
||||
async def exchange_dropbox_token(
|
||||
request: Request,
|
||||
client_id: Annotated[str, Form(...)],
|
||||
client_secret: Annotated[str, Form(...)],
|
||||
redirect_uri: Annotated[str, Form(...)],
|
||||
code: Annotated[str, Form(...)],
|
||||
folder_path: Annotated[Optional[str], Form()] = None,
|
||||
):
|
||||
"""
|
||||
Exchange an authorization code for a refresh token from Dropbox.
|
||||
This is done on the server to avoid exposing client secret in the browser.
|
||||
"""
|
||||
# Prepare the token request
|
||||
token_url = "https://api.dropboxapi.com/oauth2/token"
|
||||
|
||||
payload = {
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"code": code,
|
||||
"redirect_uri": redirect_uri,
|
||||
"grant_type": "authorization_code",
|
||||
}
|
||||
|
||||
# Use shared OAuth helper (handles secure logging and error handling)
|
||||
token_data = exchange_oauth_token(provider_name="Dropbox", token_url=token_url, payload=payload)
|
||||
|
||||
# Return just what's needed by the frontend
|
||||
return {
|
||||
"refresh_token": token_data["refresh_token"],
|
||||
"access_token": token_data["access_token"],
|
||||
"expires_in": token_data.get("expires_in", 14400),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/dropbox/update-settings")
|
||||
@require_login
|
||||
async def update_dropbox_settings(
|
||||
request: Request,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
app_key: Annotated[Optional[str], Form()] = None,
|
||||
app_secret: Annotated[Optional[str], Form()] = None,
|
||||
folder_path: Annotated[Optional[str], Form()] = None,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Update Dropbox settings in memory and persist to the database.
|
||||
"""
|
||||
try:
|
||||
logger.info("Updating Dropbox settings in memory and database")
|
||||
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
|
||||
)
|
||||
|
||||
# Update settings in memory and persist to database
|
||||
if refresh_token:
|
||||
settings.dropbox_refresh_token = refresh_token
|
||||
save_setting_to_db(db, "dropbox_refresh_token", refresh_token, changed_by=changed_by)
|
||||
logger.info("Updated DROPBOX_REFRESH_TOKEN in memory and database")
|
||||
|
||||
if app_key:
|
||||
settings.dropbox_app_key = app_key
|
||||
save_setting_to_db(db, "dropbox_app_key", app_key, changed_by=changed_by)
|
||||
logger.info("Updated DROPBOX_APP_KEY in memory and database")
|
||||
|
||||
if app_secret:
|
||||
settings.dropbox_app_secret = app_secret
|
||||
save_setting_to_db(db, "dropbox_app_secret", app_secret, changed_by=changed_by)
|
||||
logger.info("Updated DROPBOX_APP_SECRET in memory and database")
|
||||
|
||||
if folder_path:
|
||||
settings.dropbox_folder = folder_path
|
||||
save_setting_to_db(db, "dropbox_folder", folder_path, changed_by=changed_by)
|
||||
logger.info("Updated DROPBOX_FOLDER in memory and database")
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Dropbox settings have been updated in memory and saved to database",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error updating Dropbox settings: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to update Dropbox settings: {str(e)}",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/dropbox/test-token")
|
||||
@require_login
|
||||
async def test_dropbox_token(request: Request):
|
||||
"""
|
||||
Test if the configured Dropbox token is valid and return expiration information.
|
||||
"""
|
||||
try:
|
||||
logger.info("Testing Dropbox token validity")
|
||||
|
||||
if not settings.dropbox_refresh_token or not settings.dropbox_app_key or not settings.dropbox_app_secret:
|
||||
logger.warning("Dropbox credentials not fully configured")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Dropbox credentials are not fully configured",
|
||||
}
|
||||
|
||||
# Check token validity by getting current account info
|
||||
headers = {"Authorization": f"Bearer {settings.dropbox_refresh_token}"}
|
||||
response = requests.post(
|
||||
"https://api.dropboxapi.com/2/users/get_current_account",
|
||||
headers=headers,
|
||||
timeout=settings.http_request_timeout,
|
||||
)
|
||||
|
||||
# If token is invalid, try refreshing it
|
||||
if response.status_code == 401:
|
||||
logger.info("Dropbox access token invalid or expired, trying to refresh")
|
||||
|
||||
# Get a new access token using the refresh token
|
||||
refresh_url = "https://api.dropbox.com/oauth2/token"
|
||||
refresh_data = {
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": settings.dropbox_refresh_token,
|
||||
"client_id": settings.dropbox_app_key,
|
||||
"client_secret": settings.dropbox_app_secret,
|
||||
}
|
||||
|
||||
refresh_response = requests.post(refresh_url, data=refresh_data, timeout=settings.http_request_timeout)
|
||||
|
||||
if refresh_response.status_code != 200:
|
||||
logger.error(f"Failed to refresh Dropbox token: {refresh_response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Refresh token has expired or is invalid",
|
||||
"needs_reauth": True,
|
||||
}
|
||||
|
||||
token_info = refresh_response.json()
|
||||
access_token = token_info.get("access_token")
|
||||
|
||||
# Try again with the new access token
|
||||
headers = {"Authorization": f"Bearer {access_token}"}
|
||||
response = requests.post(
|
||||
"https://api.dropboxapi.com/2/users/get_current_account",
|
||||
headers=headers,
|
||||
timeout=settings.http_request_timeout,
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.error(f"Dropbox token test failed: {response.status_code} {response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Token validation failed with status {response.status_code}: {response.text}",
|
||||
}
|
||||
|
||||
# Get account info
|
||||
account_info = response.json()
|
||||
account_email = account_info.get("email", "Unknown account")
|
||||
account_name = account_info.get("name", {}).get("display_name", "Unknown user")
|
||||
|
||||
# Dropbox refresh tokens don't expire, but we should note that in our response
|
||||
token_info = {
|
||||
"expires_in_human": "Never expires (perpetual token)",
|
||||
"is_perpetual": True,
|
||||
}
|
||||
|
||||
logger.info(f"Successfully connected to Dropbox as {account_email}")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Dropbox connection successful",
|
||||
"account": account_email,
|
||||
"account_name": account_name,
|
||||
"token_info": token_info,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error testing Dropbox token: {str(e)}")
|
||||
return {"status": "error", "message": f"Connection error: {str(e)}"}
|
||||
|
||||
|
||||
@router.post("/dropbox/save-settings")
|
||||
@require_login
|
||||
async def save_dropbox_settings(
|
||||
request: Request,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
app_key: Annotated[Optional[str], Form()] = None,
|
||||
app_secret: Annotated[Optional[str], Form()] = None,
|
||||
folder_path: Annotated[Optional[str], Form()] = None,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Save Dropbox settings to database (primary) and .env file (best-effort).
|
||||
"""
|
||||
try:
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
|
||||
)
|
||||
|
||||
# Update settings in memory
|
||||
if refresh_token:
|
||||
settings.dropbox_refresh_token = refresh_token
|
||||
if app_key:
|
||||
settings.dropbox_app_key = app_key
|
||||
if app_secret:
|
||||
settings.dropbox_app_secret = app_secret
|
||||
if folder_path:
|
||||
settings.dropbox_folder = folder_path
|
||||
|
||||
# Persist to database (primary storage)
|
||||
if refresh_token:
|
||||
save_setting_to_db(db, "dropbox_refresh_token", refresh_token, changed_by=changed_by)
|
||||
if app_key:
|
||||
save_setting_to_db(db, "dropbox_app_key", app_key, changed_by=changed_by)
|
||||
if app_secret:
|
||||
save_setting_to_db(db, "dropbox_app_secret", app_secret, changed_by=changed_by)
|
||||
if folder_path:
|
||||
save_setting_to_db(db, "dropbox_folder", folder_path, changed_by=changed_by)
|
||||
|
||||
# Best-effort .env file write
|
||||
try:
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
if not os.path.exists(env_path):
|
||||
logger.warning(f".env file not found at {env_path}, skipping file write")
|
||||
else:
|
||||
logger.info(f"Updating Dropbox settings in {env_path}")
|
||||
|
||||
with open(env_path, "r") as f:
|
||||
env_lines = f.readlines()
|
||||
|
||||
dropbox_settings = {"DROPBOX_REFRESH_TOKEN": refresh_token}
|
||||
if app_key:
|
||||
dropbox_settings["DROPBOX_APP_KEY"] = app_key
|
||||
if app_secret:
|
||||
dropbox_settings["DROPBOX_APP_SECRET"] = app_secret
|
||||
if folder_path:
|
||||
dropbox_settings["DROPBOX_FOLDER"] = folder_path
|
||||
|
||||
updated = set()
|
||||
new_env_lines = []
|
||||
for line in env_lines:
|
||||
stripped_line = line.rstrip()
|
||||
is_updated = False
|
||||
for key, value in dropbox_settings.items():
|
||||
if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="):
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
updated.add(key)
|
||||
is_updated = True
|
||||
break
|
||||
if not is_updated:
|
||||
new_env_lines.append(stripped_line)
|
||||
|
||||
for key, value in dropbox_settings.items():
|
||||
if key not in updated:
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
|
||||
with open(env_path, "w") as f:
|
||||
f.write("\n".join(new_env_lines) + "\n")
|
||||
|
||||
logger.info("Successfully updated Dropbox settings in .env file")
|
||||
except Exception as env_err:
|
||||
logger.warning(f"Failed to write .env file (non-fatal): {env_err}")
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
logger.info("Successfully saved Dropbox settings")
|
||||
return {"status": "success", "message": "Dropbox settings have been saved"}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error saving Dropbox settings: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to save Dropbox settings: {str(e)}",
|
||||
)
|
||||
-1357
File diff suppressed because it is too large
Load Diff
@@ -1,492 +0,0 @@
|
||||
"""
|
||||
Google Drive API endpoints
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.utils.oauth_helper import exchange_oauth_token
|
||||
from app.utils.settings_service import save_setting_to_db
|
||||
from app.utils.settings_sync import notify_settings_updated
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/google-drive/exchange-token")
|
||||
@require_login
|
||||
async def exchange_google_drive_token(
|
||||
request: Request,
|
||||
client_id: Annotated[str, Form(...)],
|
||||
client_secret: Annotated[str, Form(...)],
|
||||
redirect_uri: Annotated[str, Form(...)],
|
||||
code: Annotated[str, Form(...)],
|
||||
folder_id: Annotated[Optional[str], Form()] = None,
|
||||
):
|
||||
"""
|
||||
Exchange an authorization code for refresh and access tokens from Google.
|
||||
This is done on the server to avoid exposing client secret in the browser.
|
||||
"""
|
||||
# Prepare the token request
|
||||
token_url = "https://oauth2.googleapis.com/token"
|
||||
|
||||
payload = {
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"code": code,
|
||||
"redirect_uri": redirect_uri,
|
||||
"grant_type": "authorization_code",
|
||||
}
|
||||
|
||||
# Use shared OAuth helper (handles secure logging and error handling)
|
||||
token_data = exchange_oauth_token(provider_name="Google Drive", token_url=token_url, payload=payload)
|
||||
|
||||
# Return just what's needed by the frontend
|
||||
return {
|
||||
"refresh_token": token_data["refresh_token"],
|
||||
"access_token": token_data["access_token"],
|
||||
"expires_in": token_data.get("expires_in", 3600),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/google-drive/update-settings")
|
||||
@require_login
|
||||
async def update_google_drive_settings(
|
||||
request: Request,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
client_id: Annotated[Optional[str], Form()] = None,
|
||||
client_secret: Annotated[Optional[str], Form()] = None,
|
||||
folder_id: Annotated[Optional[str], Form()] = None,
|
||||
use_oauth: Annotated[str, Form()] = "true",
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Update Google Drive settings in memory and persist to database
|
||||
"""
|
||||
try:
|
||||
logger.info("Updating Google Drive settings in memory and database")
|
||||
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
|
||||
)
|
||||
|
||||
# Convert use_oauth string to boolean
|
||||
use_oauth_bool = use_oauth.lower() in ("true", "1", "yes", "y", "t")
|
||||
|
||||
# Update settings in memory and persist to database
|
||||
if refresh_token:
|
||||
settings.google_drive_refresh_token = refresh_token
|
||||
save_setting_to_db(db, "google_drive_refresh_token", refresh_token, changed_by=changed_by)
|
||||
logger.info("Updated GOOGLE_DRIVE_REFRESH_TOKEN in memory and database")
|
||||
|
||||
if client_id:
|
||||
settings.google_drive_client_id = client_id
|
||||
save_setting_to_db(db, "google_drive_client_id", client_id, changed_by=changed_by)
|
||||
logger.info("Updated GOOGLE_DRIVE_CLIENT_ID in memory and database")
|
||||
|
||||
if client_secret:
|
||||
settings.google_drive_client_secret = client_secret
|
||||
save_setting_to_db(db, "google_drive_client_secret", client_secret, changed_by=changed_by)
|
||||
logger.info("Updated GOOGLE_DRIVE_CLIENT_SECRET in memory and database")
|
||||
|
||||
if folder_id:
|
||||
settings.google_drive_folder_id = folder_id
|
||||
save_setting_to_db(db, "google_drive_folder_id", folder_id, changed_by=changed_by)
|
||||
logger.info("Updated GOOGLE_DRIVE_FOLDER_ID in memory and database")
|
||||
|
||||
# Set the OAuth flag
|
||||
settings.google_drive_use_oauth = use_oauth_bool
|
||||
save_setting_to_db(
|
||||
db,
|
||||
"google_drive_use_oauth",
|
||||
str(use_oauth_bool).lower(),
|
||||
changed_by=changed_by,
|
||||
)
|
||||
logger.info(f"Updated GOOGLE_DRIVE_USE_OAUTH in memory and database to {use_oauth_bool}")
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Google Drive settings have been updated in memory and database",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error updating Google Drive settings: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to update Google Drive settings: {str(e)}",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/google-drive/test-token")
|
||||
@require_login
|
||||
async def test_google_drive_token(request: Request):
|
||||
"""
|
||||
Test if the configured Google Drive token is valid.
|
||||
Tests both OAuth and service account approaches based on configuration.
|
||||
"""
|
||||
try:
|
||||
from app.tasks.upload_to_google_drive import get_drive_service_oauth, get_google_drive_service
|
||||
|
||||
logger.info("Testing Google Drive token validity")
|
||||
|
||||
# Check if OAuth is enabled and configured
|
||||
if getattr(settings, "google_drive_use_oauth", False):
|
||||
if not (
|
||||
settings.google_drive_client_id
|
||||
and settings.google_drive_client_secret
|
||||
and settings.google_drive_refresh_token
|
||||
):
|
||||
logger.warning("Google Drive OAuth credentials not fully configured")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Google Drive OAuth credentials are not fully configured",
|
||||
}
|
||||
|
||||
try:
|
||||
# Test OAuth connection
|
||||
service = get_drive_service_oauth()
|
||||
|
||||
# Get credentials for checking token validity
|
||||
import google.oauth2.credentials
|
||||
from google.auth.transport.requests import Request
|
||||
|
||||
credentials = google.oauth2.credentials.Credentials(
|
||||
token=None,
|
||||
refresh_token=settings.google_drive_refresh_token,
|
||||
token_uri="https://oauth2.googleapis.com/token",
|
||||
client_id=settings.google_drive_client_id,
|
||||
client_secret=settings.google_drive_client_secret,
|
||||
)
|
||||
|
||||
# Force a refresh to update the token expiration
|
||||
if not credentials.valid:
|
||||
credentials.refresh(Request())
|
||||
|
||||
# Get token expiration info
|
||||
expiration_info = {}
|
||||
if hasattr(credentials, "expiry") and credentials.expiry:
|
||||
now = datetime.now()
|
||||
expiry = credentials.expiry
|
||||
time_left = expiry - now
|
||||
expiration_info = {
|
||||
"expires_at": expiry.isoformat(),
|
||||
"expires_in_seconds": max(0, int(time_left.total_seconds())),
|
||||
"expires_in_human": format_time_remaining(time_left),
|
||||
}
|
||||
|
||||
# Test basic API operation
|
||||
about = service.about().get(fields="user").execute()
|
||||
user_email = about.get("user", {}).get("emailAddress", "Unknown")
|
||||
|
||||
logger.info(f"Successfully connected to Google Drive as {user_email}")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"OAuth token is valid! Connected as {user_email}",
|
||||
"account": user_email,
|
||||
"auth_type": "oauth",
|
||||
"token_info": expiration_info,
|
||||
}
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
logger.error(f"Google Drive OAuth token test failed: {error_msg}")
|
||||
|
||||
# Check if this is a token-related error
|
||||
if "invalid_grant" in error_msg.lower() or "token" in error_msg.lower():
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"OAuth token validation failed: {error_msg}",
|
||||
"needs_reauth": True,
|
||||
}
|
||||
return {"status": "error", "message": f"Connection error: {error_msg}"}
|
||||
else:
|
||||
# Test service account connection
|
||||
if not settings.google_drive_credentials_json:
|
||||
logger.warning("Google Drive service account credentials not configured")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Google Drive service account credentials are not configured",
|
||||
}
|
||||
|
||||
try:
|
||||
service = get_google_drive_service()
|
||||
about = service.about().get(fields="user").execute()
|
||||
|
||||
# For service accounts, try to show the delegated user if available
|
||||
user_email = about.get("user", {}).get("emailAddress", "Unknown")
|
||||
delegated_user = getattr(settings, "google_drive_delegate_to", None)
|
||||
|
||||
if delegated_user:
|
||||
user_display = f"{user_email} (delegating as {delegated_user})"
|
||||
else:
|
||||
user_display = user_email
|
||||
|
||||
logger.info(f"Successfully connected to Google Drive using service account as {user_display}")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Service account is valid! Connected as {user_display}",
|
||||
"account": user_email,
|
||||
"auth_type": "service_account",
|
||||
}
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
logger.error(f"Google Drive service account test failed: {error_msg}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Service account validation failed: {error_msg}",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Unexpected error testing Google Drive token")
|
||||
return {"status": "error", "message": f"Unexpected error: {str(e)}"}
|
||||
|
||||
|
||||
@router.get("/google-drive/get-token-info")
|
||||
@require_login
|
||||
async def get_google_drive_token_info(request: Request):
|
||||
"""
|
||||
Get information about the current Google Drive token.
|
||||
Returns the access token if one exists and is valid.
|
||||
Used by the frontend to access the Google Picker API.
|
||||
"""
|
||||
try:
|
||||
logger.info("Getting Google Drive token information")
|
||||
|
||||
# Check if OAuth is enabled and configured
|
||||
if not getattr(settings, "google_drive_use_oauth", False):
|
||||
logger.warning("OAuth is not enabled, using service account instead")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "OAuth is not enabled. Service accounts don't support user-facing features.",
|
||||
}
|
||||
|
||||
if not (
|
||||
settings.google_drive_client_id
|
||||
and settings.google_drive_client_secret
|
||||
and settings.google_drive_refresh_token
|
||||
):
|
||||
logger.warning("Google Drive OAuth credentials not fully configured")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Google Drive OAuth credentials are not fully configured",
|
||||
}
|
||||
|
||||
try:
|
||||
# Get credentials and access token
|
||||
import google.oauth2.credentials
|
||||
from google.auth.transport.requests import Request
|
||||
|
||||
credentials = google.oauth2.credentials.Credentials(
|
||||
token=None,
|
||||
refresh_token=settings.google_drive_refresh_token,
|
||||
token_uri="https://oauth2.googleapis.com/token",
|
||||
client_id=settings.google_drive_client_id,
|
||||
client_secret=settings.google_drive_client_secret,
|
||||
)
|
||||
|
||||
# Force a refresh to get a fresh access token
|
||||
if not credentials.valid:
|
||||
credentials.refresh(Request())
|
||||
|
||||
# Get token expiration info
|
||||
expiration_info = {}
|
||||
if hasattr(credentials, "expiry") and credentials.expiry:
|
||||
now = datetime.now()
|
||||
expiry = credentials.expiry
|
||||
time_left = expiry - now
|
||||
expiration_info = {
|
||||
"expires_at": expiry.isoformat(),
|
||||
"expires_in_seconds": max(0, int(time_left.total_seconds())),
|
||||
"expires_in_human": format_time_remaining(time_left),
|
||||
}
|
||||
|
||||
# Return the token info
|
||||
logger.info("Successfully retrieved Google Drive access token")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Access token successfully retrieved",
|
||||
"access_token": credentials.token,
|
||||
"token_info": expiration_info,
|
||||
}
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
logger.error(f"Failed to get Google Drive token: {error_msg}")
|
||||
|
||||
# Check if this is a token-related error
|
||||
if "invalid_grant" in error_msg.lower() or "token" in error_msg.lower():
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"OAuth token retrieval failed: {error_msg}",
|
||||
"needs_reauth": True,
|
||||
}
|
||||
return {"status": "error", "message": f"Token retrieval error: {error_msg}"}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Unexpected error getting Google Drive token info")
|
||||
return {"status": "error", "message": f"Unexpected error: {str(e)}"}
|
||||
|
||||
|
||||
def format_time_remaining(time_delta):
|
||||
"""Format a timedelta into a human-readable string."""
|
||||
if time_delta.total_seconds() <= 0:
|
||||
return "Expired"
|
||||
|
||||
days = time_delta.days
|
||||
hours, remainder = divmod(time_delta.seconds, 3600)
|
||||
minutes, seconds = divmod(remainder, 60)
|
||||
|
||||
parts = []
|
||||
if days > 0:
|
||||
parts.append(f"{days} day{'s' if days != 1 else ''}")
|
||||
if hours > 0:
|
||||
parts.append(f"{hours} hour{'s' if hours != 1 else ''}")
|
||||
if minutes > 0 and days == 0: # Only show minutes if less than a day
|
||||
parts.append(f"{minutes} minute{'s' if minutes != 1 else ''}")
|
||||
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
@router.post("/google-drive/save-settings")
|
||||
@require_login
|
||||
async def save_dropbox_settings(
|
||||
request: Request,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
client_id: Annotated[Optional[str], Form()] = None,
|
||||
client_secret: Annotated[Optional[str], Form()] = None,
|
||||
folder_id: Annotated[Optional[str], Form()] = None,
|
||||
use_oauth: Annotated[str, Form()] = "true",
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Save Google Drive settings to the .env file (best-effort) and persist to database.
|
||||
"""
|
||||
try:
|
||||
# Get the path to the .env file
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
|
||||
# Convert use_oauth string to boolean
|
||||
use_oauth_bool = use_oauth.lower() in ("true", "1", "yes", "y", "t")
|
||||
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
|
||||
)
|
||||
|
||||
# Define settings to update
|
||||
drive_settings = {"GOOGLE_DRIVE_USE_OAUTH": str(use_oauth_bool).lower()}
|
||||
|
||||
# Only update these if provided
|
||||
if use_oauth_bool:
|
||||
if refresh_token:
|
||||
drive_settings["GOOGLE_DRIVE_REFRESH_TOKEN"] = refresh_token
|
||||
if client_id:
|
||||
drive_settings["GOOGLE_DRIVE_CLIENT_ID"] = client_id
|
||||
if client_secret:
|
||||
drive_settings["GOOGLE_DRIVE_CLIENT_SECRET"] = client_secret
|
||||
|
||||
# Always include folder ID if provided
|
||||
if folder_id:
|
||||
drive_settings["GOOGLE_DRIVE_FOLDER_ID"] = folder_id
|
||||
|
||||
# Try to update the .env file, but don't fail if it doesn't exist (for Docker containers)
|
||||
if os.path.exists(env_path):
|
||||
try:
|
||||
logger.info(f"Updating Google Drive settings in {env_path}")
|
||||
|
||||
# Read the current .env file
|
||||
with open(env_path, "r") as f:
|
||||
env_lines = f.readlines()
|
||||
|
||||
# Process each line and update or add settings
|
||||
updated = set()
|
||||
new_env_lines = []
|
||||
for line in env_lines:
|
||||
stripped_line = line.rstrip()
|
||||
is_updated = False
|
||||
for key, value in drive_settings.items():
|
||||
if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="):
|
||||
# Uncomment if commented out - check the original stripped line
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
updated.add(key)
|
||||
is_updated = True
|
||||
break
|
||||
if not is_updated:
|
||||
new_env_lines.append(stripped_line)
|
||||
|
||||
# Add any settings that weren't updated (they weren't in the file)
|
||||
for key, value in drive_settings.items():
|
||||
if key not in updated:
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
|
||||
# Write the updated .env file
|
||||
with open(env_path, "w") as f:
|
||||
f.write("\n".join(new_env_lines) + "\n")
|
||||
|
||||
logger.info("Successfully updated Google Drive settings in .env file")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to update .env file: {str(e)}, but will continue with in-memory update")
|
||||
else:
|
||||
logger.warning(
|
||||
f".env file not found at {env_path}, skipping file update but continuing with in-memory update"
|
||||
)
|
||||
|
||||
# Update the settings in memory (this always happens)
|
||||
if refresh_token:
|
||||
settings.google_drive_refresh_token = refresh_token
|
||||
if client_id:
|
||||
settings.google_drive_client_id = client_id
|
||||
if client_secret:
|
||||
settings.google_drive_client_secret = client_secret
|
||||
if folder_id:
|
||||
settings.google_drive_folder_id = folder_id
|
||||
|
||||
# Set OAuth flag
|
||||
settings.google_drive_use_oauth = use_oauth_bool
|
||||
|
||||
# Persist to database
|
||||
save_setting_to_db(
|
||||
db,
|
||||
"google_drive_use_oauth",
|
||||
str(use_oauth_bool).lower(),
|
||||
changed_by=changed_by,
|
||||
)
|
||||
if refresh_token:
|
||||
save_setting_to_db(db, "google_drive_refresh_token", refresh_token, changed_by=changed_by)
|
||||
if client_id:
|
||||
save_setting_to_db(db, "google_drive_client_id", client_id, changed_by=changed_by)
|
||||
if client_secret:
|
||||
save_setting_to_db(db, "google_drive_client_secret", client_secret, changed_by=changed_by)
|
||||
if folder_id:
|
||||
save_setting_to_db(db, "google_drive_folder_id", folder_id, changed_by=changed_by)
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
logger.info("Successfully updated Google Drive settings in memory and database")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Google Drive settings have been saved",
|
||||
"in_memory_only": not os.path.exists(env_path),
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error saving Google Drive settings: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to save Google Drive settings: {str(e)}",
|
||||
)
|
||||
-158
@@ -1,158 +0,0 @@
|
||||
"""
|
||||
Processing logs API endpoints
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Annotated, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from sqlalchemy import desc
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.database import get_db
|
||||
from app.models import FileRecord, ProcessingLog
|
||||
from app.utils.input_validation import validate_task_id
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
|
||||
@router.get("/logs")
|
||||
@require_login
|
||||
def list_processing_logs(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
file_id: Optional[int] = Query(None, description="Filter by file ID"),
|
||||
task_id: Optional[str] = Query(None, description="Filter by task ID"),
|
||||
limit: int = Query(100, ge=1, le=1000, description="Number of logs to return"),
|
||||
):
|
||||
"""
|
||||
Returns a JSON list of ProcessingLog entries.
|
||||
Protected by `@require_login`, so only logged-in sessions can access.
|
||||
|
||||
Query Parameters:
|
||||
- file_id: Optional filter by file ID
|
||||
- task_id: Optional filter by task ID
|
||||
- limit: Maximum number of logs to return (default 100, max 1000)
|
||||
|
||||
Example response:
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"file_id": 123,
|
||||
"task_id": "abc-123-def",
|
||||
"step_name": "process_document",
|
||||
"status": "success",
|
||||
"message": "Processing completed",
|
||||
"timestamp": "2025-05-01T12:34:56.789000"
|
||||
},
|
||||
...
|
||||
]
|
||||
"""
|
||||
query = db.query(ProcessingLog)
|
||||
|
||||
# Apply filters
|
||||
if file_id is not None:
|
||||
query = query.filter(ProcessingLog.file_id == file_id)
|
||||
if task_id is not None:
|
||||
validate_task_id(task_id)
|
||||
query = query.filter(ProcessingLog.task_id == task_id)
|
||||
|
||||
# Order by timestamp descending and limit
|
||||
logs = query.order_by(desc(ProcessingLog.timestamp)).limit(limit).all()
|
||||
|
||||
# Return a simple list of dicts
|
||||
result = []
|
||||
for log in logs:
|
||||
result.append(
|
||||
{
|
||||
"id": log.id,
|
||||
"file_id": log.file_id,
|
||||
"task_id": log.task_id,
|
||||
"step_name": log.step_name,
|
||||
"status": log.status,
|
||||
"message": log.message,
|
||||
"timestamp": log.timestamp.isoformat() if log.timestamp else None,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/logs/file/{file_id}")
|
||||
@require_login
|
||||
def get_file_processing_logs(request: Request, file_id: int, db: DbSession):
|
||||
"""
|
||||
Get all processing logs for a specific file.
|
||||
Returns logs ordered by timestamp (oldest first to show processing flow).
|
||||
|
||||
Also includes file metadata if the file exists.
|
||||
"""
|
||||
# Check if file exists
|
||||
file_record = db.query(FileRecord).filter(FileRecord.id == file_id).first()
|
||||
if not file_record:
|
||||
raise HTTPException(status_code=404, detail=f"File with ID {file_id} not found")
|
||||
|
||||
# Get all logs for this file
|
||||
logs = db.query(ProcessingLog).filter(ProcessingLog.file_id == file_id).order_by(ProcessingLog.timestamp).all()
|
||||
|
||||
# Build response
|
||||
log_list = []
|
||||
for log in logs:
|
||||
log_list.append(
|
||||
{
|
||||
"id": log.id,
|
||||
"task_id": log.task_id,
|
||||
"step_name": log.step_name,
|
||||
"status": log.status,
|
||||
"message": log.message,
|
||||
"timestamp": log.timestamp.isoformat() if log.timestamp else None,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"file": {
|
||||
"id": file_record.id,
|
||||
"original_filename": file_record.original_filename,
|
||||
"file_size": file_record.file_size,
|
||||
"mime_type": file_record.mime_type,
|
||||
"created_at": file_record.created_at.isoformat() if file_record.created_at else None,
|
||||
},
|
||||
"logs": log_list,
|
||||
"total_logs": len(log_list),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/logs/task/{task_id}")
|
||||
@require_login
|
||||
def get_task_processing_logs(request: Request, task_id: str, db: DbSession):
|
||||
"""
|
||||
Get all processing logs for a specific task.
|
||||
Returns logs ordered by timestamp (oldest first to show processing flow).
|
||||
"""
|
||||
validate_task_id(task_id)
|
||||
# Get all logs for this task
|
||||
logs = db.query(ProcessingLog).filter(ProcessingLog.task_id == task_id).order_by(ProcessingLog.timestamp).all()
|
||||
|
||||
if not logs:
|
||||
raise HTTPException(status_code=404, detail=f"No logs found for task {task_id}")
|
||||
|
||||
# Build response
|
||||
log_list = []
|
||||
for log in logs:
|
||||
log_list.append(
|
||||
{
|
||||
"id": log.id,
|
||||
"file_id": log.file_id,
|
||||
"step_name": log.step_name,
|
||||
"status": log.status,
|
||||
"message": log.message,
|
||||
"timestamp": log.timestamp.isoformat() if log.timestamp else None,
|
||||
}
|
||||
)
|
||||
|
||||
return {"task_id": task_id, "logs": log_list, "total_logs": len(log_list)}
|
||||
@@ -1,438 +0,0 @@
|
||||
"""
|
||||
OneDrive API endpoints
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Annotated, Optional
|
||||
|
||||
import requests
|
||||
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.utils.oauth_helper import exchange_oauth_token
|
||||
from app.utils.settings_service import save_setting_to_db
|
||||
from app.utils.settings_sync import notify_settings_updated
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/onedrive/exchange-token")
|
||||
@require_login
|
||||
async def exchange_onedrive_token(
|
||||
request: Request,
|
||||
client_id: Annotated[str, Form(...)],
|
||||
client_secret: Annotated[str, Form(...)],
|
||||
redirect_uri: Annotated[str, Form(...)],
|
||||
code: Annotated[str, Form(...)],
|
||||
tenant_id: Annotated[str, Form(...)],
|
||||
):
|
||||
"""
|
||||
Exchange an authorization code for a refresh token.
|
||||
This is done on the server to avoid exposing client secret in the browser.
|
||||
"""
|
||||
# Prepare the token request
|
||||
token_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"
|
||||
|
||||
payload = {
|
||||
"client_id": client_id,
|
||||
"scope": "https://graph.microsoft.com/.default offline_access",
|
||||
"code": code,
|
||||
"redirect_uri": redirect_uri,
|
||||
"grant_type": "authorization_code",
|
||||
"client_secret": client_secret,
|
||||
}
|
||||
|
||||
# Use shared OAuth helper (handles secure logging and error handling)
|
||||
token_data = exchange_oauth_token(provider_name="OneDrive", token_url=token_url, payload=payload)
|
||||
|
||||
# Return just what's needed by the frontend
|
||||
return {
|
||||
"refresh_token": token_data["refresh_token"],
|
||||
"expires_in": token_data.get("expires_in", 3600),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/onedrive/test-token")
|
||||
@require_login
|
||||
async def test_onedrive_token(request: Request):
|
||||
"""
|
||||
Test if the configured OneDrive token is valid and return expiration information.
|
||||
"""
|
||||
try:
|
||||
logger.info("Testing OneDrive token validity")
|
||||
|
||||
if (
|
||||
not settings.onedrive_refresh_token
|
||||
or not settings.onedrive_client_id
|
||||
or not settings.onedrive_client_secret
|
||||
):
|
||||
logger.warning("OneDrive credentials not fully configured")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "OneDrive credentials are not fully configured",
|
||||
}
|
||||
|
||||
# Refresh token to get a new access token and expiration info
|
||||
tenant_id = settings.onedrive_tenant_id or "common"
|
||||
token_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"
|
||||
|
||||
refresh_data = {
|
||||
"client_id": settings.onedrive_client_id,
|
||||
"client_secret": settings.onedrive_client_secret,
|
||||
"refresh_token": settings.onedrive_refresh_token,
|
||||
"grant_type": "refresh_token",
|
||||
"scope": "offline_access Files.ReadWrite",
|
||||
}
|
||||
|
||||
response = requests.post(token_url, data=refresh_data, timeout=settings.http_request_timeout)
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.error(f"Failed to refresh OneDrive token: {response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "Refresh token has expired or is invalid",
|
||||
"needs_reauth": True,
|
||||
}
|
||||
|
||||
token_data = response.json()
|
||||
access_token = token_data.get("access_token")
|
||||
expires_in = token_data.get("expires_in", 3600) # Default to 1 hour if not specified
|
||||
|
||||
# Check if we got a new refresh token (Microsoft sometimes issues a new one)
|
||||
new_refresh_token = token_data.get("refresh_token")
|
||||
if new_refresh_token and new_refresh_token != settings.onedrive_refresh_token:
|
||||
logger.info("Received new refresh token from Microsoft - will update configuration")
|
||||
|
||||
# Update refresh token in memory
|
||||
settings.onedrive_refresh_token = new_refresh_token
|
||||
|
||||
# Also try to update .env file if it exists
|
||||
try:
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
if os.path.exists(env_path):
|
||||
with open(env_path, "r") as f:
|
||||
env_lines = f.readlines()
|
||||
|
||||
updated_lines = []
|
||||
updated = False
|
||||
|
||||
for line in env_lines:
|
||||
if line.startswith("ONEDRIVE_REFRESH_TOKEN="):
|
||||
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
|
||||
updated = True
|
||||
else:
|
||||
updated_lines.append(line)
|
||||
|
||||
if not updated:
|
||||
updated_lines.append(f"ONEDRIVE_REFRESH_TOKEN={new_refresh_token}\n")
|
||||
|
||||
with open(env_path, "w") as f:
|
||||
f.writelines(updated_lines)
|
||||
|
||||
logger.info("Updated refresh token in .env file")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to update refresh token in .env file: {e}")
|
||||
|
||||
# Persist the rotated refresh token to the database
|
||||
try:
|
||||
from app.database import SessionLocal
|
||||
|
||||
_db = SessionLocal()
|
||||
try:
|
||||
save_setting_to_db(
|
||||
_db,
|
||||
"onedrive_refresh_token",
|
||||
new_refresh_token,
|
||||
changed_by="onedrive_token_rotation",
|
||||
)
|
||||
notify_settings_updated()
|
||||
finally:
|
||||
_db.close()
|
||||
except Exception as _e:
|
||||
logger.warning(f"Failed to persist rotated OneDrive refresh token to database: {_e}")
|
||||
|
||||
# Test the access token by getting user information
|
||||
user_info_url = "https://graph.microsoft.com/v1.0/me"
|
||||
headers = {"Authorization": f"Bearer {access_token}"}
|
||||
|
||||
user_response = requests.get(user_info_url, headers=headers, timeout=settings.http_request_timeout)
|
||||
|
||||
if user_response.status_code != 200:
|
||||
logger.error(f"OneDrive token test failed: {user_response.status_code} {user_response.text}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Token validation failed with status {user_response.status_code}: {user_response.text}",
|
||||
}
|
||||
|
||||
# Get user info
|
||||
user_info = user_response.json()
|
||||
display_name = user_info.get("displayName", "Unknown user")
|
||||
email = user_info.get("userPrincipalName", "Unknown email")
|
||||
|
||||
# Calculate expiration time
|
||||
now = datetime.now()
|
||||
expiry_time = now + timedelta(seconds=expires_in)
|
||||
|
||||
# Format expiration info
|
||||
time_left = expiry_time - now
|
||||
token_info = {
|
||||
"expires_at": expiry_time.isoformat(),
|
||||
"expires_in_seconds": expires_in,
|
||||
"expires_in_human": format_time_remaining(time_left),
|
||||
"refresh_token_validity": "Refresh token is valid for 90 days of inactivity",
|
||||
}
|
||||
|
||||
logger.info(f"Successfully connected to OneDrive as {email}")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "OneDrive connection successful",
|
||||
"account": email,
|
||||
"account_name": display_name,
|
||||
"token_info": token_info,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error testing OneDrive token: {str(e)}")
|
||||
return {"status": "error", "message": f"Connection error: {str(e)}"}
|
||||
|
||||
|
||||
def format_time_remaining(time_delta):
|
||||
"""Format a timedelta into a human-readable string."""
|
||||
if time_delta.total_seconds() <= 0:
|
||||
return "Expired"
|
||||
|
||||
days = time_delta.days
|
||||
hours, remainder = divmod(time_delta.seconds, 3600)
|
||||
minutes, seconds = divmod(remainder, 60)
|
||||
|
||||
parts = []
|
||||
if days > 0:
|
||||
parts.append(f"{days} day{'s' if days != 1 else ''}")
|
||||
if hours > 0:
|
||||
parts.append(f"{hours} hour{'s' if hours != 1 else ''}")
|
||||
if minutes > 0 and days == 0: # Only show minutes if less than a day
|
||||
parts.append(f"{minutes} minute{'s' if minutes != 1 else ''}")
|
||||
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
@router.post("/onedrive/save-settings")
|
||||
@require_login
|
||||
async def save_onedrive_settings(
|
||||
request: Request,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
client_id: Annotated[Optional[str], Form()] = None,
|
||||
client_secret: Annotated[Optional[str], Form()] = None,
|
||||
tenant_id: Annotated[str, Form()] = "common",
|
||||
folder_path: Annotated[Optional[str], Form()] = None,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Saves to database (primary) and .env file (best-effort).
|
||||
"""
|
||||
try:
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
|
||||
)
|
||||
|
||||
# Best-effort .env file write
|
||||
try:
|
||||
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".env")
|
||||
if not os.path.exists(env_path):
|
||||
logger.warning(f".env file not found at {env_path}, skipping file write")
|
||||
else:
|
||||
logger.info(f"Updating OneDrive settings in {env_path}")
|
||||
|
||||
with open(env_path, "r") as f:
|
||||
env_lines = f.readlines()
|
||||
|
||||
onedrive_settings = {"ONEDRIVE_REFRESH_TOKEN": refresh_token}
|
||||
if client_id:
|
||||
onedrive_settings["ONEDRIVE_CLIENT_ID"] = client_id
|
||||
if client_secret:
|
||||
onedrive_settings["ONEDRIVE_CLIENT_SECRET"] = client_secret
|
||||
if tenant_id:
|
||||
onedrive_settings["ONEDRIVE_TENANT_ID"] = tenant_id
|
||||
if folder_path:
|
||||
onedrive_settings["ONEDRIVE_FOLDER_PATH"] = folder_path
|
||||
|
||||
updated = set()
|
||||
new_env_lines = []
|
||||
for line in env_lines:
|
||||
stripped_line = line.rstrip()
|
||||
is_updated = False
|
||||
for key, value in onedrive_settings.items():
|
||||
if stripped_line.startswith(f"{key}=") or stripped_line.startswith(f"# {key}="):
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
updated.add(key)
|
||||
is_updated = True
|
||||
break
|
||||
if not is_updated:
|
||||
new_env_lines.append(stripped_line)
|
||||
|
||||
for key, value in onedrive_settings.items():
|
||||
if key not in updated:
|
||||
new_env_lines.append(f"{key}={value}")
|
||||
|
||||
with open(env_path, "w") as f:
|
||||
f.write("\n".join(new_env_lines) + "\n")
|
||||
|
||||
logger.info("Successfully updated OneDrive settings in .env file")
|
||||
except Exception as env_err:
|
||||
logger.warning(f"Failed to write .env file (non-fatal): {env_err}")
|
||||
|
||||
# Update the settings in memory
|
||||
if refresh_token:
|
||||
settings.onedrive_refresh_token = refresh_token
|
||||
if client_id:
|
||||
settings.onedrive_client_id = client_id
|
||||
if client_secret:
|
||||
settings.onedrive_client_secret = client_secret
|
||||
if tenant_id:
|
||||
settings.onedrive_tenant_id = tenant_id
|
||||
if folder_path:
|
||||
settings.onedrive_folder_path = folder_path
|
||||
|
||||
# Persist to database (primary)
|
||||
if refresh_token:
|
||||
save_setting_to_db(db, "onedrive_refresh_token", refresh_token, changed_by=changed_by)
|
||||
if client_id:
|
||||
save_setting_to_db(db, "onedrive_client_id", client_id, changed_by=changed_by)
|
||||
if client_secret:
|
||||
save_setting_to_db(db, "onedrive_client_secret", client_secret, changed_by=changed_by)
|
||||
if tenant_id:
|
||||
save_setting_to_db(db, "onedrive_tenant_id", tenant_id, changed_by=changed_by)
|
||||
if folder_path:
|
||||
save_setting_to_db(db, "onedrive_folder_path", folder_path, changed_by=changed_by)
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
logger.info("Successfully saved OneDrive settings")
|
||||
return {"status": "success", "message": "OneDrive settings have been saved"}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error saving OneDrive settings: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to save OneDrive settings: {str(e)}",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/onedrive/update-settings")
|
||||
@require_login
|
||||
async def update_onedrive_settings(
|
||||
request: Request,
|
||||
refresh_token: Annotated[str, Form(...)],
|
||||
client_id: Annotated[Optional[str], Form()] = None,
|
||||
client_secret: Annotated[Optional[str], Form()] = None,
|
||||
tenant_id: Annotated[str, Form()] = "common",
|
||||
folder_path: Annotated[Optional[str], Form()] = None,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Update OneDrive settings in memory and persist to database
|
||||
"""
|
||||
try:
|
||||
logger.info("Updating OneDrive settings in memory and database")
|
||||
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "wizard"
|
||||
)
|
||||
|
||||
# Update settings in memory and persist to database
|
||||
if refresh_token:
|
||||
settings.onedrive_refresh_token = refresh_token
|
||||
save_setting_to_db(db, "onedrive_refresh_token", refresh_token, changed_by=changed_by)
|
||||
logger.info("Updated ONEDRIVE_REFRESH_TOKEN in memory and database")
|
||||
|
||||
if client_id:
|
||||
settings.onedrive_client_id = client_id
|
||||
save_setting_to_db(db, "onedrive_client_id", client_id, changed_by=changed_by)
|
||||
logger.info("Updated ONEDRIVE_CLIENT_ID in memory and database")
|
||||
|
||||
if client_secret:
|
||||
settings.onedrive_client_secret = client_secret
|
||||
save_setting_to_db(db, "onedrive_client_secret", client_secret, changed_by=changed_by)
|
||||
logger.info("Updated ONEDRIVE_CLIENT_SECRET in memory and database")
|
||||
|
||||
if tenant_id:
|
||||
settings.onedrive_tenant_id = tenant_id
|
||||
save_setting_to_db(db, "onedrive_tenant_id", tenant_id, changed_by=changed_by)
|
||||
logger.info("Updated ONEDRIVE_TENANT_ID in memory and database")
|
||||
|
||||
if folder_path:
|
||||
settings.onedrive_folder_path = folder_path
|
||||
save_setting_to_db(db, "onedrive_folder_path", folder_path, changed_by=changed_by)
|
||||
logger.info("Updated ONEDRIVE_FOLDER_PATH in memory and database")
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
# Test the token to make sure it works
|
||||
try:
|
||||
from app.tasks.upload_to_onedrive import get_onedrive_token
|
||||
|
||||
get_onedrive_token() # Test that token can be retrieved
|
||||
logger.info("Successfully tested OneDrive token")
|
||||
except Exception as e:
|
||||
logger.error(f"Token test failed after updating settings: {str(e)}")
|
||||
return {
|
||||
"status": "warning",
|
||||
"message": "Settings updated but token test failed: " + str(e),
|
||||
}
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "OneDrive settings have been updated in memory and database",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error updating OneDrive settings: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to update OneDrive settings: {str(e)}",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/onedrive/get-full-config")
|
||||
@require_login
|
||||
async def get_onedrive_full_config(request: Request):
|
||||
"""
|
||||
Get the full OneDrive configuration for sharing with worker nodes
|
||||
"""
|
||||
try:
|
||||
# Create a configuration object with all OneDrive settings
|
||||
config = {
|
||||
"client_id": settings.onedrive_client_id or "",
|
||||
"client_secret": settings.onedrive_client_secret or "",
|
||||
"tenant_id": settings.onedrive_tenant_id or "common",
|
||||
"refresh_token": settings.onedrive_refresh_token or "",
|
||||
"folder_path": settings.onedrive_folder_path or "Documents/Uploads",
|
||||
}
|
||||
|
||||
# Generate environment variable format
|
||||
env_format = "\n".join(
|
||||
[
|
||||
f"ONEDRIVE_CLIENT_ID={config['client_id']}",
|
||||
f"ONEDRIVE_CLIENT_SECRET={config['client_secret']}",
|
||||
f"ONEDRIVE_TENANT_ID={config['tenant_id']}",
|
||||
f"ONEDRIVE_REFRESH_TOKEN={config['refresh_token']}",
|
||||
f"ONEDRIVE_FOLDER_PATH={config['folder_path']}",
|
||||
]
|
||||
)
|
||||
|
||||
return {"status": "success", "config": config, "env_format": env_format}
|
||||
except Exception as e:
|
||||
logger.exception("Error getting OneDrive configuration")
|
||||
return {"status": "error", "message": str(e)}
|
||||
@@ -1,334 +0,0 @@
|
||||
"""
|
||||
AI provider and OpenAI API endpoints.
|
||||
|
||||
Exposes three endpoints:
|
||||
- GET /api/ai/test – tests the currently configured AI provider (generic, provider-agnostic)
|
||||
- GET /api/openai/test – backward-compatible alias that tests the OpenAI API specifically
|
||||
- POST /api/ai/test-extraction – runs the metadata-extraction prompt against the configured AI provider
|
||||
with caller-supplied plaintext and returns the raw response, parsed JSON,
|
||||
and extracted tags so operators can evaluate model quality.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# Maximum number of characters accepted for a test-extraction request.
|
||||
# Keeps individual requests reasonable without blocking any real-world document.
|
||||
_MAX_EXTRACTION_TEXT_LEN = 50_000
|
||||
|
||||
|
||||
def _get_exception_chain_detail(exc: Exception) -> str:
|
||||
"""
|
||||
Extract a verbose diagnostic message by walking the full exception chain.
|
||||
|
||||
Surfaces DNS resolution failures, TCP connection refused errors, SSL issues,
|
||||
and other low-level network problems that are normally hidden behind a generic
|
||||
'Connection error.' message.
|
||||
"""
|
||||
parts: list[str] = [str(exc)]
|
||||
|
||||
cause = getattr(exc, "__cause__", None) or getattr(exc, "__context__", None)
|
||||
seen: set[int] = {id(exc)}
|
||||
while cause is not None and id(cause) not in seen:
|
||||
seen.add(id(cause))
|
||||
cause_str = str(cause)
|
||||
if cause_str and cause_str not in parts:
|
||||
parts.append(f"caused by: {type(cause).__name__}: {cause_str}")
|
||||
cause = getattr(cause, "__cause__", None) or getattr(cause, "__context__", None)
|
||||
|
||||
return " | ".join(parts)
|
||||
|
||||
|
||||
@router.get("/openai/test")
|
||||
@require_login
|
||||
async def test_openai_connection(request: Request):
|
||||
"""
|
||||
Test if the configured OpenAI API key is valid.
|
||||
"""
|
||||
try:
|
||||
import openai
|
||||
|
||||
logger.info("Testing OpenAI API key validity")
|
||||
|
||||
# Check if API key is configured
|
||||
if not settings.openai_api_key:
|
||||
logger.warning("No OpenAI API key configured")
|
||||
return {"status": "error", "message": "No OpenAI API key is configured"}
|
||||
|
||||
# Configure the client, explicitly passing base_url so the sanitized
|
||||
# value from Settings (strip_outer_quotes) is used instead of the raw
|
||||
# OPENAI_BASE_URL env var which may contain literal quote characters.
|
||||
client = openai.OpenAI(api_key=settings.openai_api_key, base_url=settings.openai_base_url)
|
||||
|
||||
# Try to make a simple request to validate the key
|
||||
try:
|
||||
# Use a models list endpoint as a simple validation
|
||||
models = client.models.list()
|
||||
|
||||
# If we got here, the key is valid
|
||||
logger.info("OpenAI API key is valid")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "OpenAI API key is valid",
|
||||
"models_available": len(models.data) if hasattr(models, "data") else "Unknown",
|
||||
}
|
||||
|
||||
except openai.APITimeoutError as e:
|
||||
detail = _get_exception_chain_detail(e)
|
||||
logger.error(f"OpenAI API request timed out: {detail}", exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Request timed out: {detail}",
|
||||
"is_auth_error": False,
|
||||
"error_type": "timeout",
|
||||
}
|
||||
|
||||
except openai.APIConnectionError as e:
|
||||
detail = _get_exception_chain_detail(e)
|
||||
base_url = getattr(getattr(client, "_client", None), "base_url", None)
|
||||
base_url_info = f" (base_url: {base_url})" if base_url else ""
|
||||
logger.error(
|
||||
f"OpenAI API connection error{base_url_info}: {detail}",
|
||||
exc_info=True,
|
||||
)
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Connection error{base_url_info}: {detail}",
|
||||
"is_auth_error": False,
|
||||
"error_type": "connection_error",
|
||||
}
|
||||
|
||||
except openai.AuthenticationError as e:
|
||||
logger.error(f"OpenAI authentication error (status {e.status_code}): {e.message}", exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Authentication failed: {e.message}",
|
||||
"is_auth_error": True,
|
||||
"error_type": "authentication_error",
|
||||
}
|
||||
|
||||
except openai.RateLimitError as e:
|
||||
logger.warning(f"OpenAI rate limit exceeded (status {e.status_code}): {e.message}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Rate limit exceeded: {e.message}",
|
||||
"is_auth_error": False,
|
||||
"error_type": "rate_limit",
|
||||
}
|
||||
|
||||
except openai.APIStatusError as e:
|
||||
logger.error(
|
||||
f"OpenAI API returned HTTP {e.status_code}: {e.message} | "
|
||||
f"request_id={e.response.headers.get('x-request-id', 'n/a')}"
|
||||
)
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"API error (HTTP {e.status_code}): {e.message}",
|
||||
"is_auth_error": e.status_code == 401,
|
||||
"error_type": "api_status_error",
|
||||
"http_status": e.status_code,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
logger.error(f"OpenAI API key test failed: {error_msg}", exc_info=True)
|
||||
|
||||
# Determine if this is an authentication error
|
||||
is_auth_error = "auth" in error_msg.lower() or "api key" in error_msg.lower()
|
||||
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"API key validation failed: {error_msg}",
|
||||
"is_auth_error": is_auth_error,
|
||||
}
|
||||
|
||||
except ImportError:
|
||||
logger.exception("OpenAI package not installed")
|
||||
return {"status": "error", "message": "OpenAI package not installed"}
|
||||
except Exception as e:
|
||||
logger.exception("Unexpected error testing OpenAI connection")
|
||||
return {"status": "error", "message": f"Unexpected error: {str(e)}"}
|
||||
|
||||
|
||||
@router.get("/ai/test")
|
||||
@require_login
|
||||
async def test_ai_provider_connection(request: Request):
|
||||
"""
|
||||
Test the currently configured AI provider connection.
|
||||
|
||||
Uses ``get_ai_provider()`` to instantiate the active provider and sends a
|
||||
minimal chat completion to verify that the credentials and endpoint are
|
||||
reachable. Works for all supported providers (OpenAI, Azure, Anthropic,
|
||||
Gemini, Ollama, OpenRouter, Portkey, LiteLLM).
|
||||
"""
|
||||
from app.utils.ai_provider import get_ai_provider
|
||||
|
||||
provider_name = settings.ai_provider
|
||||
model = settings.ai_model or settings.openai_model
|
||||
|
||||
logger.info(f"Testing AI provider connection: provider={provider_name}, model={model}")
|
||||
|
||||
try:
|
||||
provider = get_ai_provider()
|
||||
response = provider.chat_completion(
|
||||
messages=[{"role": "user", "content": "Reply with the single word: ok"}],
|
||||
model=model,
|
||||
temperature=0,
|
||||
max_tokens=5,
|
||||
)
|
||||
logger.info(f"AI provider test successful: provider={provider_name}")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"AI provider '{provider_name}' is reachable and responding",
|
||||
"provider": provider_name,
|
||||
"model": model,
|
||||
"response_preview": (response or "")[:50],
|
||||
}
|
||||
except ValueError as e:
|
||||
# Configuration errors (missing keys, unknown provider)
|
||||
logger.warning(f"AI provider configuration error: {e}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": str(e),
|
||||
"provider": provider_name,
|
||||
}
|
||||
except Exception as e:
|
||||
detail = _get_exception_chain_detail(e)
|
||||
logger.error(f"AI provider test failed for '{provider_name}': {detail}", exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Connection failed: {detail}",
|
||||
"provider": provider_name,
|
||||
}
|
||||
|
||||
|
||||
class ExtractionTestRequest(BaseModel):
|
||||
"""Request body for the AI extraction test endpoint."""
|
||||
|
||||
text: str = Field(..., min_length=1, max_length=_MAX_EXTRACTION_TEXT_LEN, description="Plain-text document content")
|
||||
|
||||
|
||||
def _build_extraction_prompt(text: str) -> str:
|
||||
"""Return the metadata-extraction prompt used in the standard processing pipeline."""
|
||||
return (
|
||||
"You are a specialized document analyzer trained to extract structured metadata from documents.\n"
|
||||
"Your task is to analyze the given text and return a well-structured JSON object.\n\n"
|
||||
"Extract and return the following fields:\n"
|
||||
"1. **filename**: Machine-readable filename "
|
||||
"(YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, periods, and underscores).\n"
|
||||
'2. **empfaenger**: The recipient, or "Unknown" if not found.\n'
|
||||
'3. **absender**: The sender, or "Unknown" if not found.\n'
|
||||
"4. **correspondent**: The entity or company that issued the document "
|
||||
'(shortest possible name, e.g., "Amazon" instead of "Amazon EU SARL, German branch").\n'
|
||||
"5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, "
|
||||
"Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].\n"
|
||||
"6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, "
|
||||
"Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, "
|
||||
"Private_Korrespondenz, Sonstige_Informationen].\n"
|
||||
"7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).\n"
|
||||
"8. **tags**: A list of up to 4 relevant thematic keywords.\n"
|
||||
'9. **language**: Detected document language (ISO 639-1 code, e.g., "de" or "en").\n'
|
||||
"10. **title**: A human-readable title summarizing the document content.\n"
|
||||
"11. **confidence_score**: A numeric value (0-100) indicating the confidence level "
|
||||
"of the extracted metadata.\n"
|
||||
"12. **reference_number**: Extracted invoice/order/reference number if available.\n"
|
||||
"13. **monetary_amounts**: A list of key monetary values detected in the document.\n\n"
|
||||
"### Important Rules:\n"
|
||||
"- **OCR Correction**: Assume the text has been corrected for OCR errors.\n"
|
||||
"- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.\n"
|
||||
"- **Title**: Concise, no addresses, and contains key identifying features.\n"
|
||||
"- **Date Selection**: Use the most relevant date if multiple are found.\n"
|
||||
"- **Output Language**: Maintain the document's original language.\n\n"
|
||||
f"Extracted text:\n{text}\n\n"
|
||||
"Return only valid JSON with no additional commentary.\n"
|
||||
)
|
||||
|
||||
|
||||
def _extract_json_from_text(text: str):
|
||||
"""Try to extract a JSON object from the LLM response text."""
|
||||
pattern = r"```(?:json)?\s*(\{.*?\})\s*```"
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1)
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start != -1 and end != -1 and end > start:
|
||||
return text[start : end + 1]
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/ai/test-extraction")
|
||||
@require_login
|
||||
async def test_ai_extraction(request: Request, body: ExtractionTestRequest):
|
||||
"""
|
||||
Run the metadata-extraction prompt against the configured AI provider.
|
||||
|
||||
Accepts plain-text document content, sends it through the same prompt used
|
||||
by the background processing pipeline, and returns:
|
||||
- ``raw_response``: verbatim LLM output
|
||||
- ``parsed_json``: the extracted JSON object (null when parsing fails)
|
||||
- ``tags``: the ``tags`` list from the parsed JSON (empty list on failure)
|
||||
- ``provider`` / ``model``: which provider / model was used
|
||||
"""
|
||||
from app.utils.ai_provider import get_ai_provider
|
||||
|
||||
provider_name = settings.ai_provider
|
||||
model = settings.ai_model or settings.openai_model
|
||||
|
||||
logger.info(f"AI extraction test requested: provider={provider_name}, model={model}")
|
||||
|
||||
try:
|
||||
provider = get_ai_provider()
|
||||
prompt = _build_extraction_prompt(body.text)
|
||||
raw_response = provider.chat_completion(
|
||||
messages=[
|
||||
{"role": "system", "content": "You are an intelligent document classifier."},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
model=model,
|
||||
temperature=0,
|
||||
)
|
||||
except ValueError as e:
|
||||
logger.warning(f"AI extraction test – configuration error: {e}")
|
||||
return {"status": "error", "message": str(e), "provider": provider_name}
|
||||
except Exception as e:
|
||||
detail = _get_exception_chain_detail(e)
|
||||
logger.error(f"AI extraction test failed for provider '{provider_name}': {detail}", exc_info=True)
|
||||
return {"status": "error", "message": f"AI call failed: {detail}", "provider": provider_name}
|
||||
|
||||
# Attempt to parse JSON from the response
|
||||
parsed_json = None
|
||||
tags: list = []
|
||||
parse_error = None
|
||||
json_text = _extract_json_from_text(raw_response)
|
||||
if json_text:
|
||||
try:
|
||||
parsed_json = json.loads(json_text)
|
||||
tags = parsed_json.get("tags", [])
|
||||
except json.JSONDecodeError as exc:
|
||||
parse_error = str(exc)
|
||||
logger.warning(f"AI extraction test: JSON parse error: {exc}")
|
||||
else:
|
||||
parse_error = "No JSON object found in response"
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"provider": provider_name,
|
||||
"model": model,
|
||||
"raw_response": raw_response,
|
||||
"parsed_json": parsed_json,
|
||||
"tags": tags,
|
||||
"parse_error": parse_error,
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
"""
|
||||
Document processing API endpoints
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
|
||||
from app.api.common import resolve_file_path
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.tasks.process_document import process_document
|
||||
from app.tasks.send_to_all import send_to_all_destinations
|
||||
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive
|
||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||
from app.tasks.upload_to_onedrive import upload_to_onedrive
|
||||
from app.tasks.upload_to_paperless import upload_to_paperless
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/process/")
|
||||
@require_login
|
||||
def process(file_path: str):
|
||||
"""API Endpoint to start document processing."""
|
||||
file_path = resolve_file_path(file_path)
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
|
||||
task = process_document.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/send_to_dropbox/")
|
||||
@require_login
|
||||
def send_to_dropbox_endpoint(file_path: str):
|
||||
"""Send a document to Dropbox."""
|
||||
file_path = resolve_file_path(file_path, "processed")
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
task = upload_to_dropbox.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/send_to_paperless/")
|
||||
@require_login
|
||||
def send_to_paperless_endpoint(file_path: str):
|
||||
"""Send a document to Paperless-ngx."""
|
||||
file_path = resolve_file_path(file_path, "processed")
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
task = upload_to_paperless.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/send_to_nextcloud/")
|
||||
@require_login
|
||||
def send_to_nextcloud_endpoint(file_path: str):
|
||||
"""Send a document to NextCloud."""
|
||||
file_path = resolve_file_path(file_path, "processed")
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
task = upload_to_nextcloud.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/send_to_google_drive/")
|
||||
@require_login
|
||||
def send_to_google_drive_endpoint(file_path: str):
|
||||
"""Send a document to Google Drive."""
|
||||
file_path = resolve_file_path(file_path, "processed")
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
task = upload_to_google_drive.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/send_to_onedrive/")
|
||||
@require_login
|
||||
def send_to_onedrive_endpoint(file_path: str):
|
||||
"""Send a document to OneDrive."""
|
||||
file_path = resolve_file_path(file_path, "processed")
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
task = upload_to_onedrive.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/send_to_all_destinations/")
|
||||
@require_login
|
||||
def send_to_all_destinations_endpoint(file_path: str):
|
||||
"""Call the aggregator task that sends this file to all configured destinations."""
|
||||
file_path = resolve_file_path(file_path, "processed")
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=400, detail=f"File {file_path} not found.")
|
||||
|
||||
task = send_to_all_destinations.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued", "file_path": file_path}
|
||||
|
||||
|
||||
@router.post("/processall")
|
||||
@require_login
|
||||
def process_all_pdfs_in_workdir():
|
||||
"""
|
||||
Finds all .pdf files in <workdir> and enqueues them for processing.
|
||||
|
||||
For large batches (>processall_throttle_threshold files), tasks are staggered
|
||||
to avoid overwhelming downstream APIs.
|
||||
"""
|
||||
target_dir = settings.workdir
|
||||
if not os.path.exists(target_dir):
|
||||
raise HTTPException(status_code=400, detail=f"Directory {target_dir} does not exist.")
|
||||
|
||||
pdf_files = []
|
||||
for filename in os.listdir(target_dir):
|
||||
if filename.lower().endswith(".pdf"):
|
||||
pdf_files.append(filename)
|
||||
|
||||
if not pdf_files:
|
||||
return {"message": "No PDF files found in that directory."}
|
||||
|
||||
task_ids = []
|
||||
num_files = len(pdf_files)
|
||||
|
||||
# Apply throttling if we have more files than the threshold
|
||||
apply_throttle = num_files > settings.processall_throttle_threshold
|
||||
|
||||
if apply_throttle:
|
||||
logger.info(
|
||||
f"Processing {num_files} files with throttling "
|
||||
f"(threshold: {settings.processall_throttle_threshold}, "
|
||||
f"delay: {settings.processall_throttle_delay}s per file)"
|
||||
)
|
||||
|
||||
for index, pdf in enumerate(pdf_files):
|
||||
file_path = os.path.join(target_dir, pdf)
|
||||
|
||||
if apply_throttle:
|
||||
# Stagger task submission with countdown
|
||||
# First file starts immediately (countdown=0)
|
||||
# Each subsequent file has an increasing delay
|
||||
countdown = index * settings.processall_throttle_delay
|
||||
task = process_document.apply_async(args=[file_path], countdown=countdown)
|
||||
logger.debug(f"Scheduled {pdf} with {countdown}s delay")
|
||||
else:
|
||||
# No throttling - enqueue immediately
|
||||
task = process_document.delay(file_path)
|
||||
|
||||
task_ids.append(task.id)
|
||||
|
||||
message = f"Enqueued {num_files} PDFs for processing"
|
||||
if apply_throttle:
|
||||
total_time = (num_files - 1) * settings.processall_throttle_delay
|
||||
message += f" (throttled over {total_time} seconds)"
|
||||
|
||||
return {"message": message, "pdf_files": pdf_files, "task_ids": task_ids, "throttled": apply_throttle}
|
||||
@@ -1,270 +0,0 @@
|
||||
"""
|
||||
Queue monitoring API endpoints.
|
||||
|
||||
Provides endpoints to query Celery/Redis queue statistics and
|
||||
database-level processing status for document pipeline visibility.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import redis
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.models import FileProcessingStep, FileRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/queue", tags=["queue"])
|
||||
|
||||
# Constants
|
||||
CELERY_INSPECT_TIMEOUT = 2.0
|
||||
MAX_ARGS_DISPLAY_LENGTH = 200
|
||||
|
||||
|
||||
def _get_redis_queue_length(redis_client: redis.Redis, queue_name: str) -> int:
|
||||
"""Get the number of messages in a Redis-backed Celery queue.
|
||||
|
||||
Args:
|
||||
redis_client: Connected Redis client instance.
|
||||
queue_name: Name of the Celery queue to inspect.
|
||||
|
||||
Returns:
|
||||
Number of messages (tasks) waiting in the queue.
|
||||
"""
|
||||
try:
|
||||
return redis_client.llen(queue_name)
|
||||
except Exception:
|
||||
logger.debug(f"Could not read queue length for '{queue_name}'")
|
||||
return 0
|
||||
|
||||
|
||||
def _get_celery_inspect_stats() -> dict[str, Any]:
|
||||
"""Query the Celery inspect API for active, reserved, and scheduled tasks.
|
||||
|
||||
Returns:
|
||||
Dictionary with active, reserved, and scheduled task summaries.
|
||||
"""
|
||||
from app.celery_app import celery
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"active": [],
|
||||
"reserved": [],
|
||||
"scheduled": [],
|
||||
"workers_online": 0,
|
||||
}
|
||||
|
||||
try:
|
||||
inspector = celery.control.inspect(timeout=CELERY_INSPECT_TIMEOUT)
|
||||
|
||||
active = inspector.active() or {}
|
||||
reserved = inspector.reserved() or {}
|
||||
scheduled = inspector.scheduled() or {}
|
||||
|
||||
result["workers_online"] = len(active)
|
||||
|
||||
for _worker, tasks in active.items():
|
||||
for task in tasks:
|
||||
result["active"].append(
|
||||
{
|
||||
"id": task.get("id", ""),
|
||||
"name": task.get("name", "unknown"),
|
||||
"args": str(task.get("args", []))[:MAX_ARGS_DISPLAY_LENGTH],
|
||||
"started": task.get("time_start"),
|
||||
}
|
||||
)
|
||||
|
||||
for _worker, tasks in reserved.items():
|
||||
for task in tasks:
|
||||
result["reserved"].append(
|
||||
{
|
||||
"id": task.get("id", ""),
|
||||
"name": task.get("name", "unknown"),
|
||||
"args": str(task.get("args", []))[:MAX_ARGS_DISPLAY_LENGTH],
|
||||
}
|
||||
)
|
||||
|
||||
for _worker, tasks in scheduled.items():
|
||||
for task in tasks:
|
||||
req = task.get("request", {})
|
||||
result["scheduled"].append(
|
||||
{
|
||||
"id": req.get("id", ""),
|
||||
"name": req.get("name", "unknown"),
|
||||
"eta": task.get("eta"),
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Celery inspect failed (workers may be offline): {exc}")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _get_db_processing_summary(db: Session) -> dict[str, Any]:
|
||||
"""Query the database for a summary of file processing states.
|
||||
|
||||
Args:
|
||||
db: SQLAlchemy database session.
|
||||
|
||||
Returns:
|
||||
Dictionary with counts of files by processing state.
|
||||
"""
|
||||
try:
|
||||
total_files = db.query(func.count(FileRecord.id)).scalar() or 0
|
||||
|
||||
# Count files with at least one in_progress step
|
||||
processing_count = (
|
||||
db.query(func.count(func.distinct(FileProcessingStep.file_id)))
|
||||
.filter(FileProcessingStep.status == "in_progress")
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Count files with at least one failure and no in_progress
|
||||
failed_subq = (
|
||||
db.query(FileProcessingStep.file_id).filter(FileProcessingStep.status == "failure").distinct().subquery()
|
||||
)
|
||||
in_progress_subq = (
|
||||
db.query(FileProcessingStep.file_id)
|
||||
.filter(FileProcessingStep.status == "in_progress")
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
failed_count = (
|
||||
db.query(func.count(func.distinct(failed_subq.c.file_id)))
|
||||
.filter(~failed_subq.c.file_id.in_(db.query(in_progress_subq.c.file_id)))
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Count files that have steps and all steps are success/skipped
|
||||
all_step_files = db.query(FileProcessingStep.file_id).distinct().subquery()
|
||||
# Files with any non-terminal step
|
||||
non_terminal = (
|
||||
db.query(FileProcessingStep.file_id)
|
||||
.filter(FileProcessingStep.status.in_(["in_progress", "pending", "failure"]))
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
completed_count = (
|
||||
db.query(func.count(func.distinct(all_step_files.c.file_id)))
|
||||
.filter(~all_step_files.c.file_id.in_(db.query(non_terminal.c.file_id)))
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Files with no processing steps at all
|
||||
files_with_steps = db.query(FileProcessingStep.file_id).distinct().subquery()
|
||||
pending_count = (
|
||||
db.query(func.count(FileRecord.id))
|
||||
.filter(~FileRecord.id.in_(db.query(files_with_steps.c.file_id)))
|
||||
.filter(FileRecord.is_duplicate.is_(False))
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Recent files being processed (last 20 in_progress or pending)
|
||||
recent_processing = (
|
||||
db.query(FileRecord.id, FileRecord.original_filename, FileProcessingStep.step_name)
|
||||
.join(FileProcessingStep, FileRecord.id == FileProcessingStep.file_id)
|
||||
.filter(FileProcessingStep.status == "in_progress")
|
||||
.order_by(FileProcessingStep.updated_at.desc())
|
||||
.limit(20)
|
||||
.all()
|
||||
)
|
||||
|
||||
recent_list = [
|
||||
{"file_id": r[0], "filename": r[1] or f"File #{r[0]}", "current_step": r[2]} for r in recent_processing
|
||||
]
|
||||
|
||||
return {
|
||||
"total_files": total_files,
|
||||
"processing": processing_count,
|
||||
"failed": failed_count,
|
||||
"completed": completed_count,
|
||||
"pending": pending_count,
|
||||
"recent_processing": recent_list,
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.error(f"Error querying DB processing summary: {exc}")
|
||||
return {
|
||||
"total_files": 0,
|
||||
"processing": 0,
|
||||
"failed": 0,
|
||||
"completed": 0,
|
||||
"pending": 0,
|
||||
"recent_processing": [],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
def get_queue_stats(db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||
"""Get comprehensive queue and processing statistics.
|
||||
|
||||
Returns queue lengths from Redis, Celery worker inspection data,
|
||||
and database-level processing summaries for the document pipeline.
|
||||
|
||||
Returns:
|
||||
Dictionary containing redis queue info, celery worker info,
|
||||
and database processing summary.
|
||||
"""
|
||||
# 1. Redis queue lengths
|
||||
queue_lengths: dict[str, int] = {}
|
||||
try:
|
||||
redis_client = redis.Redis.from_url(settings.redis_url, decode_responses=True)
|
||||
for queue_name in ["document_processor", "default", "celery"]:
|
||||
queue_lengths[queue_name] = _get_redis_queue_length(redis_client, queue_name)
|
||||
redis_client.close()
|
||||
except Exception as exc:
|
||||
logger.warning(f"Could not connect to Redis: {exc}")
|
||||
|
||||
total_queued = sum(queue_lengths.values())
|
||||
|
||||
# 2. Celery inspect
|
||||
celery_stats = _get_celery_inspect_stats()
|
||||
|
||||
# 3. DB summary
|
||||
db_summary = _get_db_processing_summary(db)
|
||||
|
||||
return {
|
||||
"queues": queue_lengths,
|
||||
"total_queued": total_queued,
|
||||
"celery": celery_stats,
|
||||
"db_summary": db_summary,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/pending-count")
|
||||
def get_pending_count(db: Session = Depends(get_db)) -> dict[str, int]:
|
||||
"""Get a lightweight count of queued + in-progress items for the files page banner.
|
||||
|
||||
Returns:
|
||||
Dictionary with total_pending count (queued in Redis + processing in DB).
|
||||
"""
|
||||
total_pending = 0
|
||||
|
||||
# Redis queue lengths
|
||||
try:
|
||||
redis_client = redis.Redis.from_url(settings.redis_url, decode_responses=True)
|
||||
for queue_name in ["document_processor", "default", "celery"]:
|
||||
total_pending += _get_redis_queue_length(redis_client, queue_name)
|
||||
redis_client.close()
|
||||
except Exception:
|
||||
logger.debug("Could not connect to Redis for pending count")
|
||||
|
||||
# DB in-progress count
|
||||
try:
|
||||
processing_count = (
|
||||
db.query(func.count(func.distinct(FileProcessingStep.file_id)))
|
||||
.filter(FileProcessingStep.status == "in_progress")
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
total_pending += processing_count
|
||||
except Exception:
|
||||
logger.debug("Could not query DB for processing count")
|
||||
|
||||
return {"total_pending": total_pending}
|
||||
@@ -1,305 +0,0 @@
|
||||
"""
|
||||
Saved searches API endpoints.
|
||||
|
||||
Provides CRUD operations for user-defined saved search filters.
|
||||
Each user can save, list, update, and delete named filter combinations
|
||||
for quick access on the files page.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import get_current_user, require_login
|
||||
from app.database import get_db
|
||||
from app.models import SavedSearch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/saved-searches", tags=["saved-searches"])
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# Allowed filter keys that can be saved.
|
||||
# Files-view keys: search, mime_type, status, storage_provider, sort_by, sort_order
|
||||
# Search-view keys: q, document_type, language, sender, text_quality
|
||||
# Shared keys: tags, date_from, date_to
|
||||
ALLOWED_FILTER_KEYS = frozenset(
|
||||
{
|
||||
"search",
|
||||
"q",
|
||||
"mime_type",
|
||||
"status",
|
||||
"date_from",
|
||||
"date_to",
|
||||
"storage_provider",
|
||||
"tags",
|
||||
"sort_by",
|
||||
"sort_order",
|
||||
"document_type",
|
||||
"language",
|
||||
"sender",
|
||||
"text_quality",
|
||||
}
|
||||
)
|
||||
|
||||
# Maximum number of saved searches per user
|
||||
MAX_SAVED_SEARCHES_PER_USER = 50
|
||||
|
||||
# Maximum length for saved search name
|
||||
MAX_NAME_LENGTH = 100
|
||||
|
||||
|
||||
def _get_user_id(request: Request) -> str:
|
||||
"""Extract user identifier from the session.
|
||||
|
||||
Returns the preferred_username, email, or 'anonymous' if auth is disabled.
|
||||
|
||||
Args:
|
||||
request: The incoming HTTP request.
|
||||
|
||||
Returns:
|
||||
A string identifying the current user.
|
||||
"""
|
||||
user = get_current_user(request)
|
||||
if user:
|
||||
return user.get("preferred_username") or user.get("email") or user.get("name", "anonymous")
|
||||
return "anonymous"
|
||||
|
||||
|
||||
def _validate_filters(filters: Any) -> dict:
|
||||
"""Validate and sanitize filter parameters.
|
||||
|
||||
Ensures only allowed filter keys are present and values are strings.
|
||||
|
||||
Args:
|
||||
filters: The raw filter value from the client.
|
||||
|
||||
Returns:
|
||||
A sanitized filter dictionary with only allowed keys.
|
||||
|
||||
Raises:
|
||||
HTTPException: If filters is not a dict or contains invalid values.
|
||||
"""
|
||||
if not isinstance(filters, dict):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="filters must be a JSON object",
|
||||
)
|
||||
sanitized = {}
|
||||
for key, value in filters.items():
|
||||
if key in ALLOWED_FILTER_KEYS and isinstance(value, str) and value.strip():
|
||||
sanitized[key] = value.strip()
|
||||
return sanitized
|
||||
|
||||
|
||||
def _serialize_saved_search(s: SavedSearch) -> dict:
|
||||
"""Serialize a SavedSearch model instance to a JSON-compatible dict.
|
||||
|
||||
Args:
|
||||
s: The SavedSearch model instance.
|
||||
|
||||
Returns:
|
||||
A dictionary with id, name, filters, created_at, and updated_at.
|
||||
"""
|
||||
return {
|
||||
"id": s.id,
|
||||
"name": s.name,
|
||||
"filters": json.loads(s.filters),
|
||||
"created_at": s.created_at.isoformat() if s.created_at else None,
|
||||
"updated_at": s.updated_at.isoformat() if s.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
@router.get("")
|
||||
@require_login
|
||||
def list_saved_searches(request: Request, db: DbSession):
|
||||
"""List all saved searches for the current user.
|
||||
|
||||
Returns:
|
||||
A list of saved search objects with id, name, filters, and timestamps.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
searches = db.query(SavedSearch).filter(SavedSearch.user_id == user_id).order_by(SavedSearch.name).all()
|
||||
return [_serialize_saved_search(s) for s in searches]
|
||||
|
||||
|
||||
@router.post("", status_code=status.HTTP_201_CREATED)
|
||||
@require_login
|
||||
def create_saved_search(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
name: str = Body(..., embed=True),
|
||||
filters: dict = Body(..., embed=True),
|
||||
):
|
||||
"""Create a new saved search for the current user.
|
||||
|
||||
Request body (JSON):
|
||||
name: Display name for the saved search (required, max 100 chars)
|
||||
filters: Dictionary of filter parameters (required)
|
||||
|
||||
Returns:
|
||||
The created saved search object.
|
||||
|
||||
Raises:
|
||||
HTTPException 422: If name or filters are invalid.
|
||||
HTTPException 409: If a saved search with the same name already exists.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
|
||||
name = name.strip() if isinstance(name, str) else ""
|
||||
if not name or len(name) > MAX_NAME_LENGTH:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"name is required and must be at most {MAX_NAME_LENGTH} characters",
|
||||
)
|
||||
|
||||
sanitized_filters = _validate_filters(filters)
|
||||
if not sanitized_filters:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="At least one filter parameter is required",
|
||||
)
|
||||
|
||||
# Check user limit
|
||||
count = db.query(SavedSearch).filter(SavedSearch.user_id == user_id).count()
|
||||
if count >= MAX_SAVED_SEARCHES_PER_USER:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"Maximum of {MAX_SAVED_SEARCHES_PER_USER} saved searches reached",
|
||||
)
|
||||
|
||||
# Check for duplicate name
|
||||
existing = db.query(SavedSearch).filter(SavedSearch.user_id == user_id, SavedSearch.name == name).first()
|
||||
if existing:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"A saved search named '{name}' already exists",
|
||||
)
|
||||
|
||||
saved_search = SavedSearch(
|
||||
user_id=user_id,
|
||||
name=name,
|
||||
filters=json.dumps(sanitized_filters),
|
||||
)
|
||||
|
||||
try:
|
||||
db.add(saved_search)
|
||||
db.commit()
|
||||
db.refresh(saved_search)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.exception(f"Failed to create saved search for user={user_id}: {exc}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to save search",
|
||||
)
|
||||
|
||||
logger.info(f"Saved search created: user={user_id}, name={name!r}")
|
||||
return _serialize_saved_search(saved_search)
|
||||
|
||||
|
||||
@router.put("/{search_id}")
|
||||
@require_login
|
||||
def update_saved_search(
|
||||
search_id: int,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
name: str | None = Body(None, embed=True),
|
||||
filters: dict | None = Body(None, embed=True),
|
||||
):
|
||||
"""Update an existing saved search.
|
||||
|
||||
Path Parameters:
|
||||
search_id: The ID of the saved search to update.
|
||||
|
||||
Request body (JSON):
|
||||
name: New display name (optional)
|
||||
filters: New filter parameters (optional)
|
||||
|
||||
Returns:
|
||||
The updated saved search object.
|
||||
|
||||
Raises:
|
||||
HTTPException 404: If the saved search is not found.
|
||||
HTTPException 409: If the new name conflicts with an existing saved search.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
saved_search = db.query(SavedSearch).filter(SavedSearch.id == search_id, SavedSearch.user_id == user_id).first()
|
||||
if not saved_search:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Saved search not found")
|
||||
|
||||
if name is not None:
|
||||
new_name = name.strip() if isinstance(name, str) else ""
|
||||
if not new_name or len(new_name) > MAX_NAME_LENGTH:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=f"name must be non-empty and at most {MAX_NAME_LENGTH} characters",
|
||||
)
|
||||
# Check for name conflict
|
||||
if new_name != saved_search.name:
|
||||
existing = (
|
||||
db.query(SavedSearch).filter(SavedSearch.user_id == user_id, SavedSearch.name == new_name).first()
|
||||
)
|
||||
if existing:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"A saved search named '{new_name}' already exists",
|
||||
)
|
||||
saved_search.name = new_name
|
||||
|
||||
if filters is not None:
|
||||
sanitized_filters = _validate_filters(filters)
|
||||
if not sanitized_filters:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="At least one filter parameter is required",
|
||||
)
|
||||
saved_search.filters = json.dumps(sanitized_filters)
|
||||
|
||||
try:
|
||||
db.commit()
|
||||
db.refresh(saved_search)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.exception(f"Failed to update saved search id={search_id}, user={user_id}: {exc}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to update saved search",
|
||||
)
|
||||
|
||||
logger.info(f"Saved search updated: id={search_id}, user={user_id}")
|
||||
return _serialize_saved_search(saved_search)
|
||||
|
||||
|
||||
@router.delete("/{search_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
@require_login
|
||||
def delete_saved_search(search_id: int, request: Request, db: DbSession):
|
||||
"""Delete a saved search.
|
||||
|
||||
Path Parameters:
|
||||
search_id: The ID of the saved search to delete.
|
||||
|
||||
Raises:
|
||||
HTTPException 404: If the saved search is not found.
|
||||
"""
|
||||
user_id = _get_user_id(request)
|
||||
saved_search = db.query(SavedSearch).filter(SavedSearch.id == search_id, SavedSearch.user_id == user_id).first()
|
||||
if not saved_search:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Saved search not found")
|
||||
|
||||
try:
|
||||
db.delete(saved_search)
|
||||
db.commit()
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.exception(f"Failed to delete saved search id={search_id}, user={user_id}: {exc}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to delete saved search",
|
||||
)
|
||||
|
||||
logger.info(f"Saved search deleted: id={search_id}, user={user_id}")
|
||||
@@ -1,111 +0,0 @@
|
||||
"""Full-text search API endpoints.
|
||||
|
||||
Provides document search across OCR text, AI metadata, filenames, and tags
|
||||
via Meilisearch. Designed to serve as the backend for the UI search bar on
|
||||
the /files page and as a standalone API for integrations.
|
||||
|
||||
Future extension point: the OCR text stored in the index is also suitable
|
||||
for RAG (Retrieval Augmented Generation) chatbot workflows.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Query, Request
|
||||
|
||||
from app.auth import require_login
|
||||
from app.utils.meilisearch_client import search_documents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/search")
|
||||
@require_login
|
||||
def search_api(
|
||||
request: Request,
|
||||
q: str = Query(..., min_length=1, max_length=512, description="Full-text search query"),
|
||||
mime_type: Optional[str] = Query(None, description="Filter by MIME type (e.g. application/pdf)"),
|
||||
document_type: Optional[str] = Query(None, description="Filter by document type (e.g. Invoice)"),
|
||||
language: Optional[str] = Query(None, description="Filter by language code (e.g. de, en)"),
|
||||
tags: Optional[str] = Query(None, description="Filter by tag (exact match)"),
|
||||
sender: Optional[str] = Query(None, description="Filter by sender/absender (exact match)"),
|
||||
text_quality: Optional[str] = Query(
|
||||
None,
|
||||
description="Filter by OCR text quality: no_text, low, medium, high",
|
||||
),
|
||||
date_from: Optional[int] = Query(None, description="Filter results created after this Unix timestamp"),
|
||||
date_to: Optional[int] = Query(None, description="Filter results created before this Unix timestamp"),
|
||||
page: int = Query(1, ge=1, description="Page number (1-based)"),
|
||||
per_page: int = Query(20, ge=1, le=100, description="Results per page"),
|
||||
):
|
||||
"""Search documents by full text, metadata, and tags.
|
||||
|
||||
Searches across:
|
||||
- Document title and filename
|
||||
- OCR / extracted text
|
||||
- Tags, sender, recipient, document type
|
||||
- Correspondent and reference number
|
||||
|
||||
Results are ranked by Meilisearch relevance and include highlighted
|
||||
snippets showing where the query terms matched.
|
||||
|
||||
Query Parameters:
|
||||
- q: Search query (required)
|
||||
- mime_type: Filter by MIME type
|
||||
- document_type: Filter by document type
|
||||
- language: Filter by language code
|
||||
- tags: Filter by tag (exact match on a single tag)
|
||||
- sender: Filter by sender/absender (exact match)
|
||||
- text_quality: Filter by OCR text quality (no_text, low, medium, high)
|
||||
- date_from: Unix timestamp lower bound
|
||||
- date_to: Unix timestamp upper bound
|
||||
- page: Page number (default: 1)
|
||||
- per_page: Results per page (default: 20, max: 100)
|
||||
|
||||
Example:
|
||||
```
|
||||
GET /api/search?q=invoice&document_type=Invoice&tags=amazon&date_from=1704067200&page=1&per_page=20
|
||||
```
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"file_id": 42,
|
||||
"original_filename": "2026-01-15_Invoice_Amazon.pdf",
|
||||
"document_title": "Amazon Invoice January 2026",
|
||||
"document_type": "Invoice",
|
||||
"tags": ["amazon", "invoice"],
|
||||
"_formatted": {
|
||||
"document_title": "Amazon <mark>Invoice</mark> January 2026",
|
||||
"ocr_text": "...total amount of the <mark>invoice</mark> is..."
|
||||
}
|
||||
}
|
||||
],
|
||||
"total": 42,
|
||||
"page": 1,
|
||||
"pages": 3,
|
||||
"query": "invoice"
|
||||
}
|
||||
```
|
||||
"""
|
||||
logger.info(f"Search request: q={q!r}, mime_type={mime_type}, page={page}, per_page={per_page}")
|
||||
|
||||
result = search_documents(
|
||||
q,
|
||||
mime_type=mime_type,
|
||||
document_type=document_type,
|
||||
language=language,
|
||||
tags=tags,
|
||||
sender=sender,
|
||||
text_quality=text_quality,
|
||||
date_from=date_from,
|
||||
date_to=date_to,
|
||||
page=page,
|
||||
per_page=per_page,
|
||||
)
|
||||
|
||||
return result
|
||||
@@ -1,516 +0,0 @@
|
||||
"""
|
||||
API endpoints for managing application settings.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Annotated, Any, Dict, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.database import get_db
|
||||
from app.utils.input_validation import validate_setting_key, validate_setting_key_format
|
||||
from app.utils.settings_service import (
|
||||
SETTING_METADATA,
|
||||
delete_setting_from_db,
|
||||
get_all_settings_from_db,
|
||||
get_audit_log,
|
||||
get_setting_history,
|
||||
get_setting_metadata,
|
||||
get_settings_by_category,
|
||||
rollback_setting,
|
||||
save_setting_to_db,
|
||||
validate_setting_value,
|
||||
)
|
||||
from app.utils.settings_sync import notify_settings_updated
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/settings", tags=["settings"])
|
||||
|
||||
|
||||
def require_admin(request: Request) -> dict:
|
||||
"""
|
||||
Dependency to ensure the user is an admin.
|
||||
Raises HTTPException if not admin.
|
||||
|
||||
Returns:
|
||||
User dict from session
|
||||
"""
|
||||
user = request.session.get("user")
|
||||
if not user or not user.get("is_admin"):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required")
|
||||
return user
|
||||
|
||||
|
||||
DbSession = Annotated[Session, Depends(get_db)]
|
||||
AdminUser = Annotated[dict, Depends(require_admin)]
|
||||
|
||||
|
||||
class SettingUpdate(BaseModel):
|
||||
"""Model for updating a setting"""
|
||||
|
||||
key: str = Field(..., description="Setting key")
|
||||
value: Optional[str] = Field(None, description="Setting value (None to delete)")
|
||||
|
||||
|
||||
class SettingResponse(BaseModel):
|
||||
"""Model for setting response"""
|
||||
|
||||
key: str
|
||||
value: Optional[str]
|
||||
metadata: Dict[str, Any]
|
||||
|
||||
|
||||
class SettingsListResponse(BaseModel):
|
||||
"""Model for list of settings"""
|
||||
|
||||
settings: Dict[str, Any]
|
||||
categories: Dict[str, list]
|
||||
db_settings: Dict[str, str]
|
||||
|
||||
|
||||
@router.get("/", response_model=SettingsListResponse)
|
||||
async def get_settings(request: Request, db: DbSession, admin: AdminUser):
|
||||
"""
|
||||
Get all application settings with metadata.
|
||||
Admin only.
|
||||
"""
|
||||
try:
|
||||
# Get current runtime settings
|
||||
current_settings = {}
|
||||
for key in SETTING_METADATA.keys():
|
||||
if hasattr(settings, key):
|
||||
value = getattr(settings, key)
|
||||
current_settings[key] = {
|
||||
"value": value,
|
||||
"metadata": get_setting_metadata(key),
|
||||
}
|
||||
|
||||
# Get settings stored in database
|
||||
db_settings = get_all_settings_from_db(db)
|
||||
|
||||
# Get settings organized by category
|
||||
categories = get_settings_by_category()
|
||||
|
||||
return SettingsListResponse(settings=current_settings, categories=categories, db_settings=db_settings)
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrieving settings: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to retrieve settings",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/credentials")
|
||||
async def list_credentials(request: Request, db: DbSession, admin: AdminUser):
|
||||
"""
|
||||
List all sensitive credential settings with their configured/unconfigured status.
|
||||
|
||||
Returns a credential audit report indicating which credentials are set and whether
|
||||
each value originates from the database or an environment variable.
|
||||
This endpoint is intended to support credential rotation workflows.
|
||||
Admin only.
|
||||
"""
|
||||
try:
|
||||
db_settings = get_all_settings_from_db(db)
|
||||
credentials = []
|
||||
|
||||
for key, meta in SETTING_METADATA.items():
|
||||
if not meta.get("sensitive", False):
|
||||
continue
|
||||
|
||||
env_value = getattr(settings, key, None)
|
||||
in_db = key in db_settings and db_settings[key]
|
||||
|
||||
if in_db:
|
||||
source = "db"
|
||||
configured = True
|
||||
elif env_value:
|
||||
source = "env"
|
||||
configured = True
|
||||
else:
|
||||
source = None
|
||||
configured = False
|
||||
|
||||
credentials.append(
|
||||
{
|
||||
"key": key,
|
||||
"category": meta.get("category", "Other"),
|
||||
"description": meta.get("description", ""),
|
||||
"configured": configured,
|
||||
"source": source,
|
||||
"restart_required": meta.get("restart_required", False),
|
||||
}
|
||||
)
|
||||
|
||||
configured_count = sum(1 for c in credentials if c["configured"])
|
||||
return {
|
||||
"credentials": credentials,
|
||||
"total": len(credentials),
|
||||
"configured_count": configured_count,
|
||||
"unconfigured_count": len(credentials) - configured_count,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrieving credential list: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to retrieve credentials",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/audit-log")
|
||||
async def list_audit_log(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
admin: AdminUser,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
):
|
||||
"""
|
||||
Retrieve the settings audit log (most recent first).
|
||||
|
||||
Returns all configuration changes recorded in the audit log.
|
||||
Sensitive values are masked in the response.
|
||||
Admin only.
|
||||
"""
|
||||
try:
|
||||
entries = get_audit_log(db, limit=limit, offset=offset)
|
||||
return {"entries": entries, "limit": limit, "offset": offset}
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrieving audit log: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to retrieve audit log",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/export-env")
|
||||
async def export_env_settings(
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
admin: AdminUser,
|
||||
source: str = "db",
|
||||
):
|
||||
"""
|
||||
Export current settings as a ``.env`` file.
|
||||
|
||||
Query params:
|
||||
- ``source=db`` (default) – only settings explicitly saved to the database.
|
||||
- ``source=effective`` – full runtime configuration (DB > ENV > defaults) for
|
||||
every key defined in SETTING_METADATA.
|
||||
|
||||
Returns a downloadable plain-text file suitable for bootstrapping another
|
||||
installation. All values — including sensitive ones — are included; only
|
||||
admins can access this endpoint.
|
||||
"""
|
||||
from fastapi.responses import Response as FastAPIResponse
|
||||
|
||||
from app.utils.settings_service import get_settings_for_export
|
||||
|
||||
if source not in ("db", "effective"):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="source must be 'db' or 'effective'",
|
||||
)
|
||||
|
||||
try:
|
||||
export_data = get_settings_for_export(db, source=source)
|
||||
lines = [
|
||||
"# DocuElevate configuration export",
|
||||
f"# Source: {source}",
|
||||
"# Generated by DocuElevate Settings Export",
|
||||
"# WARNING: This file contains sensitive values. Handle with care.",
|
||||
"",
|
||||
]
|
||||
for env_key, value in export_data.items():
|
||||
lines.append(f"{env_key}={value}")
|
||||
lines.append("") # trailing newline
|
||||
content = "\n".join(lines)
|
||||
|
||||
return FastAPIResponse(
|
||||
content=content,
|
||||
media_type="text/plain",
|
||||
headers={"Content-Disposition": f'attachment; filename="docuelevate-{source}.env"'},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error exporting settings: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to export settings",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{key}", response_model=SettingResponse)
|
||||
async def get_setting(key: str, request: Request, db: DbSession, admin: AdminUser):
|
||||
"""
|
||||
Get a specific setting by key.
|
||||
Admin only.
|
||||
"""
|
||||
validate_setting_key_format(key)
|
||||
try:
|
||||
# Get current value
|
||||
value = getattr(settings, key, None)
|
||||
|
||||
# Get metadata
|
||||
metadata = get_setting_metadata(key)
|
||||
|
||||
return SettingResponse(key=key, value=str(value) if value is not None else None, metadata=metadata)
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrieving setting {key}: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to retrieve setting: {key}",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{key}")
|
||||
async def update_setting(
|
||||
key: str,
|
||||
setting: SettingUpdate,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
admin: AdminUser,
|
||||
):
|
||||
"""
|
||||
Update a specific setting.
|
||||
Admin only.
|
||||
"""
|
||||
validate_setting_key(key)
|
||||
try:
|
||||
# Validate the setting value
|
||||
if setting.value is not None:
|
||||
is_valid, error_message = validate_setting_value(key, setting.value)
|
||||
if not is_valid:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=error_message)
|
||||
|
||||
# Determine the username for the audit log
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "admin"
|
||||
)
|
||||
|
||||
# Save to database
|
||||
success = save_setting_to_db(db, key, setting.value, changed_by=changed_by)
|
||||
if not success:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to save setting to database",
|
||||
)
|
||||
|
||||
# Notify workers that settings have changed
|
||||
notify_settings_updated()
|
||||
|
||||
# Get metadata
|
||||
metadata = get_setting_metadata(key)
|
||||
restart_required = metadata.get("restart_required", False)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Setting '{key}' updated successfully",
|
||||
"restart_required": restart_required,
|
||||
"key": key,
|
||||
"value": setting.value,
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating setting {key}: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to update setting: {key}",
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{key}")
|
||||
async def delete_setting(key: str, request: Request, db: DbSession, admin: AdminUser):
|
||||
"""
|
||||
Delete a setting from the database (reverts to environment variable or default).
|
||||
Admin only.
|
||||
"""
|
||||
validate_setting_key(key)
|
||||
try:
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "admin"
|
||||
)
|
||||
|
||||
success = delete_setting_from_db(db, key, changed_by=changed_by)
|
||||
if not success:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Setting '{key}' not found in database",
|
||||
)
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Setting '{key}' deleted from database (will use environment variable or default)",
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting setting {key}: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to delete setting: {key}",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/bulk-update")
|
||||
async def bulk_update_settings(updates: list[SettingUpdate], request: Request, db: DbSession, admin: AdminUser):
|
||||
"""
|
||||
Update multiple settings at once.
|
||||
Admin only.
|
||||
"""
|
||||
results = []
|
||||
errors = []
|
||||
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "admin"
|
||||
)
|
||||
|
||||
for update in updates:
|
||||
try:
|
||||
# Validate the setting value
|
||||
if update.value is not None:
|
||||
is_valid, error_message = validate_setting_value(update.key, update.value)
|
||||
if not is_valid:
|
||||
errors.append({"key": update.key, "error": error_message})
|
||||
continue
|
||||
|
||||
# Save to database
|
||||
success = save_setting_to_db(db, update.key, update.value, changed_by=changed_by)
|
||||
if success:
|
||||
results.append({"key": update.key, "value": update.value, "status": "success"})
|
||||
else:
|
||||
errors.append({"key": update.key, "error": "Failed to save to database"})
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating setting {update.key}: {e}")
|
||||
errors.append({"key": update.key, "error": str(e)})
|
||||
|
||||
if results:
|
||||
notify_settings_updated()
|
||||
|
||||
restart_required = any(get_setting_metadata(result["key"]).get("restart_required", False) for result in results)
|
||||
|
||||
return {
|
||||
"success": len(errors) == 0,
|
||||
"updated": results,
|
||||
"errors": errors,
|
||||
"restart_required": restart_required,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/install-ocr-languages")
|
||||
async def install_ocr_languages(request: Request, admin: AdminUser):
|
||||
"""
|
||||
Trigger on-demand installation of Tesseract language data files and
|
||||
EasyOCR model downloads for the languages currently configured in the
|
||||
application settings.
|
||||
|
||||
This endpoint is useful after changing ``tesseract_language`` or
|
||||
``easyocr_languages`` so that the required data is available without
|
||||
restarting the container. The download runs synchronously and may take
|
||||
a few seconds (or minutes for large EasyOCR models).
|
||||
|
||||
Returns a summary of which languages are now available and which could
|
||||
not be installed.
|
||||
Admin only.
|
||||
"""
|
||||
from app.utils.ocr_language_manager import ensure_ocr_languages_from_settings # noqa: PLC0415
|
||||
|
||||
try:
|
||||
result = ensure_ocr_languages_from_settings()
|
||||
tesseract_missing = result.get("tesseract_missing", [])
|
||||
easyocr_failed = result.get("easyocr_failed", [])
|
||||
all_ok = not tesseract_missing and not easyocr_failed
|
||||
return {
|
||||
"success": all_ok,
|
||||
"tesseract_missing": tesseract_missing,
|
||||
"easyocr_failed": easyocr_failed,
|
||||
"message": (
|
||||
"All configured OCR languages are available."
|
||||
if all_ok
|
||||
else f"Some languages could not be installed: tesseract={tesseract_missing}, easyocr={easyocr_failed}"
|
||||
),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error during OCR language installation: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to install OCR language data",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{key}/history")
|
||||
async def get_key_history(key: str, request: Request, db: DbSession, admin: AdminUser):
|
||||
"""
|
||||
Get the change history for a specific setting key.
|
||||
|
||||
Returns all audit log entries for that key, most recent first.
|
||||
Admin only.
|
||||
"""
|
||||
validate_setting_key_format(key)
|
||||
try:
|
||||
entries = get_setting_history(db, key)
|
||||
return {"key": key, "history": entries}
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrieving history for {key}: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to retrieve history for setting: {key}",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{key}/rollback/{history_id}")
|
||||
async def rollback_setting_to_history(
|
||||
key: str,
|
||||
history_id: int,
|
||||
request: Request,
|
||||
db: DbSession,
|
||||
admin: AdminUser,
|
||||
):
|
||||
"""
|
||||
Revert a setting to the value it had *before* a specific audit log change.
|
||||
|
||||
The ``history_id`` is the ID of the :class:`~app.models.SettingsAuditLog`
|
||||
entry whose ``old_value`` should be reinstated, effectively undoing that
|
||||
change. If ``old_value`` is ``None`` (the setting did not exist before
|
||||
that change), the setting is removed from the database and reverts to its
|
||||
ENV/default value.
|
||||
|
||||
A new audit log entry is written to record the rollback.
|
||||
Admin only.
|
||||
"""
|
||||
validate_setting_key_format(key)
|
||||
try:
|
||||
user = request.session.get("user", {}) if hasattr(request, "session") else {}
|
||||
changed_by = (
|
||||
user.get("preferred_username") or user.get("username") or user.get("email") or user.get("id") or "admin"
|
||||
)
|
||||
|
||||
success = rollback_setting(db, key, history_id, changed_by=changed_by)
|
||||
if not success:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"History entry {history_id} not found for setting '{key}'",
|
||||
)
|
||||
|
||||
notify_settings_updated()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Setting '{key}' rolled back to history entry {history_id}",
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error rolling back setting {key} to history {history_id}: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to roll back setting: {key}",
|
||||
)
|
||||
@@ -1,293 +0,0 @@
|
||||
"""
|
||||
API endpoint for processing files from URLs
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import urllib.parse
|
||||
import uuid
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, HttpUrl, field_validator
|
||||
|
||||
from app.auth import require_login
|
||||
from app.config import settings
|
||||
from app.tasks.process_document import process_document
|
||||
from app.utils.allowed_types import ALLOWED_MIME_TYPES
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class URLUploadRequest(BaseModel):
|
||||
"""Request model for URL-based file upload"""
|
||||
|
||||
url: HttpUrl
|
||||
filename: Optional[str] = None
|
||||
|
||||
@field_validator("url")
|
||||
@classmethod
|
||||
def validate_url_scheme(cls, v):
|
||||
"""Ensure only HTTP/HTTPS schemes are allowed"""
|
||||
parsed = urllib.parse.urlparse(str(v))
|
||||
if parsed.scheme not in ["http", "https"]:
|
||||
raise ValueError("Only HTTP and HTTPS URLs are allowed")
|
||||
return v
|
||||
|
||||
|
||||
def is_private_ip(hostname: str) -> bool:
|
||||
"""
|
||||
Check if a hostname resolves to a private/internal IP address.
|
||||
Protects against SSRF attacks by blocking access to internal networks.
|
||||
"""
|
||||
try:
|
||||
# Try to parse as IP address directly
|
||||
ip = ipaddress.ip_address(hostname)
|
||||
return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
|
||||
except ValueError:
|
||||
# Not a direct IP, try to resolve hostname
|
||||
try:
|
||||
import socket
|
||||
|
||||
# Get all IP addresses for this hostname
|
||||
addr_info = socket.getaddrinfo(hostname, None)
|
||||
for info in addr_info:
|
||||
ip_str = info[4][0]
|
||||
ip = ipaddress.ip_address(ip_str)
|
||||
# Block if ANY resolved IP is private/internal
|
||||
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
|
||||
return True
|
||||
return False
|
||||
except (socket.gaierror, socket.error):
|
||||
# Cannot resolve - allow for testing/development
|
||||
# In production, DNS should work properly
|
||||
# Log this for debugging
|
||||
logger.warning(f"Could not resolve hostname: {hostname}")
|
||||
return False # Changed from True to False to allow external domains in tests
|
||||
|
||||
|
||||
def validate_url_safety(url: str) -> None:
|
||||
"""
|
||||
Validate that URL is safe to fetch (SSRF protection).
|
||||
|
||||
Raises:
|
||||
HTTPException: If URL is unsafe
|
||||
"""
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
|
||||
# Check scheme
|
||||
if parsed.scheme not in ["http", "https"]:
|
||||
raise HTTPException(status_code=400, detail="Only HTTP and HTTPS URLs are supported")
|
||||
|
||||
# Check hostname exists
|
||||
if not parsed.hostname:
|
||||
raise HTTPException(status_code=400, detail="Invalid URL: no hostname")
|
||||
|
||||
# Block private/internal IPs (SSRF protection)
|
||||
if is_private_ip(parsed.hostname):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Access to private/internal IP addresses is not allowed for security reasons",
|
||||
)
|
||||
|
||||
# Block well-known metadata endpoints (cloud provider SSRF)
|
||||
metadata_endpoints = [
|
||||
"169.254.169.254", # AWS, Azure, GCP metadata
|
||||
"metadata.google.internal", # GCP
|
||||
"169.254.169.253", # AWS link-local
|
||||
]
|
||||
if parsed.hostname in metadata_endpoints:
|
||||
raise HTTPException(status_code=400, detail="Access to cloud metadata endpoints is not allowed")
|
||||
|
||||
|
||||
def validate_file_type(content_type: str, filename: str) -> bool:
|
||||
"""
|
||||
Validate that the file type is supported (i.e. processable by Gotenberg).
|
||||
|
||||
Args:
|
||||
content_type: MIME type from response headers
|
||||
filename: Filename to check extension
|
||||
|
||||
Returns:
|
||||
True if file type is allowed
|
||||
"""
|
||||
# Check content type from header
|
||||
if content_type:
|
||||
# Handle content-type with charset (e.g., "application/pdf; charset=utf-8")
|
||||
base_content_type = content_type.split(";", maxsplit=1)[0].strip().lower()
|
||||
if base_content_type in ALLOWED_MIME_TYPES:
|
||||
return True
|
||||
|
||||
# Also check by extension as fallback
|
||||
_, ext = os.path.splitext(filename)
|
||||
if ext:
|
||||
guessed_type, _ = mimetypes.guess_type(filename)
|
||||
if guessed_type and guessed_type in ALLOWED_MIME_TYPES:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
@router.post("/process-url")
|
||||
@require_login
|
||||
async def process_url(request: Request, url_request: URLUploadRequest):
|
||||
"""
|
||||
Download a file from a URL and enqueue it for processing.
|
||||
|
||||
Security features:
|
||||
- SSRF protection: blocks private IPs, localhost, cloud metadata endpoints
|
||||
- File type validation: only allows supported document/image types
|
||||
- File size limits: enforces maximum upload size
|
||||
- Timeout protection: prevents hanging on slow/malicious servers
|
||||
|
||||
Args:
|
||||
request: Starlette Request object (used by require_login decorator)
|
||||
url_request: URLUploadRequest with url and optional filename
|
||||
|
||||
Returns:
|
||||
JSON with task_id and status
|
||||
|
||||
Raises:
|
||||
HTTPException: If URL is invalid, unsafe, or file cannot be processed
|
||||
"""
|
||||
url = str(url_request.url)
|
||||
|
||||
# Validate URL safety (SSRF protection)
|
||||
validate_url_safety(url)
|
||||
|
||||
# Parse URL to extract filename if not provided
|
||||
if url_request.filename:
|
||||
original_filename = url_request.filename
|
||||
else:
|
||||
# Extract filename from URL path
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
path = parsed.path
|
||||
original_filename = os.path.basename(path) if path else "download"
|
||||
|
||||
# Sanitize filename
|
||||
safe_filename = sanitize_filename(original_filename)
|
||||
if not safe_filename:
|
||||
safe_filename = "download"
|
||||
|
||||
# Download file with security measures
|
||||
# Initialize target_path to None to prevent UnboundLocalError in exception handlers
|
||||
# that may execute before target_path is assigned during error cases
|
||||
target_path = None
|
||||
try:
|
||||
logger.info(f"Downloading file from URL: {url}")
|
||||
|
||||
# Use configured timeout to prevent hanging
|
||||
response = requests.get(
|
||||
url,
|
||||
timeout=settings.http_request_timeout,
|
||||
stream=True, # Stream to handle large files
|
||||
allow_redirects=True, # Follow redirects
|
||||
headers={
|
||||
"User-Agent": "DocuElevate/1.0", # Identify ourselves
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Validate content type
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if not validate_file_type(content_type, safe_filename):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unsupported file type: {content_type}. "
|
||||
"Supported types: PDF, Office documents, images, plain text",
|
||||
)
|
||||
|
||||
# Check content length before downloading
|
||||
content_length = response.headers.get("Content-Length")
|
||||
if content_length:
|
||||
file_size = int(content_length)
|
||||
max_size = settings.max_upload_size
|
||||
if file_size > max_size:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: {file_size} bytes (max {max_size} bytes)",
|
||||
)
|
||||
|
||||
# Generate unique filename
|
||||
unique_id = str(uuid.uuid4())
|
||||
if "." in safe_filename:
|
||||
file_extension = safe_filename.rsplit(".", 1)[1]
|
||||
target_filename = f"{unique_id}.{file_extension}"
|
||||
else:
|
||||
target_filename = unique_id
|
||||
|
||||
target_path = os.path.join(settings.workdir, target_filename)
|
||||
|
||||
# Download file in chunks to handle large files
|
||||
downloaded_size = 0
|
||||
max_size = settings.max_upload_size
|
||||
|
||||
with open(target_path, "wb") as f:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
downloaded_size += len(chunk)
|
||||
|
||||
# Check size during download
|
||||
if downloaded_size > max_size:
|
||||
# Remove partial file
|
||||
f.close()
|
||||
os.remove(target_path)
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=f"File too large: exceeded {max_size} bytes during download",
|
||||
)
|
||||
|
||||
logger.info(f"Downloaded file from URL '{url}' as '{target_filename}' ({downloaded_size} bytes)")
|
||||
|
||||
# Enqueue for processing
|
||||
task = process_document.delay(target_path, original_filename=safe_filename)
|
||||
|
||||
return {
|
||||
"task_id": task.id,
|
||||
"status": "queued",
|
||||
"message": "File downloaded from URL and queued for processing",
|
||||
"filename": safe_filename,
|
||||
"size": downloaded_size,
|
||||
}
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
logger.error(f"Timeout while downloading file from URL: {url}")
|
||||
raise HTTPException(status_code=408, detail="Request timeout: server took too long to respond")
|
||||
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
logger.error(f"Connection error while downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=502, detail=f"Failed to connect to URL: {str(e)}")
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
logger.error(f"HTTP error while downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=e.response.status_code, detail=f"HTTP error: {str(e)}")
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Error downloading file from URL: {url} - {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to download file: {str(e)}")
|
||||
|
||||
except HTTPException:
|
||||
# Re-raise FastAPI HTTPExceptions (validation errors, file too large, etc.)
|
||||
raise
|
||||
|
||||
except OSError as e:
|
||||
logger.error(f"Error saving file from URL: {url} - {str(e)}")
|
||||
# Clean up partial file if it exists
|
||||
if target_path and os.path.exists(target_path):
|
||||
os.remove(target_path)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}")
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error processing URL: {url}")
|
||||
# Clean up partial file if it exists
|
||||
if target_path and os.path.exists(target_path):
|
||||
os.remove(target_path)
|
||||
raise HTTPException(status_code=500, detail=f"Unexpected error: {str(e)}")
|
||||
@@ -1,48 +0,0 @@
|
||||
"""
|
||||
User-related API endpoints
|
||||
"""
|
||||
|
||||
import logging
|
||||
from hashlib import md5
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
|
||||
# Set up logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
async def whoami_handler(request: Request):
|
||||
"""
|
||||
Returns user info if logged in, else 401.
|
||||
"""
|
||||
user = request.session.get("user")
|
||||
if not user:
|
||||
raise HTTPException(status_code=401, detail="Not logged in")
|
||||
|
||||
email = user.get("email")
|
||||
if not email:
|
||||
raise HTTPException(status_code=400, detail="User has no email in session")
|
||||
|
||||
# Generate Gravatar URL from email
|
||||
# MD5 is used here for Gravatar's URL generation (not for security), so usedforsecurity=False
|
||||
email_hash = md5(email.strip().lower().encode(), usedforsecurity=False).hexdigest()
|
||||
gravatar_url = f"https://www.gravatar.com/avatar/{email_hash}?d=identicon"
|
||||
|
||||
# Add the gravatar URL to the user object instead of creating a new response
|
||||
user_response = user.copy() # Create a copy to avoid modifying the session
|
||||
user_response["picture"] = gravatar_url
|
||||
|
||||
return user_response
|
||||
|
||||
|
||||
# Register the same handler under two different paths
|
||||
@router.get("/whoami")
|
||||
async def whoami(request: Request):
|
||||
return await whoami_handler(request)
|
||||
|
||||
|
||||
@router.get("/auth/whoami")
|
||||
async def auth_whoami(request: Request):
|
||||
return await whoami_handler(request)
|
||||
+17
-140
@@ -1,31 +1,18 @@
|
||||
import hashlib
|
||||
import os
|
||||
import inspect
|
||||
import logging
|
||||
import pathlib
|
||||
from functools import wraps
|
||||
|
||||
from authlib.integrations.starlette_client import OAuth
|
||||
from fastapi import APIRouter, Request, status
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from starlette.responses import RedirectResponse
|
||||
|
||||
from app.config import settings
|
||||
|
||||
oauth = OAuth()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
AUTH_ENABLED = settings.auth_enabled
|
||||
|
||||
# Set up templates for authentication
|
||||
templates_dir = pathlib.Path(__file__).parents[1] / "frontend" / "templates"
|
||||
templates = Jinja2Templates(directory=str(templates_dir))
|
||||
|
||||
# Configure OAuth provider if credentials are provided
|
||||
OAUTH_CONFIGURED = False
|
||||
OAUTH_PROVIDER_NAME = "Single Sign-On"
|
||||
|
||||
if AUTH_ENABLED and settings.authentik_client_id and settings.authentik_client_secret:
|
||||
if AUTH_ENABLED:
|
||||
oauth.register(
|
||||
name="authentik",
|
||||
client_id=settings.authentik_client_id,
|
||||
@@ -33,8 +20,6 @@ if AUTH_ENABLED and settings.authentik_client_id and settings.authentik_client_s
|
||||
server_metadata_url=settings.authentik_config_url,
|
||||
client_kwargs={"scope": "openid profile email"},
|
||||
)
|
||||
OAUTH_CONFIGURED = True
|
||||
OAUTH_PROVIDER_NAME = settings.oauth_provider_name or "Authentik SSO"
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -61,137 +46,29 @@ def require_login(func):
|
||||
return wrapper
|
||||
|
||||
|
||||
def get_gravatar_url(email):
|
||||
"""Generate a Gravatar URL for the given email"""
|
||||
email = email.lower().strip()
|
||||
# MD5 is used here for Gravatar's URL generation (not for security), so usedforsecurity=False
|
||||
email_hash = hashlib.md5(email.encode("utf-8"), usedforsecurity=False).hexdigest()
|
||||
return f"https://www.gravatar.com/avatar/{email_hash}?d=identicon"
|
||||
if AUTH_ENABLED:
|
||||
@router.get("/login")
|
||||
async def login(request: Request):
|
||||
redirect_uri = request.url_for("auth")
|
||||
return await oauth.authentik.authorize_redirect(request, redirect_uri)
|
||||
|
||||
|
||||
async def login(request: Request):
|
||||
"""Show login page with appropriate authentication options"""
|
||||
return templates.TemplateResponse(
|
||||
"login.html",
|
||||
{
|
||||
"request": request,
|
||||
"error": request.query_params.get("error"),
|
||||
"message": request.query_params.get("message"),
|
||||
"show_oauth": OAUTH_CONFIGURED,
|
||||
"oauth_provider_name": OAUTH_PROVIDER_NAME,
|
||||
"app_version": settings.version, # Changed from app_version to version
|
||||
"csrf_token": getattr(request.state, "csrf_token", ""),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def oauth_login(request: Request):
|
||||
"""Handle OAuth login flow"""
|
||||
if not OAUTH_CONFIGURED:
|
||||
return RedirectResponse(url="/login?error=OAuth+not+configured", status_code=status.HTTP_302_FOUND)
|
||||
|
||||
redirect_uri = request.url_for("oauth_callback")
|
||||
return await oauth.authentik.authorize_redirect(request, redirect_uri)
|
||||
|
||||
|
||||
async def oauth_callback(request: Request):
|
||||
"""Handle OAuth callback from provider"""
|
||||
try:
|
||||
@router.get("/auth")
|
||||
async def auth(request: Request):
|
||||
token = await oauth.authentik.authorize_access_token(request)
|
||||
userinfo = token.get("userinfo")
|
||||
if not userinfo:
|
||||
return RedirectResponse(
|
||||
url="/login?error=Failed+to+retrieve+user+information", status_code=status.HTTP_302_FOUND
|
||||
)
|
||||
|
||||
# Store user info in session
|
||||
user_data = dict(userinfo)
|
||||
|
||||
# Add Gravatar picture if no picture is provided
|
||||
if not user_data.get("picture") and user_data.get("email"):
|
||||
user_data["picture"] = get_gravatar_url(user_data["email"])
|
||||
|
||||
# Check if user is admin based on OAuth groups or specific email
|
||||
# You can customize this logic based on your OAuth provider's attributes
|
||||
# For example, check if user has an "admin" group or specific email domain
|
||||
is_admin = False
|
||||
if "groups" in user_data:
|
||||
# Check if user is in admin group
|
||||
groups = user_data.get("groups", [])
|
||||
admin_group = (settings.admin_group_name or "admin").strip().lower()
|
||||
is_admin = admin_group in [group.lower() for group in groups]
|
||||
|
||||
# Set is_admin flag (defaults to False for OAuth users unless they're in admin group)
|
||||
user_data["is_admin"] = is_admin
|
||||
|
||||
request.session["user"] = user_data
|
||||
|
||||
# Log the successful authentication
|
||||
logger.info(f"[SECURITY] OAUTH_LOGIN_SUCCESS user={user_data.get('email', 'unknown')} admin={is_admin}")
|
||||
|
||||
# Redirect to original destination or default
|
||||
request.session["user"] = dict(userinfo)
|
||||
redirect_url = request.session.pop("redirect_after_login", "/upload")
|
||||
return RedirectResponse(url=redirect_url, status_code=status.HTTP_302_FOUND)
|
||||
except Exception as e:
|
||||
logger.warning(f"[SECURITY] OAUTH_LOGIN_FAILURE error={type(e).__name__}")
|
||||
return RedirectResponse(url=f"/login?error=Authentication+failed:+{str(e)}", status_code=status.HTTP_302_FOUND)
|
||||
return RedirectResponse(url=redirect_url)
|
||||
|
||||
|
||||
async def auth(request: Request):
|
||||
"""Handle local username/password authentication"""
|
||||
form_data = await request.form()
|
||||
username = form_data.get("username")
|
||||
password = form_data.get("password")
|
||||
|
||||
if username == settings.admin_username and password == settings.admin_password:
|
||||
# Create user session
|
||||
request.session["user"] = {
|
||||
"id": "admin",
|
||||
"name": "Administrator",
|
||||
"email": f"{username}@local.docuelevate",
|
||||
"preferred_username": username,
|
||||
"picture": "/static/images/default-avatar.svg",
|
||||
"is_admin": True,
|
||||
}
|
||||
logger.info(f"[SECURITY] LOCAL_LOGIN_SUCCESS user={username}")
|
||||
# Redirect to original destination or default
|
||||
redirect_url = request.session.pop("redirect_after_login", "/upload")
|
||||
return RedirectResponse(url=redirect_url, status_code=302)
|
||||
else:
|
||||
logger.warning(f"[SECURITY] LOCAL_LOGIN_FAILURE user={username}")
|
||||
return RedirectResponse(url="/login?error=Invalid+username+or+password", status_code=302)
|
||||
|
||||
|
||||
async def logout(request: Request):
|
||||
"""Handle user logout"""
|
||||
user = request.session.get("user")
|
||||
username = "unknown"
|
||||
if isinstance(user, dict):
|
||||
username = user.get("preferred_username") or user.get("email") or "unknown"
|
||||
logger.info(f"[SECURITY] LOGOUT user={username}")
|
||||
request.session.pop("user", None)
|
||||
return RedirectResponse(url="/login?message=You+have+been+logged+out+successfully", status_code=302)
|
||||
|
||||
|
||||
if AUTH_ENABLED:
|
||||
router.add_api_route("/login", login, methods=["GET"])
|
||||
router.add_api_route("/oauth-login", oauth_login, methods=["GET"])
|
||||
router.add_api_route("/oauth-callback", oauth_callback, methods=["GET"])
|
||||
router.add_api_route("/auth", auth, methods=["POST"])
|
||||
router.add_api_route("/logout", logout, methods=["GET"])
|
||||
|
||||
|
||||
@router.get("/api/auth/whoami")
|
||||
@require_login
|
||||
async def whoami(request: Request):
|
||||
"""API endpoint to get current user information"""
|
||||
user = request.session.get("user")
|
||||
return user or {"error": "Not authenticated"}
|
||||
@router.get("/logout")
|
||||
async def logout(request: Request):
|
||||
request.session.pop("user", None)
|
||||
return RedirectResponse(url="/")
|
||||
|
||||
|
||||
@router.get("/private")
|
||||
@require_login
|
||||
async def private_page(request: Request):
|
||||
"""A protected endpoint that requires login."""
|
||||
user = request.session.get("user")
|
||||
return {"message": "This is a protected page.", "user": user}
|
||||
user = request.session.get("user") # e.g. {"email": "...", ...}
|
||||
return {"message": f"This is a protected page. Hello {user['email']}!"}
|
||||
|
||||
+1
-26
@@ -1,8 +1,6 @@
|
||||
# app/celery_app.py
|
||||
|
||||
from celery import Celery
|
||||
from celery.signals import task_failure
|
||||
|
||||
from app.config import settings
|
||||
|
||||
celery = Celery(
|
||||
@@ -16,30 +14,7 @@ celery = Celery(
|
||||
celery.conf.broker_connection_retry_on_startup = True
|
||||
|
||||
# Set the default queue and routing so that tasks are enqueued on "document_processor"
|
||||
celery.conf.task_default_queue = "document_processor"
|
||||
celery.conf.task_default_queue = 'document_processor'
|
||||
celery.conf.task_routes = {
|
||||
"app.tasks.*": {"queue": "document_processor"},
|
||||
}
|
||||
|
||||
|
||||
@task_failure.connect
|
||||
def task_failure_handler(
|
||||
sender=None, task_id=None, exception=None, args=None, kwargs=None, traceback=None, einfo=None, **kw
|
||||
):
|
||||
"""Handler for Celery task failures to send notifications"""
|
||||
if getattr(settings, "notify_on_task_failure", True):
|
||||
try:
|
||||
# Import here to avoid circular imports
|
||||
from app.utils.notification import notify_celery_failure
|
||||
|
||||
notify_celery_failure(
|
||||
task_name=sender.name if sender else "Unknown",
|
||||
task_id=task_id or "N/A",
|
||||
exc=exception,
|
||||
args=args or [],
|
||||
kwargs=kwargs or {},
|
||||
)
|
||||
except Exception as e:
|
||||
import logging
|
||||
|
||||
logging.exception(f"Failed to send task failure notification: {e}")
|
||||
|
||||
+21
-76
@@ -1,97 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from celery.schedules import crontab
|
||||
|
||||
# Ensure tasks are loaded
|
||||
from app import tasks # noqa: F401 - Imports app/tasks.py so Celery can register tasks
|
||||
from app.config import settings
|
||||
|
||||
# Import the shared Celery instance
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.tasks.check_credentials import check_credentials
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf # noqa: F401
|
||||
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf # noqa: F401
|
||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt # noqa: F401
|
||||
from app.tasks.imap_tasks import pull_all_inboxes # noqa: F401
|
||||
from app.tasks.monitor_stalled_steps import monitor_stalled_steps # noqa: F401
|
||||
|
||||
# Ensure tasks are loaded
|
||||
from app import tasks # <— This imports app/tasks.py so Celery can register tasks
|
||||
|
||||
# **Ensure all tasks are imported before Celery starts**
|
||||
from app.tasks.process_document import process_document # noqa: F401
|
||||
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence # noqa: F401
|
||||
from app.tasks.refine_text_with_gpt import refine_text_with_gpt # noqa: F401
|
||||
from app.tasks.rotate_pdf_pages import rotate_pdf_pages # noqa: F401
|
||||
from app.tasks.send_to_all import send_to_all_destinations # noqa: F401
|
||||
from app.tasks.process_document import process_document # Updated import
|
||||
from app.tasks.process_with_textract import process_with_textract
|
||||
from app.tasks.refine_text_with_gpt import refine_text_with_gpt
|
||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
||||
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf
|
||||
|
||||
# Import new send tasks
|
||||
from app.tasks.upload_to_dropbox import upload_to_dropbox # noqa: F401
|
||||
from app.tasks.upload_to_email import upload_to_email # noqa: F401
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp # noqa: F401
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive # noqa: F401
|
||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud # noqa: F401
|
||||
from app.tasks.upload_to_onedrive import upload_to_onedrive # noqa: F401
|
||||
from app.tasks.upload_to_paperless import upload_to_paperless # noqa: F401
|
||||
from app.tasks.upload_to_s3 import upload_to_s3 # noqa: F401
|
||||
from app.tasks.upload_to_sftp import upload_to_sftp # noqa: F401
|
||||
from app.tasks.upload_to_webdav import upload_to_webdav # noqa: F401
|
||||
from app.tasks.uptime_kuma_tasks import ping_uptime_kuma # noqa: F401
|
||||
|
||||
# Register the settings reload signal handler so workers pick up config changes
|
||||
from app.utils.settings_sync import register_settings_reload_signal
|
||||
|
||||
register_settings_reload_signal()
|
||||
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
||||
from app.tasks.upload_to_paperless import upload_to_paperless
|
||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||
from app.tasks.imap_tasks import pull_all_inboxes
|
||||
from app.tasks.send_to_all import send_to_all_destinations
|
||||
|
||||
celery.conf.task_routes = {
|
||||
"app.tasks.*": {"queue": "default"},
|
||||
}
|
||||
|
||||
|
||||
@celery.task
|
||||
def test_task():
|
||||
return "Celery is working!"
|
||||
|
||||
|
||||
# Run the check_credentials task at startup
|
||||
check_credentials.apply_async(countdown=10) # Run 10 seconds after worker starts
|
||||
# If you want Celery Beat to run the poll task every minute, add:
|
||||
from celery.schedules import crontab
|
||||
|
||||
celery.conf.beat_schedule = {
|
||||
"poll-inboxes-every-minute": (
|
||||
{
|
||||
"task": "app.tasks.imap_tasks.pull_all_inboxes",
|
||||
"schedule": crontab(minute="*/1"), # every 1 minute
|
||||
"options": {"expires": 55}, # Ensure tasks don't pile up
|
||||
}
|
||||
if (settings.imap1_host or settings.imap2_host)
|
||||
else None
|
||||
),
|
||||
# Add Uptime Kuma ping task if configured
|
||||
"ping-uptime-kuma": (
|
||||
{
|
||||
"task": "app.tasks.uptime_kuma_tasks.ping_uptime_kuma",
|
||||
"schedule": crontab(minute=f"*/{settings.uptime_kuma_ping_interval}"),
|
||||
"options": {"expires": 55}, # Ensure tasks don't pile up
|
||||
}
|
||||
if settings.uptime_kuma_url
|
||||
else None
|
||||
),
|
||||
# Check credentials every 5 minutes
|
||||
"check-credentials-regularly": {
|
||||
"task": "app.tasks.check_credentials.check_credentials",
|
||||
"schedule": crontab(minute="*/5"), # Every 5 minutes
|
||||
"options": {"expires": 240}, # 4 minutes expiry
|
||||
"poll-inboxes-every-minute": {
|
||||
"task": "app.tasks.imap_tasks.pull_all_inboxes",
|
||||
"schedule": crontab(minute="*/1"), # every 1 minute
|
||||
},
|
||||
# Also keep daily check for logs and statistics purposes
|
||||
"check-credentials-daily": {
|
||||
"task": "app.tasks.check_credentials.check_credentials",
|
||||
"schedule": crontab(hour="0", minute="0"), # Midnight
|
||||
"options": {"expires": 3600}, # 1 hour expiry
|
||||
},
|
||||
# Monitor for stalled processing steps every minute
|
||||
"monitor-stalled-steps": {
|
||||
"task": "app.tasks.monitor_stalled_steps.monitor_stalled_steps",
|
||||
"schedule": crontab(minute="*/1"), # Every minute
|
||||
"options": {"expires": 55}, # Must complete within 55 seconds
|
||||
},
|
||||
}
|
||||
|
||||
# Remove None entries from beat_schedule
|
||||
celery.conf.beat_schedule = {k: v for k, v in celery.conf.beat_schedule.items() if v is not None}
|
||||
}
|
||||
+22
-564
@@ -1,125 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
from typing import Any, List, Optional, Union
|
||||
|
||||
from pydantic import Field, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
from pydantic_settings import BaseSettings
|
||||
from typing import Optional
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env")
|
||||
|
||||
admin_username: str
|
||||
admin_password: str
|
||||
aws_access_key_id: str
|
||||
aws_secret_access_key: str
|
||||
aws_region: str
|
||||
database_url: str
|
||||
redis_url: str
|
||||
s3_bucket_name: str
|
||||
openai_api_key: str
|
||||
openai_base_url: str = "https://api.openai.com/v1" # Default to OpenAI's endpoint
|
||||
openai_model: str = "gpt-4o-mini" # Default model
|
||||
|
||||
# AI provider abstraction layer
|
||||
# Supported values: openai, azure, anthropic, gemini, ollama, openrouter, litellm
|
||||
ai_provider: str = "openai"
|
||||
# Override model for any provider; falls back to openai_model when not set
|
||||
ai_model: Optional[str] = None
|
||||
|
||||
# Anthropic Claude settings (used when ai_provider="anthropic")
|
||||
anthropic_api_key: Optional[str] = None
|
||||
|
||||
# Google Gemini settings (used when ai_provider="gemini")
|
||||
gemini_api_key: Optional[str] = None
|
||||
|
||||
# Ollama local LLM settings (used when ai_provider="ollama")
|
||||
ollama_base_url: str = "http://localhost:11434"
|
||||
|
||||
# OpenRouter settings (used when ai_provider="openrouter")
|
||||
openrouter_api_key: Optional[str] = None
|
||||
openrouter_base_url: str = "https://openrouter.ai/api/v1"
|
||||
|
||||
# Portkey AI gateway settings (used when ai_provider="portkey")
|
||||
# See https://portkey.ai for setup instructions
|
||||
portkey_api_key: Optional[str] = None
|
||||
portkey_virtual_key: Optional[str] = None # Routes to a specific provider via Portkey vault
|
||||
portkey_config: Optional[str] = None # Portkey Config ID for advanced routing rules
|
||||
portkey_base_url: str = "https://api.portkey.ai/v1"
|
||||
|
||||
# Azure OpenAI API version (used when ai_provider="azure")
|
||||
azure_openai_api_version: str = "2024-02-01"
|
||||
workdir: str
|
||||
debug: bool = False # Default to False
|
||||
|
||||
# Making Dropbox optional
|
||||
dropbox_app_key: Optional[str] = None
|
||||
dropbox_app_secret: Optional[str] = None
|
||||
dropbox_folder: Optional[str] = None
|
||||
dropbox_refresh_token: Optional[str] = None
|
||||
|
||||
# Making Nextcloud optional
|
||||
nextcloud_upload_url: Optional[str] = None
|
||||
nextcloud_username: Optional[str] = None
|
||||
nextcloud_password: Optional[str] = None
|
||||
nextcloud_folder: Optional[str] = None
|
||||
|
||||
# Making Paperless optional
|
||||
paperless_ngx_api_token: Optional[str] = None
|
||||
paperless_host: Optional[str] = None
|
||||
paperless_custom_field_absender: Optional[str] = None # Name of the "absender" custom field in Paperless
|
||||
# JSON mapping of metadata field names to Paperless custom field names
|
||||
# Example: {"absender": "Sender", "empfaenger": "Recipient",
|
||||
# "language": "Language", "correspondent": "Correspondent"}
|
||||
paperless_custom_fields_mapping: Optional[str] = None
|
||||
|
||||
dropbox_app_key: str
|
||||
dropbox_app_secret: str
|
||||
dropbox_folder: str
|
||||
dropbox_refresh_token: str
|
||||
nextcloud_upload_url: str
|
||||
nextcloud_username: str
|
||||
nextcloud_password: str
|
||||
nextcloud_folder: str
|
||||
paperless_ngx_api_token: str
|
||||
paperless_host: str
|
||||
azure_ai_key: str
|
||||
azure_region: str
|
||||
azure_endpoint: str
|
||||
gotenberg_url: str
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OCR provider settings
|
||||
# ---------------------------------------------------------------------------
|
||||
# Comma-separated list of OCR engines to use.
|
||||
# Supported values: azure, tesseract, easyocr, mistral, google_docai, aws_textract
|
||||
# When multiple engines are listed all are run and results are merged.
|
||||
# Example: OCR_PROVIDERS=azure,tesseract
|
||||
ocr_providers: str = "azure"
|
||||
|
||||
# Strategy for merging results from multiple OCR providers.
|
||||
# - ai_merge : Ask the AI model to produce the best merged text (default).
|
||||
# - longest : Return the result with the most characters.
|
||||
# - primary : Return only the first provider's result (no merging).
|
||||
ocr_merge_strategy: str = "ai_merge"
|
||||
|
||||
# Tesseract OCR settings (used when "tesseract" is in OCR_PROVIDERS)
|
||||
tesseract_cmd: Optional[str] = None # Path to tesseract binary (e.g. /usr/bin/tesseract)
|
||||
tesseract_language: str = "eng+deu" # Tesseract language code(s), e.g. "eng" or "eng+deu"
|
||||
|
||||
# EasyOCR settings (used when "easyocr" is in OCR_PROVIDERS)
|
||||
easyocr_languages: str = "en,de" # Comma-separated language codes, e.g. "en,de,fr"
|
||||
easyocr_gpu: bool = False # Enable GPU acceleration for EasyOCR
|
||||
|
||||
# Mistral OCR settings (used when "mistral" is in OCR_PROVIDERS)
|
||||
mistral_api_key: Optional[str] = None
|
||||
mistral_ocr_model: str = "mistral-ocr-latest"
|
||||
|
||||
# Google Cloud Document AI settings (used when "google_docai" is in OCR_PROVIDERS)
|
||||
# Falls back to google_drive_credentials_json for service account credentials.
|
||||
google_docai_credentials_json: Optional[str] = None
|
||||
google_docai_project_id: Optional[str] = None
|
||||
google_docai_processor_id: Optional[str] = None
|
||||
google_docai_location: str = "us" # Processor location, e.g. "us" or "eu"
|
||||
external_hostname: str = "localhost" # Default to localhost
|
||||
|
||||
# Authentication settings
|
||||
auth_enabled: bool = True # Default to enabled
|
||||
admin_username: Optional[str] = None
|
||||
admin_password: Optional[str] = None
|
||||
session_secret: Optional[str] = None
|
||||
admin_group_name: str = "admin"
|
||||
|
||||
|
||||
# Authentik
|
||||
authentik_client_id: Optional[str] = None
|
||||
authentik_client_secret: Optional[str] = None
|
||||
authentik_config_url: Optional[str] = None
|
||||
oauth_provider_name: Optional[str] = None # Name to display for the OAuth provider
|
||||
auth_enabled: bool = True # Default to enabled
|
||||
|
||||
# IMAP 1
|
||||
imap1_host: Optional[str] = None
|
||||
@@ -139,465 +55,7 @@ class Settings(BaseSettings):
|
||||
imap2_poll_interval_minutes: int = 10
|
||||
imap2_delete_after_process: bool = False
|
||||
|
||||
# Google Drive settings
|
||||
google_drive_credentials_json: Optional[str] = ""
|
||||
google_drive_folder_id: Optional[str] = ""
|
||||
google_drive_delegate_to: Optional[str] = "" # Optional delegated user email
|
||||
|
||||
# Google Drive OAuth settings
|
||||
google_drive_use_oauth: bool = False # Default to service account method
|
||||
google_drive_client_id: Optional[str] = ""
|
||||
google_drive_client_secret: Optional[str] = ""
|
||||
google_drive_refresh_token: Optional[str] = ""
|
||||
|
||||
# WebDAV settings
|
||||
webdav_url: Optional[str] = None
|
||||
webdav_username: Optional[str] = None
|
||||
webdav_password: Optional[str] = None
|
||||
webdav_folder: Optional[str] = None
|
||||
webdav_verify_ssl: bool = True
|
||||
|
||||
# FTP settings
|
||||
ftp_host: Optional[str] = None
|
||||
ftp_port: Optional[int] = 21
|
||||
ftp_username: Optional[str] = None
|
||||
ftp_password: Optional[str] = None
|
||||
ftp_folder: Optional[str] = None
|
||||
ftp_use_tls: bool = True # Default to attempting TLS connection first
|
||||
ftp_allow_plaintext: bool = True # Default to allowing plaintext fallback
|
||||
|
||||
# SFTP settings
|
||||
sftp_host: Optional[str] = None
|
||||
sftp_port: Optional[int] = 22
|
||||
sftp_username: Optional[str] = None
|
||||
sftp_password: Optional[str] = None
|
||||
sftp_folder: Optional[str] = None
|
||||
sftp_private_key: Optional[str] = None
|
||||
sftp_private_key_passphrase: Optional[str] = None
|
||||
# Security: Host key verification is enabled by default for security
|
||||
# In development/testing, set to True to disable verification (not recommended)
|
||||
sftp_disable_host_key_verification: bool = False # Default enforces host key verification
|
||||
|
||||
# Email settings
|
||||
email_host: Optional[str] = None
|
||||
email_port: Optional[int] = 587
|
||||
email_username: Optional[str] = None
|
||||
email_password: Optional[str] = None
|
||||
email_use_tls: bool = True
|
||||
email_sender: Optional[str] = None # From address, defaults to email_username if not set
|
||||
email_default_recipient: Optional[str] = None
|
||||
|
||||
# OneDrive settings
|
||||
onedrive_client_id: Optional[str] = None
|
||||
onedrive_client_secret: Optional[str] = None
|
||||
onedrive_tenant_id: Optional[str] = "common" # Default to "common" for personal accounts
|
||||
onedrive_refresh_token: Optional[str] = None # Required for personal accounts
|
||||
onedrive_folder_path: Optional[str] = None
|
||||
|
||||
# AWS S3 settings
|
||||
aws_access_key_id: Optional[str] = None
|
||||
aws_secret_access_key: Optional[str] = None
|
||||
aws_region: Optional[str] = "us-east-1" # Default region
|
||||
s3_bucket_name: Optional[str] = None
|
||||
s3_folder_prefix: Optional[str] = "" # Optional folder prefix (e.g. "uploads/")
|
||||
s3_storage_class: Optional[str] = "STANDARD" # Default storage class
|
||||
s3_acl: Optional[str] = "private" # Default ACL
|
||||
|
||||
# Uptime Kuma settings
|
||||
uptime_kuma_url: Optional[str] = None
|
||||
uptime_kuma_ping_interval: int = 5 # Default ping interval in minutes
|
||||
|
||||
# Meilisearch settings (full-text search engine)
|
||||
# Default uses the Docker Compose / K8s service name so container-to-container
|
||||
# networking works without any extra configuration. Override to
|
||||
# "http://localhost:7700" only when running the API process outside of Docker.
|
||||
meilisearch_url: str = "http://meilisearch:7700"
|
||||
meilisearch_api_key: Optional[str] = None # Master or API key (optional for local dev)
|
||||
meilisearch_index_name: str = "documents"
|
||||
enable_search: bool = True # Enable Meilisearch full-text search integration
|
||||
|
||||
# HTTP request settings
|
||||
http_request_timeout: int = 120 # Default timeout for HTTP requests in seconds (handles large file operations)
|
||||
|
||||
# Feature flags
|
||||
allow_file_delete: bool = True # Default to allowing file deletion from database
|
||||
imap_readonly_mode: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"When enabled, IMAP processing will fetch and process attachments but will NOT modify "
|
||||
"the mailbox state (no starring, labeling, deleting, or flag changes). "
|
||||
"Use this for pre-production instances that share a mailbox with production to prevent "
|
||||
"preprod from interfering with production email processing."
|
||||
),
|
||||
)
|
||||
|
||||
# Batch processing settings
|
||||
processall_throttle_threshold: int = Field(
|
||||
default=20,
|
||||
description="Number of files above which throttling is applied in /processall endpoint",
|
||||
)
|
||||
processall_throttle_delay: int = Field(
|
||||
default=3,
|
||||
description="Delay in seconds between each task submission when throttling in /processall",
|
||||
)
|
||||
|
||||
# Client-side upload throttling settings (applied when uploading files via the web UI)
|
||||
upload_concurrency: int = Field(
|
||||
default=3,
|
||||
description=(
|
||||
"Maximum number of files uploaded simultaneously from the browser. "
|
||||
"Limits parallel uploads to prevent API overload when dragging directories. Default: 3."
|
||||
),
|
||||
)
|
||||
upload_queue_delay_ms: int = Field(
|
||||
default=500,
|
||||
description=(
|
||||
"Delay in milliseconds between starting each upload slot when queue is active. "
|
||||
"Staggers upload starts to smooth out server load. Default: 500 ms."
|
||||
),
|
||||
)
|
||||
|
||||
# Notification settings
|
||||
notification_urls: Union[List[str], str] = Field(
|
||||
default_factory=list,
|
||||
description="List of Apprise notification URLs (e.g., discord://, telegram://, etc.)",
|
||||
)
|
||||
notify_on_task_failure: bool = Field(default=True, description="Send notifications when Celery tasks fail")
|
||||
notify_on_credential_failure: bool = Field(
|
||||
default=True, description="Send notifications when credential checks fail"
|
||||
)
|
||||
notify_on_startup: bool = Field(default=True, description="Send notifications when application starts")
|
||||
notify_on_shutdown: bool = Field(default=False, description="Send notifications when application shuts down")
|
||||
notify_on_file_processed: bool = Field(
|
||||
default=True,
|
||||
description="Send notifications when files are successfully processed",
|
||||
)
|
||||
|
||||
# File upload size limits (for security - see SECURITY_AUDIT.md)
|
||||
max_upload_size: int = Field(
|
||||
default=1073741824, # 1GB in bytes (1024 * 1024 * 1024)
|
||||
description="Maximum file upload size in bytes. Default: 1GB. Prevents resource exhaustion attacks.",
|
||||
)
|
||||
max_single_file_size: Optional[int] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Maximum size for a single file chunk in bytes. If set and file exceeds this,"
|
||||
" it will be split into smaller chunks for processing. Default: None (no splitting)."
|
||||
),
|
||||
)
|
||||
max_request_body_size: int = Field(
|
||||
default=1048576, # 1MB in bytes (1024 * 1024)
|
||||
description=(
|
||||
"Maximum request body size in bytes for non-file-upload requests. Default: 1MB."
|
||||
" Prevents memory exhaustion attacks via oversized JSON/form payloads."
|
||||
" File uploads are governed by MAX_UPLOAD_SIZE instead."
|
||||
),
|
||||
)
|
||||
|
||||
# Deduplication settings - prevents processing of duplicate files
|
||||
enable_deduplication: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"Enable deduplication check before processing. If enabled, files with the same SHA-256 hash"
|
||||
" as previously processed files will not be processed again. Default: True (enabled)."
|
||||
),
|
||||
)
|
||||
show_deduplication_step: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"Show the 'Check for Duplicates' step in processing history."
|
||||
" If False, the check is still performed but not displayed. Default: True."
|
||||
),
|
||||
)
|
||||
|
||||
# Text quality check - AI-based assessment of embedded PDF text
|
||||
enable_text_quality_check: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"Enable AI-based quality check for embedded PDF text. "
|
||||
"When enabled, text extracted from non-digital PDFs is evaluated by the AI model. "
|
||||
"If the text is poor quality (OCR artefacts, typos, incoherence), the file is "
|
||||
"re-processed with OCR instead of using the embedded text. "
|
||||
"Digitally-created PDFs (Word, LibreOffice, LaTeX, etc.) are always trusted and "
|
||||
"bypass the check. Default: True (enabled)."
|
||||
),
|
||||
)
|
||||
text_quality_threshold: int = Field(
|
||||
default=85,
|
||||
description=(
|
||||
"Minimum quality score (0–100) required to accept embedded PDF text without re-OCR. "
|
||||
"Text scoring below this threshold is discarded and the file is re-processed with OCR. "
|
||||
"Default: 85. The stricter this value, the more files will be re-OCR'd."
|
||||
),
|
||||
)
|
||||
text_quality_significant_issues: Union[List[str], str] = Field(
|
||||
default_factory=lambda: ["excessive_typos", "garbage_characters", "incoherent_text", "fragmented_sentences"],
|
||||
description=(
|
||||
"Comma-separated list of quality issue labels that force OCR re-run even when the quality "
|
||||
"score is above TEXT_QUALITY_THRESHOLD. Any of these issues present in the AI assessment "
|
||||
"will trigger re-OCR. Default: excessive_typos,garbage_characters,incoherent_text,fragmented_sentences"
|
||||
),
|
||||
)
|
||||
|
||||
# Processing step timeout - prevents files from getting stuck in "in_progress" state
|
||||
step_timeout: int = Field(
|
||||
default=600,
|
||||
description=(
|
||||
"Timeout in seconds for processing steps. If a step is 'in_progress' for longer than this,"
|
||||
" it will be marked as failed. Default: 600 seconds (10 minutes)."
|
||||
),
|
||||
)
|
||||
|
||||
# Security Headers Configuration (see SECURITY_AUDIT.md and docs/DeploymentGuide.md)
|
||||
# Disabled by default since most deployments use a reverse proxy (Traefik, Nginx, etc.)
|
||||
# that already adds these headers. Enable if deploying directly without a reverse proxy.
|
||||
security_headers_enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable security headers middleware. Set to True if deploying without reverse proxy.",
|
||||
)
|
||||
|
||||
# Strict-Transport-Security (HSTS) - Forces HTTPS connections
|
||||
security_header_hsts_enabled: bool = Field(
|
||||
default=True, description="Enable HSTS header. Only effective over HTTPS."
|
||||
)
|
||||
security_header_hsts_value: str = Field(
|
||||
default="max-age=31536000; includeSubDomains",
|
||||
description="HSTS header value. Default: 1 year with subdomains.",
|
||||
)
|
||||
|
||||
# Content-Security-Policy (CSP) - Controls resource loading
|
||||
security_header_csp_enabled: bool = Field(default=True, description="Enable CSP header.")
|
||||
security_header_csp_value: str = Field(
|
||||
default=(
|
||||
"default-src 'self'; script-src 'self' 'unsafe-inline';"
|
||||
" style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;"
|
||||
),
|
||||
description="CSP header value. Customize based on your application's resource loading needs.",
|
||||
)
|
||||
|
||||
# X-Frame-Options - Prevents clickjacking
|
||||
security_header_x_frame_options_enabled: bool = Field(default=True, description="Enable X-Frame-Options header.")
|
||||
security_header_x_frame_options_value: str = Field(
|
||||
default="DENY",
|
||||
description="X-Frame-Options header value. Options: DENY, SAMEORIGIN, or ALLOW-FROM uri",
|
||||
)
|
||||
|
||||
# X-Content-Type-Options - Prevents MIME sniffing
|
||||
security_header_x_content_type_options_enabled: bool = Field(
|
||||
default=True,
|
||||
description="Enable X-Content-Type-Options header (always set to 'nosniff').",
|
||||
)
|
||||
|
||||
# Audit Logging Configuration (see SECURITY_AUDIT.md – Infrastructure Security)
|
||||
# Logs every HTTP request and security-relevant events (auth failures, 5xx errors).
|
||||
# Sensitive query-parameter values (passwords, tokens, keys) are always masked.
|
||||
audit_logging_enabled: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"Enable audit/request logging middleware. Logs every HTTP request with "
|
||||
"method, path, status code, response time, and username. "
|
||||
"Sensitive query-parameter values are automatically masked."
|
||||
),
|
||||
)
|
||||
audit_log_include_client_ip: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"Include the client IP address in audit log entries. "
|
||||
"Disable for privacy-sensitive deployments where IP logging is restricted."
|
||||
),
|
||||
)
|
||||
|
||||
# UI / Appearance
|
||||
ui_default_color_scheme: str = Field(
|
||||
default="system",
|
||||
description=(
|
||||
"Default color scheme for the web interface. "
|
||||
"Options: 'system' (follow OS preference), 'light', 'dark'. "
|
||||
"Individual users can override this with the in-app toggle; their choice is persisted in localStorage."
|
||||
),
|
||||
)
|
||||
|
||||
# Rate Limiting Configuration (see SECURITY_AUDIT.md and docs/API.md)
|
||||
# Protects against DoS attacks and API abuse
|
||||
rate_limiting_enabled: bool = Field(
|
||||
default=True,
|
||||
description="Enable rate limiting middleware. Recommended for production to prevent abuse.",
|
||||
)
|
||||
rate_limit_default: str = Field(
|
||||
default="100/minute",
|
||||
description="Default rate limit for all endpoints (format: 'count/period', e.g., '100/minute', '1000/hour').",
|
||||
)
|
||||
rate_limit_upload: str = Field(
|
||||
default="600/minute",
|
||||
description="Rate limit for file upload endpoints to prevent resource exhaustion.",
|
||||
)
|
||||
rate_limit_auth: str = Field(
|
||||
default="10/minute",
|
||||
description="Stricter rate limit for authentication endpoints to prevent brute force attacks.",
|
||||
)
|
||||
|
||||
# CORS Configuration (see SECURITY_AUDIT.md – Infrastructure Security section)
|
||||
# Disabled by default since most deployments use a reverse proxy (Traefik, Nginx, etc.)
|
||||
# that already adds CORS headers. Enable only if deploying without a reverse proxy or if
|
||||
# the proxy does not handle CORS. See docs/DeploymentGuide.md for rationale.
|
||||
cors_enabled: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"Enable CORS middleware. Set to False if reverse proxy (Traefik, Nginx) handles CORS headers. "
|
||||
"When True, CORSMiddleware is added to the application with the settings below."
|
||||
),
|
||||
)
|
||||
cors_allowed_origins: Union[List[str], str] = Field(
|
||||
default_factory=lambda: ["*"],
|
||||
description=(
|
||||
"List of allowed CORS origins. Use ['*'] to allow all origins (not recommended with "
|
||||
"cors_allow_credentials=True). Comma-separated string is also accepted via env var, "
|
||||
"e.g. CORS_ALLOWED_ORIGINS=https://app.example.com,https://admin.example.com"
|
||||
),
|
||||
)
|
||||
cors_allow_credentials: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"Allow credentials (cookies, Authorization headers) in CORS requests. "
|
||||
"Cannot be True when cors_allowed_origins=['*']. "
|
||||
"When True, set cors_allowed_origins to specific origins."
|
||||
),
|
||||
)
|
||||
cors_allowed_methods: Union[List[str], str] = Field(
|
||||
default_factory=lambda: ["GET", "POST", "PUT", "DELETE", "OPTIONS", "PATCH"],
|
||||
description="Allowed HTTP methods for CORS requests.",
|
||||
)
|
||||
cors_allowed_headers: Union[List[str], str] = Field(
|
||||
default_factory=lambda: ["*"],
|
||||
description="Allowed request headers for CORS. Use ['*'] to allow all headers.",
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def strip_outer_quotes(cls, data: Any) -> Any:
|
||||
"""
|
||||
Strip matching surrounding quotes from string values.
|
||||
|
||||
In Kubernetes (and some other environments) env var values can arrive
|
||||
with literal quote characters included, e.g. the value for DATABASE_URL
|
||||
may be ``"postgresql://..."`` (with the quotes as part of the string)
|
||||
rather than just ``postgresql://...``. Docker Compose strips these
|
||||
automatically; Kubernetes does not.
|
||||
"""
|
||||
if isinstance(data, dict):
|
||||
for key, value in data.items():
|
||||
if isinstance(value, str) and len(value) >= 2:
|
||||
if (value[0] == '"' and value[-1] == '"') or (value[0] == "'" and value[-1] == "'"):
|
||||
data[key] = value[1:-1]
|
||||
return data
|
||||
|
||||
@field_validator("notification_urls", mode="before")
|
||||
@classmethod
|
||||
def parse_notification_urls(cls, v: str | list[str]) -> list[str]:
|
||||
"""Parse notification URLs from string or list"""
|
||||
if isinstance(v, str):
|
||||
if "," in v:
|
||||
return [url.strip() for url in v.split(",") if url.strip()]
|
||||
elif v.strip():
|
||||
return [v.strip()]
|
||||
return []
|
||||
return v
|
||||
|
||||
@field_validator("text_quality_significant_issues", mode="before")
|
||||
@classmethod
|
||||
def parse_text_quality_significant_issues(cls, v: str | list[str]) -> list[str]:
|
||||
"""Parse significant issue labels from comma-separated string or list."""
|
||||
if isinstance(v, str):
|
||||
if "," in v:
|
||||
return [item.strip() for item in v.split(",") if item.strip()]
|
||||
elif v.strip():
|
||||
return [v.strip()]
|
||||
return []
|
||||
return v
|
||||
|
||||
@field_validator("cors_allowed_origins", "cors_allowed_methods", "cors_allowed_headers", mode="before")
|
||||
@classmethod
|
||||
def parse_comma_separated_list(cls, v: str | list[str]) -> list[str]:
|
||||
"""Parse comma-separated string or list for CORS list settings."""
|
||||
if isinstance(v, str):
|
||||
if "," in v:
|
||||
return [item.strip() for item in v.split(",") if item.strip()]
|
||||
elif v.strip():
|
||||
return [v.strip()]
|
||||
return []
|
||||
return v
|
||||
|
||||
@field_validator("session_secret")
|
||||
@classmethod
|
||||
def validate_session_secret(cls, v: str | None, info: object) -> str | None:
|
||||
"""Validate that session_secret is set and has sufficient length when auth is enabled"""
|
||||
if info.data.get("auth_enabled") and not v:
|
||||
raise ValueError("SESSION_SECRET must be set when AUTH_ENABLED=True")
|
||||
if info.data.get("auth_enabled") and v and len(v) < 32:
|
||||
raise ValueError("SESSION_SECRET must be at least 32 characters long")
|
||||
return v
|
||||
|
||||
# Get build date from environment or file
|
||||
@property
|
||||
def build_date(self) -> str:
|
||||
# First try to get build date from environment
|
||||
env_build_date = os.environ.get("BUILD_DATE")
|
||||
if env_build_date:
|
||||
return env_build_date
|
||||
|
||||
# Then try to get build date from BUILD_DATE file
|
||||
build_date_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "BUILD_DATE")
|
||||
if os.path.exists(build_date_file):
|
||||
with open(build_date_file, "r") as f:
|
||||
return f.read().strip()
|
||||
|
||||
# Default to unknown if not found
|
||||
return "Unknown build date"
|
||||
|
||||
# Get version from file or environment
|
||||
@property
|
||||
def version(self) -> str:
|
||||
# First try to get version from environment
|
||||
env_version = os.environ.get("APP_VERSION")
|
||||
if env_version:
|
||||
return env_version
|
||||
|
||||
# Then try to get version from VERSION file
|
||||
version_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "VERSION")
|
||||
if os.path.exists(version_file):
|
||||
with open(version_file, "r") as f:
|
||||
return f.read().strip()
|
||||
|
||||
# Default version if not found
|
||||
return "unknown"
|
||||
|
||||
@property
|
||||
def git_sha(self) -> str:
|
||||
"""Get Git commit SHA from environment or file."""
|
||||
# First try to get from environment
|
||||
env_sha = os.environ.get("GIT_COMMIT_SHA")
|
||||
if env_sha:
|
||||
return env_sha
|
||||
|
||||
# Then try to get from GIT_SHA file
|
||||
git_sha_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "GIT_SHA")
|
||||
if os.path.exists(git_sha_file):
|
||||
with open(git_sha_file, "r") as f:
|
||||
return f.read().strip()
|
||||
|
||||
# Default if not found
|
||||
return "unknown"
|
||||
|
||||
@property
|
||||
def runtime_info(self) -> str:
|
||||
"""Get runtime information from file."""
|
||||
runtime_info_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), "RUNTIME_INFO")
|
||||
if os.path.exists(runtime_info_file):
|
||||
with open(runtime_info_file, "r") as f:
|
||||
return f.read().strip()
|
||||
|
||||
# Return basic info if file not found
|
||||
return f"Version: {self.version}\nBuild Date: {self.build_date}\nGit SHA: {self.git_sha}"
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
|
||||
settings = Settings()
|
||||
|
||||
+9
-216
@@ -1,15 +1,12 @@
|
||||
# app/database.py
|
||||
|
||||
import logging
|
||||
import os
|
||||
import warnings
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
import logging
|
||||
|
||||
from sqlalchemy import create_engine, exc
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.engine.url import make_url
|
||||
from sqlalchemy.orm import Session, declarative_base, sessionmaker
|
||||
|
||||
from app.config import settings
|
||||
|
||||
@@ -23,18 +20,18 @@ engine = create_engine(DB_URL, connect_args={"check_same_thread": False})
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
def init_db():
|
||||
"""
|
||||
Ensures the SQLite database file and its parent directory exist (if using sqlite).
|
||||
Then runs Base.metadata.create_all(bind=engine) to initialize tables and
|
||||
applies any pending Alembic migrations.
|
||||
Then runs Base.metadata.create_all(bind=engine) to initialize tables.
|
||||
Logs a message if a new SQLite DB file is created.
|
||||
"""
|
||||
# 1. Parse the DB URL to see if it's sqlite
|
||||
url = make_url(DB_URL)
|
||||
if url.get_backend_name() == "sqlite":
|
||||
# 2. Extract the database path from the URL
|
||||
database_path = url.database # e.g. "/workdir/db/database.db" or ":memory:"
|
||||
|
||||
|
||||
if database_path != ":memory:":
|
||||
# 3. Ensure directory exists
|
||||
db_dir = os.path.dirname(database_path)
|
||||
@@ -46,221 +43,17 @@ def init_db() -> None:
|
||||
if not os.path.exists(database_path):
|
||||
logger.info(f"Creating new SQLite database file at {database_path}")
|
||||
open(database_path, "a").close()
|
||||
|
||||
|
||||
# 5. Now create tables if they don't exist yet
|
||||
try:
|
||||
Base.metadata.create_all(bind=engine)
|
||||
logger.info("Database initialization complete (tables created if not exist).")
|
||||
|
||||
# 6. Run Alembic migrations for existing databases
|
||||
_run_alembic_upgrade(engine)
|
||||
except exc.SQLAlchemyError as e:
|
||||
logger.error(f"Error initializing database: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def _run_alembic_upgrade(engine: Any) -> None:
|
||||
"""Run Alembic migrations programmatically to apply pending schema changes.
|
||||
|
||||
For fresh databases (created via ``Base.metadata.create_all()``), the
|
||||
Alembic version is stamped to ``head`` because all tables already exist.
|
||||
For existing databases with Alembic tracking, any pending migrations are
|
||||
applied via ``alembic upgrade head``.
|
||||
|
||||
Args:
|
||||
engine: The SQLAlchemy engine connected to the target database.
|
||||
"""
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import inspect
|
||||
|
||||
inspector = inspect(engine)
|
||||
table_names = inspector.get_table_names()
|
||||
|
||||
# Locate the migrations directory relative to this file
|
||||
migrations_dir = str(Path(__file__).resolve().parent.parent / "migrations")
|
||||
|
||||
# Build an Alembic Config that points at our migration scripts.
|
||||
# The sqlalchemy.url is intentionally left empty because we pass the
|
||||
# live connection via config.attributes["connection"] below.
|
||||
alembic_cfg = Config()
|
||||
alembic_cfg.set_main_option("script_location", migrations_dir)
|
||||
alembic_cfg.set_main_option("sqlalchemy.url", "")
|
||||
|
||||
with engine.begin() as connection:
|
||||
alembic_cfg.attributes["connection"] = connection
|
||||
|
||||
if "alembic_version" not in table_names:
|
||||
# Fresh database or one that predates Alembic tracking.
|
||||
# Base.metadata.create_all() already created everything, so
|
||||
# stamp the current version to head (no migrations need to run).
|
||||
logger.info("No Alembic version table found — stamping database to latest revision.")
|
||||
command.stamp(alembic_cfg, "head")
|
||||
else:
|
||||
# Existing database with Alembic version tracking — apply pending migrations.
|
||||
logger.info("Running pending Alembic migrations…")
|
||||
command.upgrade(alembic_cfg, "head")
|
||||
logger.info("Alembic migration check complete.")
|
||||
|
||||
|
||||
def _run_schema_migrations(engine: Any) -> None:
|
||||
"""Apply lightweight schema migrations for columns added after the initial release.
|
||||
|
||||
.. deprecated::
|
||||
This function is deprecated and will be removed in a future release.
|
||||
All schema migrations are now managed exclusively through Alembic.
|
||||
Run ``alembic upgrade head`` (or let ``init_db()`` handle it
|
||||
automatically) instead of calling this function directly.
|
||||
|
||||
Each migration is idempotent and safe to run multiple times.
|
||||
"""
|
||||
warnings.warn(
|
||||
"_run_schema_migrations() is deprecated. "
|
||||
"All schema changes are now managed by Alembic migrations. "
|
||||
"Use 'alembic upgrade head' or init_db() instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
from sqlalchemy import inspect, text
|
||||
|
||||
inspector = inspect(engine)
|
||||
|
||||
# Migration: Add 'detail' column to processing_logs (added for verbose worker log output)
|
||||
if "processing_logs" in inspector.get_table_names():
|
||||
columns = [col["name"] for col in inspector.get_columns("processing_logs")]
|
||||
if "detail" not in columns:
|
||||
logger.info("Migrating processing_logs: adding 'detail' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE processing_logs ADD COLUMN detail TEXT"))
|
||||
logger.info("Migration complete: 'detail' column added to processing_logs")
|
||||
|
||||
# Migration: Add file path columns to files table
|
||||
if "files" in inspector.get_table_names():
|
||||
columns = [col["name"] for col in inspector.get_columns("files")]
|
||||
if "original_file_path" not in columns:
|
||||
logger.info("Migrating files: adding 'original_file_path' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN original_file_path VARCHAR"))
|
||||
logger.info("Migration complete: 'original_file_path' column added to files")
|
||||
|
||||
if "processed_file_path" not in columns:
|
||||
logger.info("Migrating files: adding 'processed_file_path' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN processed_file_path VARCHAR"))
|
||||
logger.info("Migration complete: 'processed_file_path' column added to files")
|
||||
|
||||
# Migration: Add deduplication columns to files table
|
||||
if "is_duplicate" not in columns:
|
||||
logger.info("Migrating files: adding 'is_duplicate' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN is_duplicate BOOLEAN DEFAULT FALSE NOT NULL"))
|
||||
logger.info("Migration complete: 'is_duplicate' column added to files")
|
||||
|
||||
if "duplicate_of_id" not in columns:
|
||||
logger.info("Migrating files: adding 'duplicate_of_id' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN duplicate_of_id INTEGER"))
|
||||
logger.info("Migration complete: 'duplicate_of_id' column added to files")
|
||||
|
||||
# Migration: Add search/OCR fields to files table
|
||||
if "ocr_text" not in columns:
|
||||
logger.info("Migrating files: adding 'ocr_text' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN ocr_text TEXT"))
|
||||
logger.info("Migration complete: 'ocr_text' column added to files")
|
||||
|
||||
if "ai_metadata" not in columns:
|
||||
logger.info("Migrating files: adding 'ai_metadata' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN ai_metadata TEXT"))
|
||||
logger.info("Migration complete: 'ai_metadata' column added to files")
|
||||
|
||||
if "document_title" not in columns:
|
||||
logger.info("Migrating files: adding 'document_title' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN document_title VARCHAR"))
|
||||
logger.info("Migration complete: 'document_title' column added to files")
|
||||
|
||||
if "ocr_quality_score" not in columns:
|
||||
logger.info("Migrating files: adding 'ocr_quality_score' column")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE files ADD COLUMN ocr_quality_score INTEGER"))
|
||||
logger.info("Migration complete: 'ocr_quality_score' column added to files")
|
||||
|
||||
# Migration: Drop unique index on filehash to allow duplicate records
|
||||
try:
|
||||
indexes = inspector.get_indexes("files")
|
||||
unique_filehash_indexes = [
|
||||
index for index in indexes if index.get("unique") and "filehash" in index.get("column_names", [])
|
||||
]
|
||||
if unique_filehash_indexes:
|
||||
logger.info("Migrating files: dropping unique index on 'filehash'")
|
||||
with engine.begin() as conn:
|
||||
for index in unique_filehash_indexes:
|
||||
conn.execute(text(f"DROP INDEX IF EXISTS {index['name']}"))
|
||||
logger.info("Migration complete: unique index on 'filehash' removed")
|
||||
except Exception as exc:
|
||||
logger.warning(f"Skipping filehash unique index drop: {exc}")
|
||||
|
||||
# Migration: Create saved_searches table for user-defined filter combinations
|
||||
if "saved_searches" not in inspector.get_table_names():
|
||||
logger.info("Migrating: creating 'saved_searches' table")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE saved_searches (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id VARCHAR NOT NULL,
|
||||
name VARCHAR NOT NULL,
|
||||
filters TEXT NOT NULL,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE (user_id, name)
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_saved_searches_user_id ON saved_searches (user_id)"))
|
||||
logger.info("Migration complete: 'saved_searches' table created")
|
||||
|
||||
# Migration: Add performance indexes for common query patterns
|
||||
_ensure_indexes(engine, inspector)
|
||||
|
||||
|
||||
def _ensure_indexes(engine: Any, inspector: Any) -> None:
|
||||
"""
|
||||
Create performance indexes for common query patterns.
|
||||
|
||||
Each ``CREATE INDEX IF NOT EXISTS`` is idempotent and safe to run
|
||||
on every startup. The indexes target the columns most frequently
|
||||
used in file listing/filtering, status computation and log retrieval.
|
||||
"""
|
||||
from sqlalchemy import text
|
||||
|
||||
_PERF_INDEXES = [
|
||||
("ix_files_created_at", "files", "created_at"),
|
||||
("ix_files_mime_type", "files", "mime_type"),
|
||||
("ix_processing_logs_file_id", "processing_logs", "file_id"),
|
||||
("ix_processing_logs_timestamp", "processing_logs", "timestamp"),
|
||||
("ix_file_processing_steps_status", "file_processing_steps", "status"),
|
||||
]
|
||||
|
||||
table_names = inspector.get_table_names()
|
||||
columns_by_table: dict[str, set[str]] = {}
|
||||
with engine.begin() as conn:
|
||||
for idx_name, table, column in _PERF_INDEXES:
|
||||
if table in table_names:
|
||||
if table not in columns_by_table:
|
||||
columns_by_table[table] = {col["name"] for col in inspector.get_columns(table)}
|
||||
if column in columns_by_table[table]:
|
||||
conn.execute(text(f"CREATE INDEX IF NOT EXISTS {idx_name} ON {table} ({column})"))
|
||||
|
||||
logger.info("Performance indexes ensured")
|
||||
|
||||
|
||||
def get_db() -> Generator[Session, None, None]:
|
||||
def get_db():
|
||||
"""
|
||||
Dependency for FastAPI routes or general DB usage.
|
||||
Yields a SQLAlchemy session, and closes it upon exit.
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sqlite3
|
||||
import os
|
||||
import logging
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def run_migrations():
|
||||
"""
|
||||
Run database migrations to add missing columns or make other schema changes.
|
||||
"""
|
||||
logger.info("Running database migrations...")
|
||||
|
||||
# Parse the DATABASE_URL to get the SQLite database path
|
||||
db_url = settings.database_url
|
||||
if not db_url.startswith("sqlite:///"):
|
||||
logger.warning(f"Non-SQLite database detected: {db_url}. Migrations may need to be adapted.")
|
||||
return
|
||||
|
||||
# Extract the database path from the URL
|
||||
db_path = db_url.replace("sqlite:///", "")
|
||||
if not os.path.exists(db_path):
|
||||
logger.error(f"Database file not found at {db_path}")
|
||||
return
|
||||
|
||||
# Connect to the database
|
||||
conn = None
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check if the processing_logs table exists
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='processing_logs';")
|
||||
if not cursor.fetchone():
|
||||
logger.info("Creating processing_logs table...")
|
||||
cursor.execute("""
|
||||
CREATE TABLE processing_logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
file_id INTEGER,
|
||||
step_name VARCHAR,
|
||||
status VARCHAR,
|
||||
message TEXT,
|
||||
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (file_id) REFERENCES files (id)
|
||||
);
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
# Check if the task_id column exists in processing_logs
|
||||
cursor.execute("PRAGMA table_info(processing_logs);")
|
||||
columns = [row[1] for row in cursor.fetchall()]
|
||||
|
||||
if 'task_id' not in columns:
|
||||
logger.info("Adding task_id column to processing_logs table...")
|
||||
cursor.execute("ALTER TABLE processing_logs ADD COLUMN task_id VARCHAR;")
|
||||
conn.commit()
|
||||
logger.info("Created task_id column in processing_logs")
|
||||
|
||||
# Create an index on task_id for faster lookups
|
||||
cursor.execute("CREATE INDEX idx_processing_logs_task_id ON processing_logs (task_id);")
|
||||
conn.commit()
|
||||
logger.info("Created index on task_id column")
|
||||
|
||||
logger.info("Database migrations completed successfully.")
|
||||
except Exception as e:
|
||||
logger.error(f"Error during database migration: {e}")
|
||||
if conn:
|
||||
conn.rollback()
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Configure logging
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
run_migrations()
|
||||
+50
-8
@@ -1,10 +1,52 @@
|
||||
"""
|
||||
Frontend routes for the application.
|
||||
This module is now a re-export of the modularized view routers.
|
||||
"""
|
||||
# app/frontend.py
|
||||
import os
|
||||
from pathlib import Path
|
||||
from fastapi import APIRouter, Request, Depends
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# Import and re-export the router from the views package
|
||||
from app.views import router # noqa: F401
|
||||
from app.auth import require_login
|
||||
from app.database import SessionLocal
|
||||
|
||||
# Keep the original router name for compatibility
|
||||
# This allows existing imports in main.py to continue working
|
||||
router = APIRouter()
|
||||
|
||||
templates_dir = Path(__file__).parent.parent / "frontend" / "templates"
|
||||
templates = Jinja2Templates(directory=str(templates_dir))
|
||||
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@router.get("/files")
|
||||
@require_login
|
||||
def files_page(request: Request):
|
||||
"""
|
||||
Return the 'files.html' template.
|
||||
The actual file data is fetched via XHR from /api/files in the template.
|
||||
"""
|
||||
return templates.TemplateResponse("files.html", {"request": request})
|
||||
|
||||
# ... existing routes for /, /upload, /about, etc. ...
|
||||
|
||||
|
||||
@router.get("/", include_in_schema=False)
|
||||
async def serve_index(request: Request):
|
||||
return templates.TemplateResponse("index.html", {"request": request})
|
||||
|
||||
@router.get("/about", include_in_schema=False)
|
||||
async def serve_about(request: Request):
|
||||
return templates.TemplateResponse("about.html", {"request": request})
|
||||
|
||||
@router.get("/upload", include_in_schema=False)
|
||||
@require_login
|
||||
async def serve_upload(request: Request):
|
||||
return templates.TemplateResponse("upload.html", {"request": request})
|
||||
|
||||
@router.get("/favicon.ico", include_in_schema=False)
|
||||
def favicon():
|
||||
# If you have a real favicon in `frontend/static/favicon.ico`:
|
||||
favicon_path = Path(__file__).parent.parent / "frontend" / "static" / "favicon.ico"
|
||||
return str(favicon_path)
|
||||
|
||||
+155
-189
@@ -1,236 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request, status
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi import FastAPI, HTTPException, UploadFile, File, status, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
from starlette.config import Config
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
from starlette.config import Config
|
||||
from starlette.middleware.trustedhost import TrustedHostMiddleware
|
||||
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
|
||||
from pathlib import Path
|
||||
|
||||
from app.database import init_db
|
||||
from app.db_migration import run_migrations
|
||||
from app.config import settings
|
||||
from app.tasks.process_document import process_document # Updated import
|
||||
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
||||
from app.tasks.upload_to_paperless import upload_to_paperless
|
||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||
from app.tasks.send_to_all import send_to_all_destinations
|
||||
|
||||
from app.api import router as api_router
|
||||
from app.frontend import router as frontend_router
|
||||
from app.auth import router as auth_router
|
||||
from app.config import settings
|
||||
from app.database import init_db
|
||||
from app.middleware.audit_log import AuditLogMiddleware
|
||||
from app.middleware.csrf import CSRFMiddleware
|
||||
from app.middleware.rate_limit import create_limiter, get_rate_limit_exceeded_handler
|
||||
from app.middleware.request_size_limit import RequestSizeLimitMiddleware
|
||||
from app.middleware.security_headers import SecurityHeadersMiddleware
|
||||
from app.utils.config_validator import check_all_configs
|
||||
from app.utils.notification import init_apprise, notify_shutdown, notify_startup
|
||||
|
||||
# Import the routers - now using views directly instead of frontend
|
||||
from app.views import router as frontend_router
|
||||
|
||||
# Explicitly include the files router
|
||||
from app.views.files import router as files_router
|
||||
|
||||
# Load configuration from .env for the session key
|
||||
config = Config(".env")
|
||||
# Use settings.session_secret which has proper validation
|
||||
# Fallback to raising an error if not set when auth is enabled
|
||||
if settings.auth_enabled and not settings.session_secret:
|
||||
raise ValueError(
|
||||
"SESSION_SECRET must be set when AUTH_ENABLED=True. "
|
||||
"Generate one with: python -c 'import secrets; print(secrets.token_hex(32))'"
|
||||
)
|
||||
SESSION_SECRET = (
|
||||
settings.session_secret or "INSECURE_DEFAULT_FOR_DEVELOPMENT_ONLY_DO_NOT_USE_IN_PRODUCTION_MINIMUM_32_CHARS"
|
||||
SESSION_SECRET = config(
|
||||
"SESSION_SECRET",
|
||||
default="YOUR_DEFAULT_SESSION_SECRET_MUST_BE_32_CHARS_OR_MORE"
|
||||
)
|
||||
|
||||
app = FastAPI(title="Document Processing API")
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""
|
||||
Manage application lifespan events (startup and shutdown).
|
||||
This replaces the deprecated @app.on_event decorators.
|
||||
"""
|
||||
# Startup: Initialize database
|
||||
init_db() # Create tables if they don't exist
|
||||
|
||||
# Load settings from database after DB initialization
|
||||
from app.database import SessionLocal
|
||||
from app.utils.config_loader import load_settings_from_db
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
load_settings_from_db(settings, db)
|
||||
logging.info("Database settings loaded successfully")
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to load database settings: {e}")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
# Ensure OCR language data is available (background download, non-blocking)
|
||||
from app.utils.ocr_language_manager import ensure_ocr_languages_async
|
||||
|
||||
ensure_ocr_languages_async()
|
||||
|
||||
# Force settings dump to log for troubleshooting
|
||||
from app.utils.config_validator import dump_all_settings
|
||||
|
||||
dump_all_settings()
|
||||
|
||||
# Validate configuration
|
||||
config_issues = check_all_configs()
|
||||
|
||||
# Log overall status
|
||||
has_issues = any(config_issues["email"]) or any(
|
||||
len(issues) > 0 for provider, issues in config_issues["storage"].items()
|
||||
)
|
||||
if has_issues:
|
||||
logging.warning("Application started with configuration issues - some features may be unavailable")
|
||||
else:
|
||||
logging.info("Application started with valid configuration")
|
||||
|
||||
logging.info("Router organization: Using refactored API routers from app/api/ directory")
|
||||
|
||||
# Initialize notification system
|
||||
init_apprise()
|
||||
|
||||
# Send startup notification
|
||||
notify_startup()
|
||||
|
||||
# Application is now running
|
||||
yield
|
||||
|
||||
# Shutdown: Cleanup tasks
|
||||
logging.info("Application shutting down")
|
||||
|
||||
# Send shutdown notification
|
||||
notify_shutdown()
|
||||
|
||||
|
||||
app = FastAPI(title="DocuElevate", lifespan=lifespan)
|
||||
|
||||
# Initialize rate limiter and attach to app state
|
||||
limiter = create_limiter(redis_url=settings.redis_url, enabled=settings.rate_limiting_enabled)
|
||||
app.state.limiter = limiter
|
||||
app.add_exception_handler(RateLimitExceeded, get_rate_limit_exceeded_handler())
|
||||
|
||||
# Middleware stack (order matters - applied in reverse order)
|
||||
# Last added middleware is executed first
|
||||
|
||||
# 1) Security Headers Middleware (outermost - adds headers to final response)
|
||||
# Configure via SECURITY_HEADERS_ENABLED environment variable
|
||||
# Set to False if reverse proxy (Traefik, Nginx) handles security headers
|
||||
app.add_middleware(SecurityHeadersMiddleware, config=settings)
|
||||
|
||||
# 2) Request Size Limit Middleware - enforces body size limits before reading
|
||||
# MAX_REQUEST_BODY_SIZE: limit for non-file requests (default 1 MB)
|
||||
# MAX_UPLOAD_SIZE: limit for multipart/form-data uploads (default 1 GB)
|
||||
# See SECURITY_AUDIT.md – Code Security section
|
||||
app.add_middleware(RequestSizeLimitMiddleware, config=settings)
|
||||
|
||||
# 3) CSRF Protection Middleware - validates CSRF tokens for state-changing operations
|
||||
# Only active when AUTH_ENABLED=True. Exempts OAuth callback endpoints.
|
||||
# Tokens are stored in the session and validated via X-CSRF-Token header or form field.
|
||||
app.add_middleware(CSRFMiddleware, config=settings)
|
||||
|
||||
# 2) Audit Logging Middleware - logs all requests with sensitive data masking
|
||||
# Configure via AUDIT_LOGGING_ENABLED environment variable
|
||||
# See SECURITY_AUDIT.md – Infrastructure Security section
|
||||
app.add_middleware(AuditLogMiddleware, config=settings)
|
||||
|
||||
# 3) Session Middleware (for request.session to work)
|
||||
# 1) Session Middleware (for request.session to work)
|
||||
app.add_middleware(SessionMiddleware, secret_key=SESSION_SECRET)
|
||||
|
||||
# 3a) CORS Middleware - handles cross-origin requests and preflight (OPTIONS) responses.
|
||||
# Disabled by default: set CORS_ENABLED=True only when NOT using a reverse proxy
|
||||
# (Traefik, Nginx) that already injects CORS headers. When enabled, this middleware
|
||||
# runs after the session layer so preflight requests bypass CSRF/auth checks.
|
||||
# Allowed origins, methods, headers, and credentials are all configurable via env vars.
|
||||
# See SECURITY_AUDIT.md – Infrastructure Security section and docs/DeploymentGuide.md.
|
||||
if settings.cors_enabled:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_allowed_origins,
|
||||
allow_credentials=settings.cors_allow_credentials,
|
||||
allow_methods=settings.cors_allowed_methods,
|
||||
allow_headers=settings.cors_allowed_headers,
|
||||
)
|
||||
|
||||
# 4) Respect the X-Forwarded-* headers from reverse proxy (Traefik, Nginx)
|
||||
# 2) Respect the X-Forwarded-* headers from Traefik
|
||||
app.add_middleware(ProxyHeadersMiddleware, trusted_hosts="*")
|
||||
|
||||
# 5) Restrict valid hosts to prevent Host header attacks
|
||||
app.add_middleware(
|
||||
TrustedHostMiddleware,
|
||||
allowed_hosts=[settings.external_hostname, "localhost", "127.0.0.1"],
|
||||
)
|
||||
# 3) (Optional but recommended) Restrict valid hosts:
|
||||
app.add_middleware(TrustedHostMiddleware, allowed_hosts=[
|
||||
"docparse.hosterra.net",
|
||||
"localhost",
|
||||
"127.0.0.1"
|
||||
])
|
||||
|
||||
# Mount the static files directory
|
||||
static_dir = pathlib.Path(__file__).parents[1] / "frontend" / "static"
|
||||
if os.path.exists(static_dir):
|
||||
app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
|
||||
else:
|
||||
print(f"WARNING: Static directory not found at {static_dir}. Static files will not be served.")
|
||||
# Mount the static folder for CSS/JS:
|
||||
frontend_static_dir = Path(__file__).parent.parent / "frontend" / "static"
|
||||
app.mount("/static", StaticFiles(directory=frontend_static_dir), name="static")
|
||||
|
||||
@app.on_event("startup")
|
||||
def on_startup():
|
||||
init_db() # Create tables if they don't exist
|
||||
run_migrations() # Run migrations to add any missing columns
|
||||
|
||||
# Custom exception handlers that return JSON for API routes and HTML for frontend routes
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_exception_handler(request: Request, exc: HTTPException):
|
||||
@app.post("/process/")
|
||||
def process(file_path: str):
|
||||
"""
|
||||
Handle all HTTPException instances.
|
||||
Returns JSON for API routes, HTML templates for frontend routes.
|
||||
API Endpoint to start document processing.
|
||||
This enqueues document processing which handles the full pipeline.
|
||||
"""
|
||||
# For API routes, always return JSON
|
||||
if request.url.path.startswith("/api/"):
|
||||
return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})
|
||||
if not os.path.isabs(file_path):
|
||||
file_path = os.path.join(settings.workdir, file_path)
|
||||
|
||||
# For frontend routes, return appropriate HTML templates
|
||||
templates = Jinja2Templates(directory=str(static_dir.parent / "templates"))
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"File {file_path} not found."
|
||||
)
|
||||
|
||||
# Handle 404 errors with a custom template
|
||||
if exc.status_code == 404:
|
||||
return templates.TemplateResponse("404.html", {"request": request}, status_code=status.HTTP_404_NOT_FOUND)
|
||||
task = process_document.delay(file_path) # Updated function call
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
# For other HTTP errors, we could create specific templates or use a generic one
|
||||
# For now, return a simple error page
|
||||
@app.post("/send_to_dropbox/")
|
||||
def send_to_dropbox(file_path: str):
|
||||
if not os.path.isabs(file_path):
|
||||
file_path = os.path.join(settings.workdir, 'processed', file_path)
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"File {file_path} not found."
|
||||
)
|
||||
task = upload_to_dropbox.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
@app.post("/send_to_paperless/")
|
||||
def send_to_paperless(file_path: str):
|
||||
if not os.path.isabs(file_path):
|
||||
file_path = os.path.join(settings.workdir, 'processed', file_path)
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"File {file_path} not found."
|
||||
)
|
||||
task = upload_to_paperless.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
@app.post("/send_to_nextcloud/")
|
||||
def send_to_nextcloud(file_path: str):
|
||||
if not os.path.isabs(file_path):
|
||||
file_path = os.path.join(settings.workdir, 'processed', file_path)
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"File {file_path} not found."
|
||||
)
|
||||
task = upload_to_nextcloud.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
@app.post("/send_to_all_destinations/")
|
||||
def send_to_all_destinations_endpoint(file_path: str):
|
||||
"""
|
||||
Call the aggregator task that sends this file to dropbox, nextcloud, and paperless.
|
||||
"""
|
||||
if not os.path.isabs(file_path):
|
||||
file_path = os.path.join(settings.workdir, 'processed', file_path)
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"File {file_path} not found."
|
||||
)
|
||||
|
||||
task = send_to_all_destinations.delay(file_path)
|
||||
return {"task_id": task.id, "status": "queued", "file_path": file_path}
|
||||
|
||||
@app.post("/processall")
|
||||
def process_all_pdfs_in_workdir():
|
||||
"""
|
||||
Finds all .pdf files in <workdir> and enqueues them for processing.
|
||||
"""
|
||||
target_dir = settings.workdir
|
||||
if not os.path.exists(target_dir):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Directory {target_dir} does not exist."
|
||||
)
|
||||
|
||||
pdf_files = []
|
||||
for filename in os.listdir(target_dir):
|
||||
if filename.lower().endswith(".pdf"):
|
||||
pdf_files.append(filename)
|
||||
|
||||
if not pdf_files:
|
||||
return {"message": "No PDF files found in that directory."}
|
||||
|
||||
task_ids = []
|
||||
for pdf in pdf_files:
|
||||
file_path = os.path.join(target_dir, pdf)
|
||||
task = process_document.delay(file_path) # Updated function call
|
||||
task_ids.append(task.id)
|
||||
|
||||
return {
|
||||
"message": f"Enqueued {len(pdf_files)} PDFs to upload_to_s3",
|
||||
"pdf_files": pdf_files,
|
||||
"task_ids": task_ids
|
||||
}
|
||||
|
||||
@app.post("/ui-upload")
|
||||
async def ui_upload(file: UploadFile = File(...)):
|
||||
"""Endpoint to accept a user-uploaded file and enqueue it for processing."""
|
||||
workdir = "/workdir"
|
||||
target_path = os.path.join(workdir, file.filename)
|
||||
try:
|
||||
with open(target_path, "wb") as f:
|
||||
content = await file.read()
|
||||
f.write(content)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Failed to save file: {e}"
|
||||
)
|
||||
|
||||
task = process_document.delay(target_path) # Updated function call
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
# Custom 404 - we can still return the Jinja2 template, or the old static file:
|
||||
# For a dynamic 404 using the base layout, see "frontend/404.html" usage below:
|
||||
@app.exception_handler(404)
|
||||
async def custom_404_handler(request: Request, exc: HTTPException):
|
||||
# Serve the 404 template directly
|
||||
templates = Jinja2Templates(directory=str(frontend_static_dir.parent / "templates"))
|
||||
return templates.TemplateResponse(
|
||||
"404.html", # Reuse 404 template for other errors, or create a generic error template
|
||||
"404.html",
|
||||
{"request": request},
|
||||
status_code=exc.status_code,
|
||||
status_code=status.HTTP_404_NOT_FOUND
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(500)
|
||||
async def custom_500_handler(request: Request, exc: Exception):
|
||||
"""
|
||||
Handle internal server errors (500).
|
||||
Returns JSON for API routes, HTML templates for frontend routes.
|
||||
"""
|
||||
# For API routes, return JSON instead of HTML
|
||||
if request.url.path.startswith("/api/"):
|
||||
return JSONResponse(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
content={"detail": "Internal server error"},
|
||||
)
|
||||
|
||||
# Serve the 500 template for non-API routes
|
||||
templates = Jinja2Templates(directory=str(static_dir.parent / "templates"))
|
||||
templates = Jinja2Templates(directory=str(frontend_static_dir.parent / "templates"))
|
||||
# Option 1: Keep it simple, just show a funny 500 message:
|
||||
return templates.TemplateResponse(
|
||||
"500.html",
|
||||
{"request": request, "exc": exc},
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
)
|
||||
|
||||
|
||||
@app.get("/test-500")
|
||||
def test_500():
|
||||
raise RuntimeError("Testing forced 500 error!")
|
||||
|
||||
|
||||
# Include the routers
|
||||
# Include the frontend and auth routers
|
||||
app.include_router(frontend_router)
|
||||
app.include_router(files_router) # Explicitly include the files router
|
||||
app.include_router(auth_router)
|
||||
app.include_router(api_router, prefix="/api")
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""Middleware package for DocuElevate."""
|
||||
|
||||
from app.middleware.security_headers import SecurityHeadersMiddleware
|
||||
|
||||
__all__ = ["SecurityHeadersMiddleware"]
|
||||
@@ -1,250 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Audit Logging Middleware for DocuElevate.
|
||||
|
||||
This middleware logs all HTTP requests and security-relevant events. Sensitive
|
||||
data (passwords, tokens, secrets, API keys) is masked before logging so that
|
||||
credentials are never recorded in application logs.
|
||||
|
||||
Security-relevant events that receive elevated ``[SECURITY]`` log entries:
|
||||
- Authentication failures (401 Unauthorized)
|
||||
- Authorisation denials (403 Forbidden)
|
||||
- Login / logout endpoint access
|
||||
- Server errors (5xx responses)
|
||||
|
||||
Logged per request:
|
||||
- HTTP method
|
||||
- Request path (query-param values for known sensitive keys are replaced with ``[REDACTED]``)
|
||||
- Response status code
|
||||
- Response time in milliseconds
|
||||
- Client IP address (configurable)
|
||||
- Authenticated username when available
|
||||
|
||||
See SECURITY_AUDIT.md – Infrastructure Security section for background.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Callable
|
||||
|
||||
from fastapi import Request, Response
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Query-parameter / form-field names whose *values* must never appear in logs.
|
||||
# Matching is case-insensitive.
|
||||
_SENSITIVE_PARAM_PATTERN = re.compile(
|
||||
r"^(password|passwd|pwd|secret|token|access_token|refresh_token|"
|
||||
r"api_key|apikey|key|credential|credentials|auth|authorization|"
|
||||
r"client_secret|private_key|session)$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# HTTP headers whose values must never appear in logs.
|
||||
_SENSITIVE_HEADERS = frozenset(
|
||||
{
|
||||
"authorization",
|
||||
"cookie",
|
||||
"set-cookie",
|
||||
"x-api-key",
|
||||
"x-auth-token",
|
||||
}
|
||||
)
|
||||
|
||||
# Endpoints considered security-sensitive for elevated logging.
|
||||
_AUTH_PATHS = frozenset({"/auth", "/login", "/logout", "/oauth-login", "/oauth-callback"})
|
||||
|
||||
|
||||
def mask_query_string(query_string: str) -> str:
|
||||
"""
|
||||
Replace values of sensitive query parameters with ``[REDACTED]``.
|
||||
|
||||
Args:
|
||||
query_string: Raw URL query string (e.g. ``"user=alice&password=secret"``).
|
||||
|
||||
Returns:
|
||||
Query string with sensitive values replaced.
|
||||
"""
|
||||
if not query_string:
|
||||
return query_string
|
||||
|
||||
parts = []
|
||||
for pair in query_string.split("&"):
|
||||
if "=" in pair:
|
||||
name, _, value = pair.partition("=")
|
||||
if _SENSITIVE_PARAM_PATTERN.match(name):
|
||||
parts.append(f"{name}=[REDACTED]")
|
||||
else:
|
||||
parts.append(pair)
|
||||
else:
|
||||
parts.append(pair)
|
||||
return "&".join(parts)
|
||||
|
||||
|
||||
def get_client_ip(request: Request) -> str:
|
||||
"""
|
||||
Extract the real client IP, honouring X-Forwarded-For when present.
|
||||
|
||||
Args:
|
||||
request: Incoming HTTP request.
|
||||
|
||||
Returns:
|
||||
Client IP address string.
|
||||
"""
|
||||
forwarded_for = request.headers.get("x-forwarded-for")
|
||||
if forwarded_for:
|
||||
# Take only the first (leftmost) address – that is the original client.
|
||||
return forwarded_for.split(",")[0].strip()
|
||||
if request.client:
|
||||
return request.client.host
|
||||
return "unknown"
|
||||
|
||||
|
||||
def get_username(request: Request) -> str:
|
||||
"""
|
||||
Extract the authenticated username from the session, if available.
|
||||
|
||||
Args:
|
||||
request: Incoming HTTP request.
|
||||
|
||||
Returns:
|
||||
Username string, or ``"anonymous"`` when not authenticated.
|
||||
"""
|
||||
try:
|
||||
user = request.session.get("user") if hasattr(request, "session") else None
|
||||
except Exception:
|
||||
user = None
|
||||
|
||||
if not user:
|
||||
return "anonymous"
|
||||
|
||||
if isinstance(user, dict):
|
||||
return (
|
||||
user.get("preferred_username")
|
||||
or user.get("username")
|
||||
or user.get("email")
|
||||
or user.get("id")
|
||||
or "authenticated"
|
||||
)
|
||||
return str(user)
|
||||
|
||||
|
||||
class AuditLogMiddleware(BaseHTTPMiddleware):
|
||||
"""
|
||||
Middleware to log HTTP requests and security-relevant events.
|
||||
|
||||
Each request produces a single ``INFO``-level audit log line.
|
||||
Requests that result in 401/403 responses, or that target
|
||||
authentication endpoints, additionally produce a ``WARNING``-level
|
||||
security event line. Server errors (5xx) produce an ``ERROR``-level
|
||||
security event line.
|
||||
|
||||
Configuration is read from the application settings object passed at
|
||||
construction time via the ``config`` keyword argument.
|
||||
"""
|
||||
|
||||
def __init__(self, app, config) -> None:
|
||||
"""
|
||||
Initialise the audit-log middleware.
|
||||
|
||||
Args:
|
||||
app: FastAPI / ASGI application instance.
|
||||
config: Application settings object (must expose
|
||||
``audit_logging_enabled`` and
|
||||
``audit_log_include_client_ip`` boolean attributes).
|
||||
"""
|
||||
super().__init__(app)
|
||||
self.enabled = config.audit_logging_enabled
|
||||
self.include_ip = config.audit_log_include_client_ip
|
||||
|
||||
if self.enabled:
|
||||
logger.info(f"Audit logging middleware enabled (include_client_ip={self.include_ip})")
|
||||
else:
|
||||
logger.info("Audit logging middleware disabled")
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
"""
|
||||
Process the request, call the next handler, then emit audit log entries.
|
||||
|
||||
Args:
|
||||
request: Incoming HTTP request.
|
||||
call_next: Next middleware or route handler in the chain.
|
||||
|
||||
Returns:
|
||||
HTTP response (unmodified).
|
||||
"""
|
||||
if not self.enabled:
|
||||
return await call_next(request)
|
||||
|
||||
start_time = time.monotonic()
|
||||
response = await call_next(request)
|
||||
duration_ms = int((time.monotonic() - start_time) * 1000)
|
||||
|
||||
self._log_request(request, response.status_code, duration_ms)
|
||||
return response
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _build_path_with_masked_query(self, request: Request) -> str:
|
||||
"""Return the request path with sensitive query-param values masked."""
|
||||
path = request.url.path
|
||||
raw_query = request.url.query
|
||||
if raw_query:
|
||||
masked = mask_query_string(raw_query)
|
||||
return f"{path}?{masked}"
|
||||
return path
|
||||
|
||||
def _log_request(self, request: Request, status_code: int, duration_ms: int) -> None:
|
||||
"""
|
||||
Emit audit log entries for a completed request.
|
||||
|
||||
Args:
|
||||
request: The HTTP request object.
|
||||
status_code: HTTP response status code.
|
||||
duration_ms: Total request processing time in milliseconds.
|
||||
"""
|
||||
method = request.method
|
||||
path = self._build_path_with_masked_query(request)
|
||||
username = get_username(request)
|
||||
ip_part = f" - {get_client_ip(request)}" if self.include_ip else ""
|
||||
|
||||
# Core request log line (always INFO).
|
||||
logger.info(f"[AUDIT] {method} {path} {status_code} {duration_ms}ms{ip_part} - {username}")
|
||||
|
||||
# Security-event log lines for noteworthy conditions.
|
||||
self._log_security_event(method, path, status_code, username, ip_part)
|
||||
|
||||
def _log_security_event(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
status_code: int,
|
||||
username: str,
|
||||
ip_part: str,
|
||||
) -> None:
|
||||
"""
|
||||
Emit an additional security-event log line when warranted.
|
||||
|
||||
Args:
|
||||
method: HTTP method (GET, POST, …).
|
||||
path: Sanitised request path (with masked query params).
|
||||
status_code: HTTP response status code.
|
||||
username: Authenticated username or ``"anonymous"``.
|
||||
ip_part: Pre-formatted IP string (may be empty string).
|
||||
"""
|
||||
base_path = path.split("?", maxsplit=1)[0]
|
||||
|
||||
if status_code == 401:
|
||||
logger.warning(f"[SECURITY] AUTH_FAILURE {method} {path} 401{ip_part} - {username}")
|
||||
elif status_code == 403:
|
||||
logger.warning(f"[SECURITY] ACCESS_DENIED {method} {path} 403{ip_part} - {username}")
|
||||
elif base_path in _AUTH_PATHS and method == "POST":
|
||||
# Login attempts (successful or not) are always noted.
|
||||
logger.info(f"[SECURITY] AUTH_ATTEMPT {method} {path} {status_code}{ip_part} - {username}")
|
||||
elif status_code >= 500:
|
||||
logger.error(f"[SECURITY] SERVER_ERROR {method} {path} {status_code}{ip_part} - {username}")
|
||||
@@ -1,160 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
CSRF Protection Middleware for DocuElevate.
|
||||
|
||||
This middleware implements Cross-Site Request Forgery (CSRF) protection for all
|
||||
state-changing HTTP operations (POST, PUT, DELETE, PATCH).
|
||||
|
||||
How it works:
|
||||
- A cryptographically secure token is generated per session and stored in the session.
|
||||
- The token is attached to ``request.state.csrf_token`` so Jinja2 templates can render it.
|
||||
- For every state-changing request the middleware validates the submitted token by
|
||||
checking (in order):
|
||||
1. The ``X-CSRF-Token`` HTTP request header (used by AJAX / fetch calls).
|
||||
2. The ``csrf_token`` field in ``application/x-www-form-urlencoded`` bodies
|
||||
(used by traditional HTML forms such as the login form).
|
||||
Multipart file-upload requests must always supply the token via the header.
|
||||
- Validation is only enforced when ``AUTH_ENABLED=True``. When authentication is
|
||||
disabled (development / single-user mode) the middleware is a no-op.
|
||||
|
||||
Exempt paths (CSRF is not checked even for state-changing methods):
|
||||
- ``/oauth-callback`` – OAuth 2.0 callback; protected by the ``state`` parameter.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
from typing import Callable
|
||||
|
||||
from fastapi import Request, Response
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.responses import JSONResponse, RedirectResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# HTTP methods that change server state and therefore require a valid CSRF token.
|
||||
CSRF_PROTECTED_METHODS = {"POST", "PUT", "DELETE", "PATCH"}
|
||||
|
||||
# Paths that must never be CSRF-checked (e.g. OAuth flow endpoints that carry
|
||||
# their own replay-protection mechanism).
|
||||
CSRF_EXEMPT_PATHS = {
|
||||
"/oauth-callback",
|
||||
}
|
||||
|
||||
|
||||
class CSRFMiddleware(BaseHTTPMiddleware):
|
||||
"""
|
||||
Middleware that generates and validates CSRF tokens for state-changing requests.
|
||||
|
||||
Token lifecycle
|
||||
---------------
|
||||
1. On the first request for a session a 64-character hex token is created with
|
||||
``secrets.token_hex(32)`` and stored in ``request.session["csrf_token"]``.
|
||||
2. On every subsequent request the existing token is read from the session.
|
||||
3. The token is always attached to ``request.state.csrf_token`` so that
|
||||
Jinja2 templates (and response processors) can embed it.
|
||||
|
||||
Validation
|
||||
----------
|
||||
For ``POST``, ``PUT``, ``DELETE``, and ``PATCH`` requests the middleware
|
||||
checks whether the submitted token matches the session token using a
|
||||
constant-time comparison (``secrets.compare_digest``) to prevent timing
|
||||
attacks.
|
||||
|
||||
Failure response
|
||||
----------------
|
||||
- API routes (``/api/*``): HTTP 403 JSON response.
|
||||
- All other routes: HTTP 302 redirect to ``/login?error=…``.
|
||||
"""
|
||||
|
||||
def __init__(self, app, config):
|
||||
"""
|
||||
Initialise the middleware.
|
||||
|
||||
Args:
|
||||
app: The ASGI application.
|
||||
config: Application settings object (``app.config.Settings``).
|
||||
``config.auth_enabled`` controls whether CSRF enforcement is active.
|
||||
"""
|
||||
super().__init__(app)
|
||||
self.config = config
|
||||
self.enabled = config.auth_enabled
|
||||
if self.enabled:
|
||||
logger.info("CSRF protection middleware enabled")
|
||||
else:
|
||||
logger.info("CSRF protection middleware disabled (AUTH_ENABLED=False)")
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
"""
|
||||
Process the request: generate/attach the token and validate it when required.
|
||||
|
||||
Args:
|
||||
request: Incoming HTTP request.
|
||||
call_next: Next middleware or route handler in the ASGI chain.
|
||||
|
||||
Returns:
|
||||
HTTP response, or an error response when CSRF validation fails.
|
||||
"""
|
||||
if not self.enabled:
|
||||
return await call_next(request)
|
||||
|
||||
# Generate or retrieve the per-session CSRF token.
|
||||
csrf_token = request.session.get("csrf_token")
|
||||
if not csrf_token:
|
||||
csrf_token = secrets.token_hex(32)
|
||||
request.session["csrf_token"] = csrf_token
|
||||
|
||||
# Attach token to request state so templates and route handlers can access it.
|
||||
request.state.csrf_token = csrf_token
|
||||
|
||||
# Validate for state-changing methods on non-exempt paths.
|
||||
if request.method in CSRF_PROTECTED_METHODS and request.url.path not in CSRF_EXEMPT_PATHS:
|
||||
submitted_token = await self._get_submitted_token(request)
|
||||
if not submitted_token or not secrets.compare_digest(csrf_token, submitted_token):
|
||||
logger.warning(f"[SECURITY] CSRF_VALIDATION_FAILED method={request.method} path={request.url.path}")
|
||||
if request.url.path.startswith("/api/"):
|
||||
return JSONResponse(
|
||||
status_code=403,
|
||||
content={"detail": "CSRF token missing or invalid"},
|
||||
)
|
||||
return RedirectResponse(url="/login?error=Invalid+request", status_code=302)
|
||||
|
||||
return await call_next(request)
|
||||
|
||||
@staticmethod
|
||||
async def _get_submitted_token(request: Request) -> str | None:
|
||||
"""
|
||||
Extract the CSRF token submitted by the client.
|
||||
|
||||
Checks (in priority order):
|
||||
1. ``X-CSRF-Token`` request header – preferred for AJAX / fetch requests.
|
||||
2. ``csrf_token`` form field in ``application/x-www-form-urlencoded`` bodies –
|
||||
used by plain HTML forms (e.g. the login form).
|
||||
|
||||
Multipart bodies (file uploads) are intentionally not parsed here to avoid
|
||||
buffering large uploads in middleware; those endpoints must send the token
|
||||
via the header instead.
|
||||
|
||||
Args:
|
||||
request: The incoming HTTP request.
|
||||
|
||||
Returns:
|
||||
The submitted CSRF token string, or ``None`` if not found.
|
||||
"""
|
||||
# 1. Check the request header (AJAX / fetch).
|
||||
token = request.headers.get("X-CSRF-Token")
|
||||
if token:
|
||||
return token
|
||||
|
||||
# 2. For URL-encoded form bodies only (plain HTML form submissions).
|
||||
content_type = request.headers.get("content-type", "")
|
||||
if "application/x-www-form-urlencoded" in content_type:
|
||||
try:
|
||||
form = await request.form()
|
||||
token = form.get("csrf_token")
|
||||
if token:
|
||||
return str(token)
|
||||
except Exception as exc:
|
||||
logger.debug(f"CSRF: could not parse form body: {exc}")
|
||||
|
||||
return None
|
||||
@@ -1,114 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Rate Limiting Middleware for DocuElevate.
|
||||
|
||||
This middleware provides rate limiting capabilities to protect API endpoints from abuse
|
||||
and DoS attacks. It uses SlowAPI with Redis backend for distributed rate limiting.
|
||||
|
||||
Key features:
|
||||
- Per-IP rate limiting by default
|
||||
- Per-user rate limiting for authenticated endpoints
|
||||
- Configurable global and per-endpoint limits
|
||||
- Redis-backed for distributed deployments
|
||||
- Fallback to in-memory for development
|
||||
|
||||
See docs/ConfigurationGuide.md and docs/API.md for configuration and usage.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Callable
|
||||
|
||||
from fastapi import Request
|
||||
from slowapi import Limiter, _rate_limit_exceeded_handler
|
||||
from slowapi.util import get_remote_address
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_identifier(request: Request) -> str:
|
||||
"""
|
||||
Get unique identifier for rate limiting.
|
||||
|
||||
Uses authenticated user ID if available, otherwise falls back to IP address.
|
||||
This provides better rate limiting for authenticated users and prevents
|
||||
IP-based bypassing for authenticated endpoints.
|
||||
|
||||
Args:
|
||||
request: FastAPI request object
|
||||
|
||||
Returns:
|
||||
Unique identifier string for rate limiting
|
||||
"""
|
||||
# Check if user is authenticated (from session)
|
||||
if hasattr(request, "session") and request.session.get("user"):
|
||||
user = request.session.get("user")
|
||||
# Use username or user_id as identifier
|
||||
if isinstance(user, dict):
|
||||
identifier = user.get("username") or user.get("user_id") or user.get("id")
|
||||
if identifier:
|
||||
logger.debug(f"Rate limiting by user: {identifier}")
|
||||
return f"user:{identifier}"
|
||||
|
||||
# Fall back to IP address for unauthenticated requests
|
||||
ip = get_remote_address(request)
|
||||
logger.debug(f"Rate limiting by IP: {ip}")
|
||||
return ip
|
||||
|
||||
|
||||
def create_limiter(redis_url: str = None, enabled: bool = True) -> Limiter:
|
||||
"""
|
||||
Create and configure the rate limiter.
|
||||
|
||||
Args:
|
||||
redis_url: Redis connection URL for distributed rate limiting
|
||||
enabled: Whether rate limiting is enabled (default: True)
|
||||
|
||||
Returns:
|
||||
Configured Limiter instance
|
||||
"""
|
||||
if not enabled:
|
||||
logger.info("Rate limiting is disabled")
|
||||
# Return a limiter with very high limits when disabled
|
||||
return Limiter(
|
||||
key_func=get_identifier,
|
||||
default_limits=["10000/minute"], # Effectively unlimited
|
||||
enabled=False,
|
||||
)
|
||||
|
||||
# Use Redis if available, otherwise fall back to in-memory
|
||||
storage_uri = redis_url if redis_url else "memory://"
|
||||
|
||||
if redis_url:
|
||||
logger.info(f"Rate limiting enabled with Redis backend: {redis_url}")
|
||||
else:
|
||||
logger.warning(
|
||||
"Rate limiting using in-memory storage (not suitable for production with multiple workers). "
|
||||
"Configure REDIS_URL for distributed rate limiting."
|
||||
)
|
||||
|
||||
# Create limiter with default limits
|
||||
# Default: 100 requests per minute per IP/user
|
||||
limiter = Limiter(
|
||||
key_func=get_identifier,
|
||||
default_limits=["100/minute"],
|
||||
storage_uri=storage_uri,
|
||||
strategy="fixed-window", # Can be: fixed-window, moving-window, or fixed-window-elastic-expiry
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
logger.info("Rate limiter initialized successfully")
|
||||
return limiter
|
||||
|
||||
|
||||
def get_rate_limit_exceeded_handler() -> Callable:
|
||||
"""
|
||||
Get the rate limit exceeded exception handler.
|
||||
|
||||
Returns a handler that provides user-friendly 429 responses with
|
||||
Retry-After header when rate limit is exceeded.
|
||||
|
||||
Returns:
|
||||
Exception handler function
|
||||
"""
|
||||
return _rate_limit_exceeded_handler
|
||||
@@ -1,68 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Rate limiting decorators for DocuElevate API endpoints.
|
||||
|
||||
This module provides convenient decorators to apply rate limits to specific endpoints.
|
||||
Import the limiter from main.py state and use these decorators to protect endpoints.
|
||||
"""
|
||||
|
||||
# Import will happen at runtime to avoid circular dependencies
|
||||
_limiter = None
|
||||
|
||||
|
||||
def get_limiter():
|
||||
"""Get the limiter instance from the app state."""
|
||||
global _limiter
|
||||
if _limiter is None:
|
||||
from app.main import app
|
||||
|
||||
_limiter = app.state.limiter
|
||||
return _limiter
|
||||
|
||||
|
||||
def limit(rate_limit: str):
|
||||
"""
|
||||
Apply a rate limit to an endpoint.
|
||||
|
||||
Args:
|
||||
rate_limit: Rate limit string (e.g., "10/minute", "100/hour")
|
||||
|
||||
Returns:
|
||||
Decorator function
|
||||
|
||||
Example:
|
||||
@router.post("/login")
|
||||
@limit("10/minute")
|
||||
async def login(request: Request):
|
||||
...
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
limiter = get_limiter()
|
||||
# Apply the slowapi limit decorator
|
||||
return limiter.limit(rate_limit)(func)
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def exempt():
|
||||
"""
|
||||
Exempt an endpoint from rate limiting.
|
||||
|
||||
Returns:
|
||||
Decorator function
|
||||
|
||||
Example:
|
||||
@router.get("/health")
|
||||
@exempt()
|
||||
def health_check():
|
||||
...
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
limiter = get_limiter()
|
||||
# Apply the slowapi exempt decorator
|
||||
return limiter.exempt(func)
|
||||
|
||||
return decorator
|
||||
@@ -1,114 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Request Size Limit Middleware for DocuElevate.
|
||||
|
||||
This middleware enforces configurable size limits on incoming HTTP request bodies
|
||||
to prevent memory exhaustion and Denial-of-Service (DoS) attacks.
|
||||
|
||||
Two independent limits are enforced:
|
||||
- ``MAX_REQUEST_BODY_SIZE``: applied to all non-multipart requests (JSON, form data, etc.).
|
||||
Default: 1 MB. Configurable via the ``MAX_REQUEST_BODY_SIZE`` environment variable.
|
||||
- ``MAX_UPLOAD_SIZE``: applied to multipart/form-data (file upload) requests.
|
||||
Default: 1 GB. Configurable via the ``MAX_UPLOAD_SIZE`` environment variable.
|
||||
|
||||
When a request exceeds the applicable limit the middleware immediately returns
|
||||
``HTTP 413 Request Entity Too Large`` without reading the full body, which keeps
|
||||
memory usage bounded.
|
||||
|
||||
See SECURITY_AUDIT.md – Code Security section for background.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RequestSizeLimitMiddleware(BaseHTTPMiddleware):
|
||||
"""
|
||||
Middleware that rejects requests whose body exceeds a configured size limit.
|
||||
|
||||
File-upload requests (``Content-Type: multipart/form-data``) are checked
|
||||
against ``config.max_upload_size``; all other requests are checked against
|
||||
``config.max_request_body_size``.
|
||||
|
||||
The check is performed on the ``Content-Length`` header before the body is
|
||||
read, so oversized requests are rejected without buffering the payload into
|
||||
memory. If the client omits the ``Content-Length`` header the request is
|
||||
passed through to the normal handler (where endpoint-level checks still
|
||||
apply for file uploads).
|
||||
"""
|
||||
|
||||
def __init__(self, app, config):
|
||||
"""
|
||||
Initialize the middleware.
|
||||
|
||||
Args:
|
||||
app: The ASGI application to wrap.
|
||||
config: Application settings object with ``max_request_body_size``
|
||||
and ``max_upload_size`` attributes.
|
||||
"""
|
||||
super().__init__(app)
|
||||
self.max_body_size = config.max_request_body_size
|
||||
self.max_upload_size = config.max_upload_size
|
||||
logger.info(
|
||||
f"Request size limit middleware enabled – "
|
||||
f"body limit: {self.max_body_size} bytes, "
|
||||
f"upload limit: {self.max_upload_size} bytes"
|
||||
)
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
"""
|
||||
Check the ``Content-Length`` header and reject oversized requests early.
|
||||
|
||||
Args:
|
||||
request: Incoming HTTP request.
|
||||
call_next: Next middleware or route handler.
|
||||
|
||||
Returns:
|
||||
HTTP 413 response if the request is too large, otherwise the
|
||||
downstream response.
|
||||
"""
|
||||
content_length_header = request.headers.get("content-length")
|
||||
if content_length_header is not None:
|
||||
try:
|
||||
content_length = int(content_length_header)
|
||||
except ValueError:
|
||||
# Malformed header – let downstream handle it
|
||||
return await call_next(request)
|
||||
|
||||
content_type = request.headers.get("content-type", "")
|
||||
is_multipart = "multipart/form-data" in content_type
|
||||
|
||||
if is_multipart:
|
||||
limit = self.max_upload_size
|
||||
limit_description = "file upload"
|
||||
config_var = "MAX_UPLOAD_SIZE"
|
||||
else:
|
||||
limit = self.max_body_size
|
||||
limit_description = "request body"
|
||||
config_var = "MAX_REQUEST_BODY_SIZE"
|
||||
|
||||
if content_length > limit:
|
||||
logger.warning(
|
||||
f"Rejected oversized {limit_description}: "
|
||||
f"{content_length} bytes > {limit} bytes limit "
|
||||
f"(configure with {config_var})"
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=413,
|
||||
content={
|
||||
"detail": (
|
||||
f"Request body too large: {content_length} bytes "
|
||||
f"(maximum allowed: {limit} bytes). "
|
||||
f"Adjust the {config_var} environment variable to change this limit. "
|
||||
f"See SECURITY_AUDIT.md for details."
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
return await call_next(request)
|
||||
@@ -1,117 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Security Headers Middleware for DocuElevate.
|
||||
|
||||
This middleware adds security headers to HTTP responses to improve browser-side security.
|
||||
Headers can be configured via environment variables to support different deployment scenarios:
|
||||
- Direct deployment: Enable all security headers
|
||||
- Reverse proxy deployment (Traefik, Nginx, etc.): Disable headers if proxy adds them
|
||||
|
||||
See docs/DeploymentGuide.md for configuration guidance.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Callable
|
||||
|
||||
from fastapi import Request, Response
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SecurityHeadersMiddleware(BaseHTTPMiddleware):
|
||||
"""
|
||||
Middleware to add security headers to HTTP responses.
|
||||
|
||||
This middleware adds the following security headers when enabled:
|
||||
- Strict-Transport-Security (HSTS): Forces HTTPS connections
|
||||
- Content-Security-Policy (CSP): Controls resource loading
|
||||
- X-Frame-Options: Prevents clickjacking attacks
|
||||
- X-Content-Type-Options: Prevents MIME-sniffing attacks
|
||||
|
||||
Headers are configurable via environment variables to support different deployment scenarios.
|
||||
"""
|
||||
|
||||
def __init__(self, app, config):
|
||||
"""
|
||||
Initialize the security headers middleware.
|
||||
|
||||
Args:
|
||||
app: FastAPI application instance
|
||||
config: Configuration object with security header settings
|
||||
"""
|
||||
super().__init__(app)
|
||||
self.config = config
|
||||
self.enabled = config.security_headers_enabled
|
||||
|
||||
if self.enabled:
|
||||
logger.info("Security headers middleware enabled")
|
||||
logger.debug(
|
||||
f"HSTS: {config.security_header_hsts_enabled}, "
|
||||
f"CSP: {config.security_header_csp_enabled}, "
|
||||
f"X-Frame-Options: {config.security_header_x_frame_options_enabled}, "
|
||||
f"X-Content-Type-Options: {config.security_header_x_content_type_options_enabled}"
|
||||
)
|
||||
else:
|
||||
logger.info("Security headers middleware disabled (likely handled by reverse proxy)")
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
"""
|
||||
Process the request and add security headers to the response.
|
||||
|
||||
Args:
|
||||
request: Incoming HTTP request
|
||||
call_next: Next middleware or route handler
|
||||
|
||||
Returns:
|
||||
HTTP response with security headers added (if enabled)
|
||||
"""
|
||||
# Process the request
|
||||
response = await call_next(request)
|
||||
|
||||
# Add security headers if enabled
|
||||
if self.enabled:
|
||||
self._add_security_headers(response)
|
||||
|
||||
return response
|
||||
|
||||
def _add_security_headers(self, response: Response) -> None:
|
||||
"""
|
||||
Add configured security headers to the response.
|
||||
|
||||
Args:
|
||||
response: HTTP response to add headers to
|
||||
"""
|
||||
# Strict-Transport-Security (HSTS)
|
||||
# Forces browsers to use HTTPS for all future requests to this domain
|
||||
# max-age: Time in seconds browsers should remember to only use HTTPS
|
||||
# includeSubDomains: Apply to all subdomains
|
||||
# preload: Allow inclusion in browser HSTS preload lists
|
||||
if self.config.security_header_hsts_enabled:
|
||||
hsts_value = self.config.security_header_hsts_value
|
||||
response.headers["Strict-Transport-Security"] = hsts_value
|
||||
logger.debug(f"Added HSTS header: {hsts_value}")
|
||||
|
||||
# Content-Security-Policy (CSP)
|
||||
# Controls which resources browsers are allowed to load for this page
|
||||
# This helps prevent XSS attacks and other code injection attacks
|
||||
if self.config.security_header_csp_enabled:
|
||||
csp_value = self.config.security_header_csp_value
|
||||
response.headers["Content-Security-Policy"] = csp_value
|
||||
logger.debug(f"Added CSP header: {csp_value[:50]}...")
|
||||
|
||||
# X-Frame-Options
|
||||
# Prevents the page from being loaded in a frame/iframe
|
||||
# This helps prevent clickjacking attacks
|
||||
if self.config.security_header_x_frame_options_enabled:
|
||||
x_frame_value = self.config.security_header_x_frame_options_value
|
||||
response.headers["X-Frame-Options"] = x_frame_value
|
||||
logger.debug(f"Added X-Frame-Options header: {x_frame_value}")
|
||||
|
||||
# X-Content-Type-Options
|
||||
# Prevents browsers from MIME-sniffing responses away from declared content-type
|
||||
# This helps prevent XSS attacks based on content-type confusion
|
||||
if self.config.security_header_x_content_type_options_enabled:
|
||||
response.headers["X-Content-Type-Options"] = "nosniff"
|
||||
logger.debug("Added X-Content-Type-Options header: nosniff")
|
||||
+10
-106
@@ -1,13 +1,10 @@
|
||||
# app/models.py
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
|
||||
from sqlalchemy import Column, String, Integer, DateTime, func, ForeignKey
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from app.database import Base
|
||||
|
||||
# Foreign key constants
|
||||
_FILES_ID_FK = "files.id"
|
||||
|
||||
|
||||
class DocumentMetadata(Base):
|
||||
__tablename__ = "documents"
|
||||
|
||||
@@ -18,15 +15,13 @@ class DocumentMetadata(Base):
|
||||
tags = Column(String)
|
||||
summary = Column(String)
|
||||
|
||||
|
||||
class FileRecord(Base):
|
||||
__tablename__ = "files"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
|
||||
|
||||
# Hash of the file content (e.g. SHA-256)
|
||||
# Note: duplicates are allowed so filehash is not unique
|
||||
filehash = Column(String, index=True, nullable=False)
|
||||
filehash = Column(String, unique=True, index=True, nullable=False)
|
||||
|
||||
# The name of the file as it was originally uploaded (if known)
|
||||
original_filename = Column(String)
|
||||
@@ -34,112 +29,21 @@ class FileRecord(Base):
|
||||
# The name/path we store on disk (e.g. /workdir/tmp/<uuid>.pdf)
|
||||
local_filename = Column(String, nullable=False)
|
||||
|
||||
# Immutable original copy path (e.g. /workdir/original/<uuid>.pdf)
|
||||
# This is the first copy made when the file is ingested
|
||||
original_file_path = Column(String)
|
||||
|
||||
# Processed copy path (e.g. /workdir/processed/2024-01-01_Invoice.pdf)
|
||||
# This is the final file with embedded metadata before upload
|
||||
processed_file_path = Column(String)
|
||||
|
||||
# Size of the file in bytes
|
||||
file_size = Column(Integer, nullable=False)
|
||||
|
||||
# MIME type or extension (optional)
|
||||
mime_type = Column(String, index=True)
|
||||
|
||||
# Deduplication tracking: True if this file is a duplicate of another file
|
||||
# When a duplicate is detected, this file record is created but marked as duplicate
|
||||
is_duplicate = Column(Boolean, default=False, nullable=False, index=True)
|
||||
|
||||
# If this is a duplicate, record the ID of the original file for reference
|
||||
duplicate_of_id = Column(Integer, ForeignKey(_FILES_ID_FK), nullable=True)
|
||||
|
||||
# Full OCR/extracted text for full-text search and RAG
|
||||
ocr_text = Column(Text, nullable=True)
|
||||
|
||||
# AI-assessed quality score for the OCR/extracted text (0–100; NULL = not yet assessed)
|
||||
ocr_quality_score = Column(Integer, nullable=True)
|
||||
|
||||
# AI-extracted metadata stored as JSON string (filename, tags, title, sender, etc.)
|
||||
ai_metadata = Column(Text, nullable=True)
|
||||
|
||||
# Human-readable document title from AI metadata
|
||||
document_title = Column(String, nullable=True)
|
||||
mime_type = Column(String)
|
||||
|
||||
# Timestamp when we inserted this record
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
|
||||
|
||||
|
||||
class FileProcessingStep(Base):
|
||||
"""
|
||||
Tracks the current status of each processing step for a file.
|
||||
This provides a definitive, queryable state for each step without scanning logs.
|
||||
"""
|
||||
|
||||
__tablename__ = "file_processing_steps"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
file_id = Column(Integer, ForeignKey(_FILES_ID_FK), nullable=False, index=True)
|
||||
step_name = Column(String, nullable=False, index=True) # e.g., "hash_file", "upload_to_dropbox"
|
||||
status = Column(String, nullable=False, index=True) # "pending", "in_progress", "success", "failure", "skipped"
|
||||
started_at = Column(DateTime(timezone=True), nullable=True) # When step started
|
||||
completed_at = Column(DateTime(timezone=True), nullable=True) # When step finished (success/failure)
|
||||
error_message = Column(Text, nullable=True) # Error message if status is "failure"
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
__table_args__ = (UniqueConstraint("file_id", "step_name", name="unique_file_step"),)
|
||||
|
||||
|
||||
class ProcessingLog(Base):
|
||||
__tablename__ = "processing_logs"
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
file_id = Column(Integer, ForeignKey(_FILES_ID_FK), nullable=True, index=True) # Optional file association
|
||||
file_id = Column(Integer, ForeignKey("files.id"), nullable=True) # Optional file association
|
||||
task_id = Column(String, index=True) # Celery task ID
|
||||
step_name = Column(String) # e.g., "OCR", "convert_to_pdf", "upload_s3"
|
||||
status = Column(String) # "pending", "in_progress", "success", "failure"
|
||||
step_name = Column(String) # e.g., "OCR", "convert_to_pdf", "upload_s3"
|
||||
status = Column(String) # "pending", "in_progress", "success", "failure"
|
||||
message = Column(String, nullable=True) # Error text or success note
|
||||
detail = Column(Text, nullable=True) # Verbose worker log output for diagnostics
|
||||
timestamp = Column(DateTime(timezone=True), server_default=func.now(), index=True)
|
||||
|
||||
|
||||
class ApplicationSettings(Base):
|
||||
"""Store application settings in database with precedence over environment variables"""
|
||||
|
||||
__tablename__ = "application_settings"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
key = Column(String, unique=True, index=True, nullable=False) # Setting key (e.g., 'database_url')
|
||||
value = Column(String, nullable=True) # Setting value (stored as string, converted as needed)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class SettingsAuditLog(Base):
|
||||
"""Audit log for all configuration changes made via the settings UI."""
|
||||
|
||||
__tablename__ = "settings_audit_log"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
key = Column(String, nullable=False, index=True) # Setting key that was changed
|
||||
old_value = Column(String, nullable=True) # Previous value (None if first-time set)
|
||||
new_value = Column(String, nullable=True) # New value (None if deleted)
|
||||
changed_by = Column(String, nullable=False) # Username of the admin who made the change
|
||||
changed_at = Column(DateTime(timezone=True), server_default=func.now(), index=True)
|
||||
action = Column(String, nullable=False) # "update" or "delete"
|
||||
|
||||
|
||||
class SavedSearch(Base):
|
||||
"""User-defined saved search filters for quick access to frequently used filter combinations."""
|
||||
|
||||
__tablename__ = "saved_searches"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
user_id = Column(String, nullable=False, index=True) # Username or user identifier from session
|
||||
name = Column(String, nullable=False) # Human-readable name for the saved search
|
||||
filters = Column(Text, nullable=False) # JSON-encoded filter parameters
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
__table_args__ = (UniqueConstraint("user_id", "name", name="unique_user_search_name"),)
|
||||
timestamp = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
# Import tasks so they can be discovered by Celery
|
||||
from app.tasks.process_document import process_document # noqa: F401
|
||||
from app.tasks.process_with_azure_document_intelligence import process_with_azure_document_intelligence # noqa: F401
|
||||
|
||||
@@ -1,288 +0,0 @@
|
||||
import asyncio
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
from app.api.azure import test_azure_connection
|
||||
from app.api.dropbox import test_dropbox_token
|
||||
from app.api.google_drive import test_google_drive_token
|
||||
from app.api.onedrive import test_onedrive_token
|
||||
|
||||
# Import the test functions from API routes
|
||||
from app.api.openai import test_ai_provider_connection, test_openai_connection
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
|
||||
# Import config validation utilities
|
||||
from app.utils.config_validator import get_provider_status, validate_storage_configs
|
||||
from app.utils.notification import notify_credential_failure
|
||||
|
||||
|
||||
# Create an enhanced mock Request object for API functions that expect it
|
||||
class MockRequest:
|
||||
"""Mock request object with session and other attributes needed for API functions"""
|
||||
|
||||
def __init__(self):
|
||||
self.session = {"user": {"id": "credential_checker", "name": "System Credential Checker"}}
|
||||
self.app = None
|
||||
self.headers = {}
|
||||
self.query_params = {}
|
||||
self.path_params = {}
|
||||
|
||||
async def json(self):
|
||||
return {}
|
||||
|
||||
async def form(self):
|
||||
return {}
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Path to store failure counts
|
||||
FAILURE_STATE_FILE = os.path.join(settings.workdir, "credential_failures.json")
|
||||
|
||||
|
||||
def get_failure_state():
|
||||
"""Read the failure state from file"""
|
||||
try:
|
||||
if os.path.exists(FAILURE_STATE_FILE):
|
||||
with open(FAILURE_STATE_FILE, "r") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading failure state file: {e}")
|
||||
|
||||
# Default empty state
|
||||
return {}
|
||||
|
||||
|
||||
def save_failure_state(state):
|
||||
"""Save failure state to file"""
|
||||
try:
|
||||
with open(FAILURE_STATE_FILE, "w") as f:
|
||||
json.dump(state, f)
|
||||
except Exception as e:
|
||||
logger.error(f"Error saving failure state file: {e}")
|
||||
|
||||
|
||||
# Helper function to get the inner function without the decorator
|
||||
def unwrap_decorated_function(func):
|
||||
"""Get the original function from a decorated function"""
|
||||
if hasattr(func, "__wrapped__"):
|
||||
return unwrap_decorated_function(func.__wrapped__)
|
||||
return func
|
||||
|
||||
|
||||
# Create synchronous versions of the test functions that bypass authentication
|
||||
def sync_test_ai_provider_connection():
|
||||
"""Synchronous wrapper for the AI provider test function that bypasses auth."""
|
||||
inner_func = unwrap_decorated_function(test_ai_provider_connection)
|
||||
request = MockRequest()
|
||||
if inspect.iscoroutinefunction(inner_func):
|
||||
return asyncio.run(inner_func(request))
|
||||
return inner_func(request)
|
||||
|
||||
|
||||
def sync_test_openai_connection():
|
||||
"""Synchronous wrapper for the OpenAI test function that bypasses auth."""
|
||||
# Get the original function without the @require_login decorator
|
||||
inner_func = unwrap_decorated_function(test_openai_connection)
|
||||
request = MockRequest()
|
||||
if inspect.iscoroutinefunction(inner_func):
|
||||
return asyncio.run(inner_func(request))
|
||||
return inner_func(request)
|
||||
|
||||
|
||||
def sync_test_azure_connection():
|
||||
"""Synchronous wrapper for the Azure test function that bypasses auth"""
|
||||
inner_func = unwrap_decorated_function(test_azure_connection)
|
||||
request = MockRequest()
|
||||
if inspect.iscoroutinefunction(inner_func):
|
||||
return asyncio.run(inner_func(request))
|
||||
return inner_func(request)
|
||||
|
||||
|
||||
def sync_test_dropbox_token():
|
||||
"""Synchronous wrapper for the Dropbox test function that bypasses auth"""
|
||||
inner_func = unwrap_decorated_function(test_dropbox_token)
|
||||
request = MockRequest()
|
||||
if inspect.iscoroutinefunction(inner_func):
|
||||
return asyncio.run(inner_func(request))
|
||||
return inner_func(request)
|
||||
|
||||
|
||||
def sync_test_google_drive_token():
|
||||
"""Synchronous wrapper for the Google Drive test function that bypasses auth"""
|
||||
inner_func = unwrap_decorated_function(test_google_drive_token)
|
||||
request = MockRequest()
|
||||
if inspect.iscoroutinefunction(inner_func):
|
||||
return asyncio.run(inner_func(request))
|
||||
return inner_func(request)
|
||||
|
||||
|
||||
def sync_test_onedrive_token():
|
||||
"""Synchronous wrapper for the OneDrive test function that bypasses auth"""
|
||||
inner_func = unwrap_decorated_function(test_onedrive_token)
|
||||
request = MockRequest()
|
||||
if inspect.iscoroutinefunction(inner_func):
|
||||
return asyncio.run(inner_func(request))
|
||||
return inner_func(request)
|
||||
|
||||
|
||||
@celery.task
|
||||
def check_credentials():
|
||||
"""Check all configured credentials and notify if any are invalid"""
|
||||
logger.info("Starting credential check task")
|
||||
|
||||
# Load current failure state
|
||||
failure_state = get_failure_state()
|
||||
|
||||
# Track failures
|
||||
failures = []
|
||||
|
||||
# Get provider configurations from config_validator
|
||||
provider_status = get_provider_status()
|
||||
storage_configs = validate_storage_configs()
|
||||
|
||||
# Define services with their test functions and configuration status
|
||||
services = [
|
||||
{
|
||||
"name": "AI Provider",
|
||||
"check_func": sync_test_ai_provider_connection,
|
||||
"configured": provider_status.get("AI Provider", {}).get("configured", False),
|
||||
"config_issues": [],
|
||||
},
|
||||
{
|
||||
"name": "Azure Document Intelligence",
|
||||
"check_func": sync_test_azure_connection,
|
||||
"configured": provider_status.get("Azure AI", {}).get("configured", False),
|
||||
"config_issues": [], # Azure isn't in storage_configs
|
||||
},
|
||||
{
|
||||
"name": "Dropbox",
|
||||
"check_func": sync_test_dropbox_token,
|
||||
"configured": provider_status.get("Dropbox", {}).get("configured", False),
|
||||
"config_issues": storage_configs.get("dropbox", []),
|
||||
},
|
||||
{
|
||||
"name": "Google Drive",
|
||||
"check_func": sync_test_google_drive_token,
|
||||
"configured": provider_status.get("Google Drive", {}).get("configured", False),
|
||||
"config_issues": storage_configs.get("google_drive", []),
|
||||
},
|
||||
{
|
||||
"name": "OneDrive",
|
||||
"check_func": sync_test_onedrive_token,
|
||||
"configured": provider_status.get("OneDrive", {}).get("configured", False),
|
||||
"config_issues": storage_configs.get("onedrive", []),
|
||||
},
|
||||
]
|
||||
|
||||
# Check each service
|
||||
results = {}
|
||||
current_time = int(time.time())
|
||||
|
||||
for service in services:
|
||||
service_name = service["name"]
|
||||
logger.info(f"Checking credentials for {service_name}")
|
||||
|
||||
# Skip services that aren't configured
|
||||
if not service["configured"]:
|
||||
config_issues = service["config_issues"]
|
||||
issue_msg = "Not properly configured" + (f": {', '.join(config_issues)}" if config_issues else "")
|
||||
logger.info(f"Skipping {service_name}: {issue_msg}")
|
||||
|
||||
results[service_name] = {"status": "unconfigured", "message": issue_msg}
|
||||
continue
|
||||
|
||||
try:
|
||||
# Call the synchronized test function and get the result
|
||||
result = service["check_func"]()
|
||||
|
||||
# All test functions return a dict with "status" field
|
||||
is_valid = result.get("status") == "success"
|
||||
error_message = result.get("message", "Unknown error")
|
||||
|
||||
# Store the result
|
||||
results[service_name] = {"status": "valid" if is_valid else "invalid", "message": error_message}
|
||||
|
||||
if not is_valid:
|
||||
failures.append(service_name)
|
||||
|
||||
# Get current failure count for this service
|
||||
service_state = failure_state.get(service_name, {"count": 0, "last_notified": 0})
|
||||
service_state["count"] = service_state.get("count", 0) + 1
|
||||
|
||||
# Only notify if we haven't reached the notification threshold (3 failures)
|
||||
# or if this is the first failure after a recovery
|
||||
if service_state["count"] <= 3 or service_state.get("recovered", False):
|
||||
notify_credential_failure(service_name, error_message)
|
||||
service_state["last_notified"] = current_time
|
||||
service_state["recovered"] = False
|
||||
logger.warning(
|
||||
f"{service_name} credentials check failed ({service_state['count']} times): {error_message}"
|
||||
)
|
||||
else:
|
||||
# We're in cooldown mode
|
||||
logger.warning(
|
||||
f"{service_name} credentials check failed ({service_state['count']} times): "
|
||||
f"{error_message} - notification suppressed"
|
||||
)
|
||||
|
||||
# Update failure state
|
||||
failure_state[service_name] = service_state
|
||||
else:
|
||||
logger.info(f"{service_name} credentials are valid")
|
||||
|
||||
# Check if this was previously failing and now recovered
|
||||
if service_name in failure_state and failure_state[service_name].get("count", 0) > 0:
|
||||
logger.info(f"{service_name} has recovered after {failure_state[service_name]['count']} failures")
|
||||
|
||||
# Mark it as recovered and reset count
|
||||
failure_state[service_name] = {"count": 0, "recovered": True, "last_notified": 0}
|
||||
elif service_name in failure_state:
|
||||
# Just make sure recovered flag is cleared if it was there
|
||||
failure_state[service_name]["recovered"] = True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking {service_name} credentials: {e}", exc_info=True)
|
||||
failures.append(service_name)
|
||||
error_message = f"Exception during credential check: {str(e)}"
|
||||
|
||||
# Get current failure count for this service
|
||||
service_state = failure_state.get(service_name, {"count": 0, "last_notified": 0})
|
||||
service_state["count"] = service_state.get("count", 0) + 1
|
||||
|
||||
# Only notify if we haven't reached the notification threshold or if we just recovered
|
||||
if service_state["count"] <= 3 or service_state.get("recovered", False):
|
||||
notify_credential_failure(service_name, error_message)
|
||||
service_state["last_notified"] = current_time
|
||||
service_state["recovered"] = False
|
||||
|
||||
# Update failure state
|
||||
failure_state[service_name] = service_state
|
||||
|
||||
# Store the error result
|
||||
results[service_name] = {"status": "error", "message": error_message}
|
||||
|
||||
# Save updated failure state
|
||||
save_failure_state(failure_state)
|
||||
|
||||
# Count only services that were actually checked (configured services)
|
||||
configured_services = [s for s in services if s["configured"]]
|
||||
num_configured = len(configured_services)
|
||||
|
||||
# Summarize results
|
||||
logger.info(
|
||||
f"Credential check completed. Configured services: {num_configured}, "
|
||||
f"Valid: {num_configured - len(failures)}, Invalid: {len(failures)}"
|
||||
)
|
||||
|
||||
return {
|
||||
"checked": num_configured,
|
||||
"unconfigured": len(services) - num_configured,
|
||||
"failures": len(failures),
|
||||
"results": results,
|
||||
"failure_state": failure_state,
|
||||
}
|
||||
+42
-321
@@ -1,351 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import requests
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import filetype
|
||||
import puremagic
|
||||
import requests
|
||||
from celery import shared_task
|
||||
|
||||
from app.config import settings
|
||||
from app.tasks.process_document import process_document
|
||||
from app.utils import log_task_progress
|
||||
from app.tasks.process_document import process_document # Updated import
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _detect_mime_type_from_magic(file_path: str) -> Optional[str]:
|
||||
"""
|
||||
Detect MIME type from file headers using platform-agnostic libraries.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file on disk.
|
||||
|
||||
Returns:
|
||||
Detected MIME type or None if unknown.
|
||||
"""
|
||||
try:
|
||||
matches = puremagic.from_file(file_path)
|
||||
if matches:
|
||||
return matches[0].mime_type
|
||||
except puremagic.PureError:
|
||||
pass
|
||||
|
||||
guess = filetype.guess(file_path)
|
||||
if guess:
|
||||
return guess.mime
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _detect_mime_type(file_path: str, original_filename: Optional[str]) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
Detect MIME type using extension, original filename, or magic bytes.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file on disk.
|
||||
original_filename: Optional original filename provided at upload time.
|
||||
|
||||
Returns:
|
||||
Tuple of detected MIME type and encoding.
|
||||
"""
|
||||
mime_type, encoding = mimetypes.guess_type(file_path)
|
||||
if mime_type:
|
||||
return mime_type, encoding
|
||||
|
||||
if original_filename:
|
||||
mime_type, encoding = mimetypes.guess_type(original_filename)
|
||||
if mime_type:
|
||||
return mime_type, encoding
|
||||
|
||||
return _detect_mime_type_from_magic(file_path), encoding
|
||||
|
||||
|
||||
def _detect_extension(file_path: str, original_filename: Optional[str], mime_type: Optional[str]) -> str:
|
||||
"""
|
||||
Detect file extension from file path, original filename, or MIME type.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file on disk.
|
||||
original_filename: Optional original filename provided at upload time.
|
||||
mime_type: Detected MIME type if available.
|
||||
|
||||
Returns:
|
||||
File extension with leading dot (e.g., ".pdf") or empty string if unknown.
|
||||
"""
|
||||
file_ext = os.path.splitext(file_path)[1].lower()
|
||||
if file_ext:
|
||||
return file_ext
|
||||
|
||||
if original_filename:
|
||||
file_ext = os.path.splitext(original_filename)[1].lower()
|
||||
if file_ext:
|
||||
return file_ext
|
||||
|
||||
if mime_type:
|
||||
return mimetypes.guess_extension(mime_type) or ""
|
||||
|
||||
try:
|
||||
matches = puremagic.from_file(file_path)
|
||||
if matches and matches[0].extension:
|
||||
return f".{matches[0].extension.lstrip('.')}"
|
||||
except puremagic.PureError:
|
||||
pass
|
||||
|
||||
guess = filetype.guess(file_path)
|
||||
if guess and guess.extension:
|
||||
return f".{guess.extension.lstrip('.')}"
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def _build_filename(file_path: str, original_filename: Optional[str], file_ext: str) -> str:
|
||||
"""
|
||||
Build a filename for upload that includes a valid extension when possible.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file on disk.
|
||||
original_filename: Optional original filename provided at upload time.
|
||||
file_ext: Detected file extension.
|
||||
|
||||
Returns:
|
||||
Filename to send to Gotenberg.
|
||||
"""
|
||||
if original_filename and os.path.splitext(original_filename)[1]:
|
||||
return original_filename
|
||||
|
||||
base_name = os.path.basename(file_path)
|
||||
if file_ext and not base_name.lower().endswith(file_ext):
|
||||
return f"{base_name}{file_ext}"
|
||||
|
||||
return base_name
|
||||
|
||||
|
||||
@shared_task(bind=True)
|
||||
def convert_to_pdf(self, file_path: str, original_filename: Optional[str] = None) -> Optional[str]:
|
||||
@shared_task
|
||||
def convert_to_pdf(file_path):
|
||||
"""
|
||||
Converts a file to PDF using Gotenberg's API.
|
||||
Determines the appropriate Gotenberg endpoint based on the file's MIME type.
|
||||
On success, saves the PDF locally and enqueues it for processing.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to convert
|
||||
original_filename: Optional original filename (if different from path basename)
|
||||
On success, saves the PDF locally and enqueues it for S3 upload.
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting PDF conversion: {file_path}")
|
||||
log_task_progress(task_id, "convert_to_pdf", "in_progress", f"Converting file: {os.path.basename(file_path)}")
|
||||
|
||||
gotenberg_url = getattr(settings, "gotenberg_url", None)
|
||||
if not gotenberg_url:
|
||||
logger.error(f"[{task_id}] Gotenberg URL is not configured in settings.")
|
||||
log_task_progress(task_id, "convert_to_pdf", "failure", "Gotenberg URL not configured")
|
||||
logger.error("Gotenberg URL is not configured in settings.")
|
||||
return
|
||||
|
||||
# Try to guess the MIME type based on file content and extension
|
||||
mime_type, encoding = _detect_mime_type(file_path, original_filename)
|
||||
file_ext = _detect_extension(file_path, original_filename, mime_type)
|
||||
logger.info(f"[{task_id}] Guessed MIME type for '{file_path}' is: {mime_type}, extension: {file_ext}")
|
||||
if not mime_type and not file_ext:
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"detect_file_type",
|
||||
"failure",
|
||||
"Unable to determine file type for conversion",
|
||||
detail=(
|
||||
f"File: {file_path}\n"
|
||||
f"Original filename: {original_filename or 'N/A'}\n"
|
||||
"No extension and no detectable magic header."
|
||||
),
|
||||
)
|
||||
logger.error(f"[{task_id}] Unable to determine file type for conversion: {file_path}")
|
||||
return None
|
||||
log_task_progress(task_id, "detect_file_type", "success", f"File type: {mime_type or file_ext}")
|
||||
# Try to guess the MIME type based on file content (using extension-based fallback)
|
||||
mime_type, encoding = mimetypes.guess_type(file_path)
|
||||
logger.info(f"Guessed MIME type for '{file_path}' is: {mime_type}")
|
||||
|
||||
# Determine which Gotenberg endpoint to use
|
||||
endpoint = None
|
||||
form_data = {}
|
||||
files = {}
|
||||
form_key = "files" # Default form key for most endpoints
|
||||
|
||||
# Dictionary mapping file extensions to their handlers
|
||||
OFFICE_EXTENSIONS = {
|
||||
".doc",
|
||||
".docx",
|
||||
".docm",
|
||||
".dot",
|
||||
".dotx",
|
||||
".dotm", # Word
|
||||
".xls",
|
||||
".xlsx",
|
||||
".xlsm",
|
||||
".xlsb",
|
||||
".xlt",
|
||||
".xltx",
|
||||
".xlw", # Excel
|
||||
".ppt",
|
||||
".pptx",
|
||||
".pptm",
|
||||
".pps",
|
||||
".ppsx",
|
||||
".pot",
|
||||
".potx", # PowerPoint
|
||||
".odt",
|
||||
".ods",
|
||||
".odp",
|
||||
".odg",
|
||||
".odf", # OpenOffice/LibreOffice
|
||||
".rtf",
|
||||
".txt",
|
||||
".csv", # Text formats
|
||||
".pdf", # PDF (already in PDF format but can be processed)
|
||||
}
|
||||
|
||||
IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".webp", ".svg"}
|
||||
|
||||
HTML_EXTENSIONS = {".html", ".htm"}
|
||||
|
||||
# Use LibreOffice endpoint for office documents and images
|
||||
if (
|
||||
(mime_type and "office" in mime_type)
|
||||
or (mime_type and "opendocument" in mime_type)
|
||||
or (mime_type and mime_type.startswith("image/"))
|
||||
or file_ext in OFFICE_EXTENSIONS
|
||||
or file_ext in IMAGE_EXTENSIONS
|
||||
):
|
||||
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
||||
files = {"files": (_build_filename(file_path, original_filename, file_ext), open(file_path, "rb"))}
|
||||
|
||||
# Add some quality settings for better PDF output
|
||||
form_data = {
|
||||
"landscape": "false",
|
||||
"exportBookmarks": "true",
|
||||
"exportNotes": "false",
|
||||
"losslessImageCompression": "true", # Use lossless compression for images
|
||||
"pdfa": "PDF/A-2b", # Produce PDF/A-2b compatible output
|
||||
}
|
||||
|
||||
# Use Chromium endpoint for HTML documents
|
||||
elif (mime_type and mime_type == "text/html") or file_ext in HTML_EXTENSIONS:
|
||||
endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
|
||||
# Gotenberg requires the form field to be exactly 'index.html'
|
||||
# The content filename doesn't matter, just the form field key
|
||||
files = {"index.html": ("index.html", open(file_path, "rb"))}
|
||||
|
||||
# Add options for better HTML to PDF conversion
|
||||
form_data = {
|
||||
"paperWidth": "8.27", # A4 width in inches
|
||||
"paperHeight": "11.7", # A4 height in inches
|
||||
"marginTop": "0.4",
|
||||
"marginBottom": "0.4",
|
||||
"marginLeft": "0.4",
|
||||
"marginRight": "0.4",
|
||||
"printBackground": "true",
|
||||
"preferCssPageSize": "false",
|
||||
"waitDelay": "2s", # Wait for JavaScript to execute
|
||||
}
|
||||
|
||||
# Use Markdown route for markdown files
|
||||
elif (mime_type and mime_type in ["text/markdown", "text/x-markdown"]) or file_ext in [".md", ".markdown"]:
|
||||
# For Markdown, we need both the markdown file and an HTML wrapper
|
||||
endpoint = f"{gotenberg_url}/forms/chromium/convert/markdown"
|
||||
|
||||
# Create a simple HTML wrapper for the markdown
|
||||
# IMPORTANT: The filename in the template must match the key used in the files dictionary
|
||||
markdown_filename = os.path.basename(file_path)
|
||||
html_wrapper = f"""<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Converted Markdown</title>
|
||||
<style>
|
||||
body {{
|
||||
font-family: Arial, sans-serif;
|
||||
line-height: 1.6;
|
||||
margin: 2em;
|
||||
max-width: 50em;
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
{{{{ toHTML "{markdown_filename}" }}}}
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
# Create a temporary HTML wrapper file
|
||||
wrapper_path = os.path.join(os.path.dirname(file_path), "md_wrapper.html")
|
||||
with open(wrapper_path, "w") as f:
|
||||
f.write(html_wrapper)
|
||||
|
||||
try:
|
||||
files = {
|
||||
"index.html": ("index.html", open(wrapper_path, "rb")),
|
||||
markdown_filename: (markdown_filename, open(file_path, "rb")),
|
||||
}
|
||||
|
||||
form_data = {
|
||||
"paperWidth": "8.27", # A4 width in inches
|
||||
"paperHeight": "11.7", # A4 height in inches
|
||||
"marginTop": "0.4",
|
||||
"marginBottom": "0.4",
|
||||
"marginLeft": "0.4",
|
||||
"marginRight": "0.4",
|
||||
}
|
||||
finally:
|
||||
# Clean up the temporary wrapper file after preparing the request
|
||||
if os.path.exists(wrapper_path):
|
||||
os.remove(wrapper_path)
|
||||
|
||||
# Fallback to LibreOffice for everything else
|
||||
if mime_type:
|
||||
if mime_type == "text/html":
|
||||
endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
|
||||
# The Chromium HTML endpoint expects the HTML file to be provided under the key "index.html"
|
||||
form_key = "index.html"
|
||||
elif mime_type.startswith("image/"):
|
||||
# For images, we use the LibreOffice endpoint (which supports image conversion)
|
||||
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
||||
elif mime_type.startswith("text/plain"):
|
||||
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
||||
elif mime_type in ["text/markdown", "text/x-markdown"]:
|
||||
# Optionally, you could use the Chromium markdown endpoint if you have an HTML wrapper.
|
||||
# For now, we'll fallback to LibreOffice.
|
||||
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
||||
else:
|
||||
# For all other MIME types (e.g. Office documents), use the LibreOffice endpoint.
|
||||
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
||||
else:
|
||||
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
||||
files = {"files": (_build_filename(file_path, original_filename, file_ext), open(file_path, "rb"))}
|
||||
logger.warning(f"Using fallback conversion for unknown type: {mime_type} / {file_ext}")
|
||||
|
||||
if not endpoint:
|
||||
logger.error(f"[{task_id}] Could not determine Gotenberg endpoint for file type: {mime_type}")
|
||||
log_task_progress(task_id, "convert_to_pdf", "failure", f"Unknown file type: {mime_type}")
|
||||
return None
|
||||
# If MIME detection fails, fallback to extension-based detection.
|
||||
ext = os.path.splitext(file_path)[1].lower()
|
||||
if ext in [".html", ".htm"]:
|
||||
endpoint = f"{gotenberg_url}/forms/chromium/convert/html"
|
||||
form_key = "index.html"
|
||||
else:
|
||||
endpoint = f"{gotenberg_url}/forms/libreoffice/convert"
|
||||
|
||||
try:
|
||||
logger.info(f"[{task_id}] Converting {file_path} using endpoint: {endpoint}")
|
||||
log_task_progress(task_id, "call_gotenberg", "in_progress", "Calling Gotenberg API")
|
||||
|
||||
# Send the conversion request to Gotenberg
|
||||
response = requests.post(endpoint, files=files, data=form_data, timeout=settings.http_request_timeout)
|
||||
with open(file_path, "rb") as f:
|
||||
files = {form_key: f}
|
||||
response = requests.post(endpoint, files=files)
|
||||
|
||||
if response.status_code == 200:
|
||||
# Save the converted PDF
|
||||
converted_file_path = os.path.splitext(file_path)[0] + ".pdf"
|
||||
with open(converted_file_path, "wb") as out_file:
|
||||
out_file.write(response.content)
|
||||
|
||||
logger.info(f"[{task_id}] Converted file saved as PDF: {converted_file_path}")
|
||||
log_task_progress(task_id, "call_gotenberg", "success", "PDF conversion successful")
|
||||
log_task_progress(
|
||||
task_id, "convert_to_pdf", "success", f"Converted to PDF: {os.path.basename(converted_file_path)}"
|
||||
)
|
||||
|
||||
# Enqueue the PDF for further processing, preserving original filename if provided
|
||||
if original_filename:
|
||||
# Change extension to .pdf for the original filename
|
||||
original_base = os.path.splitext(original_filename)[0]
|
||||
pdf_original_filename = f"{original_base}.pdf"
|
||||
process_document.delay(converted_file_path, original_filename=pdf_original_filename)
|
||||
else:
|
||||
process_document.delay(converted_file_path)
|
||||
|
||||
logger.info(f"Converted file saved as PDF: {converted_file_path}")
|
||||
process_document.delay(converted_file_path) # Updated function call
|
||||
return converted_file_path
|
||||
else:
|
||||
error_msg = f"Status code: {response.status_code}"
|
||||
logger.error(
|
||||
f"[{task_id}] Conversion failed for {file_path}. {error_msg}, Response: {response.text[:500]}..."
|
||||
)
|
||||
log_task_progress(task_id, "call_gotenberg", "failure", error_msg)
|
||||
log_task_progress(task_id, "convert_to_pdf", "failure", f"Conversion failed: {error_msg}")
|
||||
return None
|
||||
logger.error(f"Conversion failed for {file_path}. Status code: {response.status_code}")
|
||||
except Exception as e:
|
||||
logger.exception(f"[{task_id}] Error converting {file_path} to PDF: {e}")
|
||||
log_task_progress(task_id, "convert_to_pdf", "failure", f"Exception: {str(e)}")
|
||||
return None
|
||||
logger.exception(f"Error converting {file_path} to PDF: {e}")
|
||||
|
||||
@@ -1,69 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pypdf # Upgraded from PyPDF2 to fix CVE-2023-36464
|
||||
import fitz # PyMuPDF for PDF metadata editing
|
||||
import json
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.tasks.finalize_document_storage import finalize_document_storage
|
||||
from app.utils import task_logger, log_task
|
||||
|
||||
# Import the shared Celery instance
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
from app.tasks.finalize_document_storage import finalize_document_storage
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import get_unique_filepath_with_counter, log_task_progress
|
||||
from app.utils.filename_utils import sanitize_filename
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
def unique_filepath(directory, base_filename, extension=".pdf"):
|
||||
"""
|
||||
Returns a unique filepath in the specified directory.
|
||||
If 'base_filename.pdf' exists, it will append an underscore and counter.
|
||||
"""
|
||||
candidate = os.path.join(directory, base_filename + extension)
|
||||
if not os.path.exists(candidate):
|
||||
return candidate
|
||||
counter = 1
|
||||
while True:
|
||||
candidate = os.path.join(directory, f"{base_filename}_{counter}{extension}")
|
||||
if not os.path.exists(candidate):
|
||||
return candidate
|
||||
counter += 1
|
||||
|
||||
# Directory constants - defined here to avoid hardcoded strings (BAN-B108)
|
||||
# Note: These are application-specific subdirectories within settings.workdir,
|
||||
# not system temporary directories. The workdir is a configurable path specific
|
||||
# to this application. For actual temporary file creation, tempfile module is
|
||||
# used (see line 70: tempfile.NamedTemporaryFile)
|
||||
TMP_SUBDIR = "tmp"
|
||||
PROCESSED_SUBDIR = "processed"
|
||||
|
||||
|
||||
def persist_metadata(metadata, final_pdf_path, original_file_path=None, processed_file_path=None):
|
||||
def persist_metadata(metadata, final_pdf_path):
|
||||
"""
|
||||
Saves the metadata dictionary to a JSON file with the same base name as the final PDF.
|
||||
For example, if final_pdf_path is "<workdir>/processed/MyFile.pdf",
|
||||
the metadata will be saved as "<workdir>/processed/MyFile.json".
|
||||
|
||||
Optionally augments the metadata with file path references for traceability.
|
||||
|
||||
Args:
|
||||
metadata: Dictionary of metadata to save
|
||||
final_pdf_path: Path to the final PDF file
|
||||
original_file_path: Optional path to the immutable original file
|
||||
processed_file_path: Optional path to the processed file
|
||||
|
||||
Returns:
|
||||
str: Path to the created JSON file
|
||||
"""
|
||||
base, _ = os.path.splitext(final_pdf_path)
|
||||
json_path = base + ".json"
|
||||
|
||||
# Augment metadata with file path references if provided
|
||||
metadata_with_paths = metadata.copy()
|
||||
if original_file_path:
|
||||
metadata_with_paths["original_file_path"] = original_file_path
|
||||
if processed_file_path:
|
||||
metadata_with_paths["processed_file_path"] = processed_file_path
|
||||
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(metadata_with_paths, f, ensure_ascii=False, indent=2)
|
||||
json.dump(metadata, f, ensure_ascii=False, indent=2)
|
||||
return json_path
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, metadata: dict, file_id: int = None):
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
@log_task("embed_metadata")
|
||||
def embed_metadata_into_pdf(local_file_path: str, extracted_text: str, metadata: dict):
|
||||
"""
|
||||
Embeds extracted metadata into the PDF's standard metadata fields.
|
||||
The mapping is as follows:
|
||||
@@ -75,213 +53,87 @@ def embed_metadata_into_pdf(self, local_file_path: str, extracted_text: str, met
|
||||
After processing, the file is moved to
|
||||
<workdir>/processed/<suggested_filename.pdf>
|
||||
where <suggested_filename.pdf> is derived from metadata["filename"].
|
||||
The output PDF is saved incrementally while preserving its original encryption.
|
||||
Additionally, the metadata is persisted to a JSON file with the same base name.
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting metadata embedding for: {local_file_path}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"embed_metadata_into_pdf",
|
||||
"in_progress",
|
||||
f"Embedding metadata into {os.path.basename(local_file_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Get file_id from database if not provided (fallback only, prefer passing file_id explicitly)
|
||||
if file_id is None:
|
||||
with SessionLocal() as db:
|
||||
file_record = db.query(FileRecord).filter_by(local_filename=local_file_path).first()
|
||||
if file_record:
|
||||
file_id = file_record.id
|
||||
|
||||
# Check for file existence; if not found, try the known shared tmp directory.
|
||||
if not os.path.exists(local_file_path):
|
||||
alt_path = os.path.join(settings.workdir, TMP_SUBDIR, os.path.basename(local_file_path))
|
||||
alt_path = os.path.join(settings.workdir, "tmp", os.path.basename(local_file_path))
|
||||
if os.path.exists(alt_path):
|
||||
local_file_path = alt_path
|
||||
task_logger(f"Using alternative path: {local_file_path}", step_name="embed_metadata")
|
||||
else:
|
||||
logger.error(f"[{task_id}] Local file {local_file_path} not found, cannot embed metadata.")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"embed_metadata_into_pdf",
|
||||
"failure",
|
||||
"File not found",
|
||||
file_id=file_id,
|
||||
detail=(
|
||||
f"Local file not found, cannot embed metadata.\n"
|
||||
f"Tried path: {local_file_path}\n"
|
||||
f"Also tried: {alt_path}"
|
||||
),
|
||||
)
|
||||
task_logger(f"Local file {local_file_path} not found, cannot embed metadata.",
|
||||
level="error", step_name="embed_metadata")
|
||||
return {"error": "File not found"}
|
||||
|
||||
# Work on a safe copy in a secure temporary directory
|
||||
# Work on a safe copy in /tmp
|
||||
tmp_dir = "/tmp"
|
||||
original_file = local_file_path
|
||||
# Create a temporary file with the same extension as the original
|
||||
_, ext = os.path.splitext(local_file_path)
|
||||
tmp_file = tempfile.NamedTemporaryFile(mode="wb", suffix=ext, prefix="processed_", delete=False)
|
||||
processed_file = tmp_file.name
|
||||
tmp_file.close()
|
||||
processed_file = os.path.join(tmp_dir, f"processed_{os.path.basename(local_file_path)}")
|
||||
|
||||
# Create a safe copy to work on
|
||||
shutil.copy(original_file, processed_file)
|
||||
task_logger(f"Created working copy at {processed_file}", step_name="embed_metadata")
|
||||
|
||||
try:
|
||||
logger.info(f"[{task_id}] Embedding metadata into {processed_file}...")
|
||||
log_task_progress(task_id, "modify_pdf", "in_progress", "Modifying PDF metadata", file_id=file_id)
|
||||
task_logger(f"Embedding metadata into {processed_file}", step_name="embed_metadata")
|
||||
|
||||
# Open the PDF and modify metadata
|
||||
with open(processed_file, "rb") as file:
|
||||
pdf_reader = pypdf.PdfReader(file)
|
||||
pdf_writer = pypdf.PdfWriter()
|
||||
# Open the PDF
|
||||
doc = fitz.open(processed_file)
|
||||
# Set PDF metadata using only the standard keys.
|
||||
doc.set_metadata({
|
||||
"title": metadata.get("filename", "Unknown Document"),
|
||||
"author": metadata.get("absender", "Unknown"),
|
||||
"subject": metadata.get("document_type", "Unknown"),
|
||||
"keywords": ", ".join(metadata.get("tags", []))
|
||||
})
|
||||
# Save incrementally and preserve encryption
|
||||
doc.save(processed_file, incremental=True, encryption=fitz.PDF_ENCRYPT_KEEP)
|
||||
doc.close()
|
||||
|
||||
# Copy all pages from the reader to the writer
|
||||
for page in pdf_reader.pages:
|
||||
pdf_writer.add_page(page)
|
||||
|
||||
# Set PDF metadata
|
||||
pdf_writer.add_metadata(
|
||||
{
|
||||
"/Title": metadata.get("filename", "Unknown Document"),
|
||||
"/Author": metadata.get("absender", "Unknown"),
|
||||
"/Subject": metadata.get("document_type", "Unknown"),
|
||||
"/Keywords": ", ".join(metadata.get("tags", [])),
|
||||
}
|
||||
)
|
||||
|
||||
# Write the modified PDF
|
||||
with open(processed_file, "wb") as output_file:
|
||||
pdf_writer.write(output_file)
|
||||
|
||||
logger.info(f"[{task_id}] Metadata embedded successfully in {processed_file}")
|
||||
log_task_progress(task_id, "modify_pdf", "success", "PDF metadata embedded", file_id=file_id)
|
||||
task_logger("Metadata embedded successfully", step_name="embed_metadata")
|
||||
|
||||
# Use the suggested filename from metadata; if not provided, use the original basename.
|
||||
# SECURITY: Sanitize filename to prevent path traversal vulnerabilities
|
||||
suggested_filename = metadata.get("filename", os.path.splitext(os.path.basename(local_file_path))[0])
|
||||
# Sanitize the filename to remove path separators and dangerous characters
|
||||
suggested_filename = sanitize_filename(suggested_filename)
|
||||
# Remove any extension and then add .pdf
|
||||
suggested_filename = os.path.splitext(suggested_filename)[0]
|
||||
# Define the final directory based on settings.workdir and ensure it exists.
|
||||
final_dir = os.path.join(settings.workdir, PROCESSED_SUBDIR)
|
||||
final_dir = os.path.join(settings.workdir, "processed")
|
||||
os.makedirs(final_dir, exist_ok=True)
|
||||
# Get a unique filepath in case of collisions using -0001, -0002 suffix format
|
||||
final_file_path = get_unique_filepath_with_counter(final_dir, suggested_filename, extension=".pdf")
|
||||
# Get a unique filepath in case of collisions.
|
||||
final_file_path = unique_filepath(final_dir, suggested_filename, extension=".pdf")
|
||||
|
||||
logger.info(f"[{task_id}] Moving file to: {final_file_path}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"move_to_processed",
|
||||
"in_progress",
|
||||
f"Moving to processed: {os.path.basename(final_file_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
# Move the processed file using shutil.move to handle cross-device moves.
|
||||
shutil.move(processed_file, final_file_path)
|
||||
task_logger(f"Moved processed file to {final_file_path}", step_name="embed_metadata")
|
||||
|
||||
# Ensure the temporary file is deleted if it still exists.
|
||||
if os.path.exists(processed_file):
|
||||
os.remove(processed_file)
|
||||
log_task_progress(
|
||||
task_id, "move_to_processed", "success", f"Moved to: {os.path.basename(final_file_path)}", file_id=file_id
|
||||
)
|
||||
|
||||
# Get the original_file_path from the database
|
||||
original_file_path = None
|
||||
with SessionLocal() as db:
|
||||
if file_id:
|
||||
file_record = db.query(FileRecord).filter_by(id=file_id).first()
|
||||
if file_record:
|
||||
original_file_path = file_record.original_file_path
|
||||
# Update the processed_file_path in the database
|
||||
file_record.processed_file_path = final_file_path
|
||||
# Persist extracted text and AI metadata to DB for full-text search / RAG
|
||||
file_record.ocr_text = extracted_text or None
|
||||
if metadata:
|
||||
try:
|
||||
file_record.ai_metadata = json.dumps(metadata, ensure_ascii=False)
|
||||
except Exception as json_exc:
|
||||
logger.warning(f"[{task_id}] Could not serialise ai_metadata: {json_exc}")
|
||||
file_record.document_title = (
|
||||
metadata.get("title") or metadata.get("filename") or file_record.original_filename
|
||||
)
|
||||
db.commit()
|
||||
logger.info(f"[{task_id}] Updated database with processed_file_path and search fields")
|
||||
|
||||
# Index into Meilisearch for full-text search (non-blocking, best-effort)
|
||||
try:
|
||||
from app.utils.meilisearch_client import index_document
|
||||
|
||||
index_document(file_record, extracted_text or "", metadata or {})
|
||||
except Exception as search_exc:
|
||||
logger.warning(f"[{task_id}] Meilisearch indexing failed (non-fatal): {search_exc}")
|
||||
|
||||
# Persist the metadata into a JSON file with the same base name.
|
||||
# Include file path references for traceability
|
||||
logger.info(f"[{task_id}] Persisting metadata to JSON")
|
||||
log_task_progress(task_id, "save_metadata_json", "in_progress", "Saving metadata JSON", file_id=file_id)
|
||||
json_path = persist_metadata(
|
||||
metadata, final_file_path, original_file_path=original_file_path, processed_file_path=final_file_path
|
||||
)
|
||||
logger.info(f"[{task_id}] Metadata persisted to {json_path}")
|
||||
log_task_progress(
|
||||
task_id, "save_metadata_json", "success", f"Saved: {os.path.basename(json_path)}", file_id=file_id
|
||||
)
|
||||
json_path = persist_metadata(metadata, final_file_path)
|
||||
task_logger(f"Metadata persisted to {json_path}", step_name="embed_metadata")
|
||||
|
||||
# Trigger the next step: final storage.
|
||||
logger.info(f"[{task_id}] Queueing final storage task")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"embed_metadata_into_pdf",
|
||||
"success",
|
||||
"Metadata embedded, queuing finalization",
|
||||
file_id=file_id,
|
||||
detail=(
|
||||
f"Metadata embedded into PDF successfully.\n"
|
||||
f"Original file: {original_file}\n"
|
||||
f"Final file: {final_file_path}\n"
|
||||
f"Metadata JSON: {json_path}\n"
|
||||
f"Suggested filename: {suggested_filename}.pdf"
|
||||
),
|
||||
)
|
||||
finalize_document_storage.delay(original_file, final_file_path, metadata, file_id=file_id)
|
||||
finalize_doc_task = finalize_document_storage.delay(original_file, final_file_path, metadata)
|
||||
task_logger(f"Triggered final document storage with task ID: {finalize_doc_task.id}",
|
||||
step_name="embed_metadata")
|
||||
|
||||
# After triggering final storage, delete the original file if it is in workdir/tmp.
|
||||
# SECURITY: Use pathlib for safe path validation to prevent path traversal
|
||||
workdir_tmp_path = Path(settings.workdir) / TMP_SUBDIR
|
||||
try:
|
||||
original_file_path = Path(original_file).resolve()
|
||||
workdir_tmp_resolved = workdir_tmp_path.resolve()
|
||||
|
||||
# Check if file is within workdir/tmp and exists
|
||||
if original_file_path.is_relative_to(workdir_tmp_resolved) and original_file_path.exists():
|
||||
try:
|
||||
original_file_path.unlink()
|
||||
logger.info(f"[{task_id}] Deleted original file from {original_file}")
|
||||
except Exception as e:
|
||||
logger.error(f"[{task_id}] Could not delete original file {original_file}: {e}")
|
||||
except (ValueError, OSError) as e:
|
||||
logger.error(f"[{task_id}] Error validating path for deletion {original_file}: {e}")
|
||||
workdir_tmp = os.path.join(settings.workdir, "tmp")
|
||||
if original_file.startswith(workdir_tmp) and os.path.exists(original_file):
|
||||
try:
|
||||
os.remove(original_file)
|
||||
task_logger(f"Deleted original file from {original_file}", step_name="embed_metadata")
|
||||
except Exception as e:
|
||||
task_logger(f"Could not delete original file {original_file}: {e}",
|
||||
level="warning", step_name="embed_metadata")
|
||||
|
||||
return {"file": final_file_path, "metadata_file": json_path, "status": "Metadata embedded"}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"[{task_id}] Failed to embed metadata into {processed_file}: {e}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"embed_metadata_into_pdf",
|
||||
"failure",
|
||||
f"Exception: {str(e)}",
|
||||
file_id=file_id,
|
||||
detail=(
|
||||
f"Failed to embed metadata into {processed_file}.\nOriginal file: {original_file}\nException: {str(e)}"
|
||||
),
|
||||
)
|
||||
# Clean up temporary file in case of error
|
||||
if os.path.exists(processed_file):
|
||||
try:
|
||||
os.remove(processed_file)
|
||||
logger.info(f"[{task_id}] Cleaned up temporary file {processed_file}")
|
||||
except Exception as cleanup_error:
|
||||
logger.error(f"[{task_id}] Could not clean up temporary file {processed_file}: {cleanup_error}")
|
||||
task_logger(f"Failed to embed metadata into {processed_file}: {e}",
|
||||
level="error", step_name="embed_metadata")
|
||||
return {"error": str(e)}
|
||||
|
||||
@@ -1,22 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import os
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
|
||||
from app.utils import task_logger, log_task
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
|
||||
# Import the shared Celery instance
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
from app.tasks.embed_metadata_into_pdf import embed_metadata_into_pdf
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import log_task_progress
|
||||
from app.utils.ai_provider import get_ai_provider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
import openai
|
||||
|
||||
# Initialize OpenAI client dynamically
|
||||
client = openai.OpenAI(
|
||||
api_key=settings.openai_api_key,
|
||||
base_url=settings.openai_base_url
|
||||
)
|
||||
|
||||
def extract_json_from_text(text):
|
||||
"""
|
||||
@@ -32,160 +34,91 @@ def extract_json_from_text(text):
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start != -1 and end != -1 and end > start:
|
||||
return text[start : end + 1]
|
||||
return text[start:end+1]
|
||||
return None
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def extract_metadata_with_gpt(self, filename: str, cleaned_text: str, file_id: int = None):
|
||||
"""
|
||||
Uses OpenAI to classify document metadata.
|
||||
|
||||
Args:
|
||||
filename: Can be either a basename (e.g., "file.pdf") or a full path (e.g., "/workdir/processed/file.pdf")
|
||||
cleaned_text: The extracted text from the document
|
||||
file_id: Optional file ID for tracking
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting metadata extraction for: {filename}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_metadata_with_gpt",
|
||||
"in_progress",
|
||||
f"Extracting metadata for {os.path.basename(filename)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Get file_id from database if not provided
|
||||
if file_id is None:
|
||||
tmp_dir = os.path.join(settings.workdir, "tmp")
|
||||
# Handle both basename and full path
|
||||
if os.path.isabs(filename):
|
||||
file_path = filename
|
||||
else:
|
||||
file_path = os.path.join(tmp_dir, filename)
|
||||
if os.path.exists(file_path):
|
||||
with SessionLocal() as db:
|
||||
file_record = db.query(FileRecord).filter_by(local_filename=file_path).first()
|
||||
if file_record:
|
||||
file_id = file_record.id
|
||||
|
||||
prompt = (
|
||||
"You are a specialized document analyzer trained to extract structured metadata from documents.\n"
|
||||
"Your task is to analyze the given text and return a well-structured JSON object.\n\n"
|
||||
"Extract and return the following fields:\n"
|
||||
"1. **filename**: Machine-readable filename "
|
||||
"(YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, periods, and underscores).\n"
|
||||
'2. **empfaenger**: The recipient, or "Unknown" if not found.\n'
|
||||
'3. **absender**: The sender, or "Unknown" if not found.\n'
|
||||
"4. **correspondent**: The entity or company that issued the document "
|
||||
'(shortest possible name, e.g., "Amazon" instead of "Amazon EU SARL, German branch").\n'
|
||||
"5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, "
|
||||
"Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].\n"
|
||||
"6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, "
|
||||
"Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, "
|
||||
"Private_Korrespondenz, Sonstige_Informationen].\n"
|
||||
"7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).\n"
|
||||
"8. **tags**: A list of up to 4 relevant thematic keywords.\n"
|
||||
'9. **language**: Detected document language (ISO 639-1 code, e.g., "de" or "en").\n'
|
||||
"10. **title**: A human-readable title summarizing the document content.\n"
|
||||
"11. **confidence_score**: A numeric value (0-100) indicating the confidence level "
|
||||
"of the extracted metadata.\n"
|
||||
"12. **reference_number**: Extracted invoice/order/reference number if available.\n"
|
||||
"13. **monetary_amounts**: A list of key monetary values detected in the document.\n\n"
|
||||
"### Important Rules:\n"
|
||||
"- **OCR Correction**: Assume the text has been corrected for OCR errors.\n"
|
||||
"- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.\n"
|
||||
"- **Title**: Concise, no addresses, and contains key identifying features.\n"
|
||||
"- **Date Selection**: Use the most relevant date if multiple are found.\n"
|
||||
"- **Output Language**: Maintain the document's original language.\n\n"
|
||||
f"Extracted text:\n{cleaned_text}\n\n"
|
||||
"Return only valid JSON with no additional commentary.\n"
|
||||
)
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
@log_task("extract_metadata")
|
||||
def extract_metadata_with_gpt(s3_filename: str, cleaned_text: str):
|
||||
"""Uses OpenAI to classify document metadata."""
|
||||
task_id = extract_metadata_with_gpt.request.id
|
||||
session = SessionLocal()
|
||||
try:
|
||||
logger.info(f"[{task_id}] Sending classification request for {filename}...")
|
||||
log_task_progress(task_id, "call_ai_provider", "in_progress", "Calling AI provider API", file_id=file_id)
|
||||
provider = get_ai_provider()
|
||||
model = settings.ai_model or settings.openai_model
|
||||
content = provider.chat_completion(
|
||||
task_logger(f"Starting metadata extraction for {s3_filename}",
|
||||
step_name="extract_metadata", task_id=task_id)
|
||||
|
||||
prompt = f"""
|
||||
You are a specialized document analyzer trained to extract structured metadata from documents.
|
||||
Your task is to analyze the given text and return a well-structured JSON object.
|
||||
|
||||
Extract and return the following fields:
|
||||
1. **filename**: Machine-readable filename (YYYY-MM-DD_DescriptiveTitle, use only letters, numbers, periods, and underscores).
|
||||
2. **empfaenger**: The recipient, or "Unknown" if not found.
|
||||
3. **absender**: The sender, or "Unknown" if not found.
|
||||
4. **correspondent**: The entity or company that issued the document (shortest possible name, e.g., "Amazon" instead of "Amazon EU SARL, German branch").
|
||||
5. **kommunikationsart**: One of [Behoerdlicher_Brief, Rechnung, Kontoauszug, Vertrag, Quittung, Privater_Brief, Einladung, Gewerbliche_Korrespondenz, Newsletter, Werbung, Sonstiges].
|
||||
6. **kommunikationskategorie**: One of [Amtliche_Postbehoerdliche_Dokumente, Finanz_und_Vertragsdokumente, Geschaeftliche_Kommunikation, Private_Korrespondenz, Sonstige_Informationen].
|
||||
7. **document_type**: Precise classification (e.g., Invoice, Contract, Information, Unknown).
|
||||
8. **tags**: A list of up to 4 relevant thematic keywords.
|
||||
9. **language**: Detected document language (ISO 639-1 code, e.g., "de" or "en").
|
||||
10. **title**: A human-readable title summarizing the document content.
|
||||
11. **confidence_score**: A numeric value (0-100) indicating the confidence level of the extracted metadata.
|
||||
12. **reference_number**: Extracted invoice/order/reference number if available.
|
||||
13. **monetary_amounts**: A list of key monetary values detected in the document.
|
||||
|
||||
### Important Rules:
|
||||
- **OCR Correction**: Assume the text has been corrected for OCR errors.
|
||||
- **Tagging**: Max 4 tags, avoiding generic or overly specific terms.
|
||||
- **Title**: Concise, no addresses, and contains key identifying features.
|
||||
- **Date Selection**: Use the most relevant date if multiple are found.
|
||||
- **Output Language**: Maintain the document's original language.
|
||||
|
||||
Extracted text:
|
||||
{cleaned_text}
|
||||
|
||||
Return only valid JSON with no additional commentary.
|
||||
"""
|
||||
|
||||
task_logger(f"Sending classification request for {s3_filename}", step_name="extract_metadata")
|
||||
completion = client.chat.completions.create(
|
||||
model=settings.openai_model,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are an intelligent document classifier."},
|
||||
{"role": "user", "content": prompt},
|
||||
{"role": "user", "content": prompt}
|
||||
],
|
||||
model=model,
|
||||
temperature=0,
|
||||
temperature=0
|
||||
)
|
||||
|
||||
logger.info(f"[{task_id}] Raw classification response for {filename}: {content[:200]}...")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"call_ai_provider",
|
||||
"success",
|
||||
"Received AI provider response",
|
||||
file_id=file_id,
|
||||
detail=f"Raw classification response:\n{content}",
|
||||
)
|
||||
content = completion.choices[0].message.content
|
||||
task_logger(f"Received raw classification response for {s3_filename}", step_name="extract_metadata")
|
||||
|
||||
json_text = extract_json_from_text(content)
|
||||
if not json_text:
|
||||
logger.error(f"[{task_id}] Could not find valid JSON in GPT response for {filename}.")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_metadata_with_gpt",
|
||||
"failure",
|
||||
"Invalid JSON in response",
|
||||
file_id=file_id,
|
||||
detail=f"Could not parse valid JSON from GPT response.\nRaw response:\n{content}",
|
||||
)
|
||||
task_logger(f"Could not find valid JSON in GPT response for {s3_filename}",
|
||||
level="error", step_name="extract_metadata")
|
||||
return {}
|
||||
|
||||
metadata = json.loads(json_text)
|
||||
|
||||
# SECURITY: Validate filename format from GPT to prevent path traversal
|
||||
# The prompt requests filenames with only letters, numbers, periods, and underscores
|
||||
# Enforce this constraint to prevent malicious filenames
|
||||
suggested_filename = metadata.get("filename", "")
|
||||
if suggested_filename:
|
||||
# Check if filename contains only safe characters AND explicitly check for ".."
|
||||
# Defense in depth: While the regex [\w\-\. ]+ already excludes / and \,
|
||||
# we explicitly reject ".." to guard against:
|
||||
# 1. Potential locale-specific \w behavior
|
||||
# 2. Files literally named ".." which are valid but problematic
|
||||
# 3. Future code changes that might relax the regex
|
||||
if not re.match(r"^[\w\-\. ]+$", suggested_filename) or ".." in suggested_filename:
|
||||
logger.warning(f"[{task_id}] Invalid filename format from GPT: '{suggested_filename}', using fallback")
|
||||
# Reset to empty to trigger fallback to original filename
|
||||
metadata["filename"] = ""
|
||||
|
||||
logger.info(f"[{task_id}] Extracted metadata: {metadata}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"parse_metadata",
|
||||
"success",
|
||||
f"Parsed metadata: {list(metadata.keys())}",
|
||||
file_id=file_id,
|
||||
detail=f"Extracted metadata:\n{json.dumps(metadata, ensure_ascii=False, indent=2)}",
|
||||
)
|
||||
task_logger(f"Successfully extracted metadata from {s3_filename}", step_name="extract_metadata")
|
||||
|
||||
# Trigger the next step: embedding metadata into the PDF
|
||||
# Pass the filename (can be basename or full path) so embed_metadata_into_pdf can find the file on disk
|
||||
logger.info(f"[{task_id}] Queueing metadata embedding task")
|
||||
log_task_progress(
|
||||
task_id, "extract_metadata_with_gpt", "success", "Metadata extracted, queuing embed task", file_id=file_id
|
||||
)
|
||||
embed_metadata_into_pdf.delay(filename, cleaned_text, metadata, file_id)
|
||||
embed_task = embed_metadata_into_pdf.delay(s3_filename, cleaned_text, metadata)
|
||||
task_logger(f"Triggered embed_metadata task with ID: {embed_task.id}", step_name="extract_metadata")
|
||||
|
||||
return {"s3_file": os.path.basename(filename), "metadata": metadata}
|
||||
# Update database record
|
||||
file_record = session.query(FileRecord).filter(FileRecord.local_filename.like(f'%{s3_filename}')).first()
|
||||
if file_record:
|
||||
# Since we can't store dict directly, you might want to store it as JSON string
|
||||
# or add specific columns for key metadata values
|
||||
task_logger(f"Found file record ID {file_record.id}, updating metadata", step_name="extract_metadata")
|
||||
else:
|
||||
task_logger(f"No file record found for {s3_filename}", level="warning", step_name="extract_metadata")
|
||||
|
||||
return {"file": s3_filename, "metadata": metadata}
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"[{task_id}] AI provider classification failed for {filename}: {e}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_metadata_with_gpt",
|
||||
"failure",
|
||||
f"Exception: {str(e)}",
|
||||
file_id=file_id,
|
||||
detail=f"AI provider classification failed for {filename}.\nException: {str(e)}",
|
||||
)
|
||||
task_logger(f"OpenAI classification failed for {s3_filename}: {e}",
|
||||
level="error", step_name="extract_metadata")
|
||||
return {}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@@ -1,93 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
# Import the shared Celery instance
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.celery_app import celery
|
||||
from app.utils import task_logger, log_task
|
||||
|
||||
# Import the aggregator task and validator
|
||||
from app.tasks.send_to_all import get_configured_services_from_validator, send_to_all_destinations
|
||||
# Import the aggregator task
|
||||
from app.tasks.send_to_all import send_to_all_destinations
|
||||
|
||||
# Import database and logging utils from main
|
||||
from app.utils import log_task_progress
|
||||
|
||||
# Import notification utility
|
||||
from app.utils.notification import notify_file_processed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def finalize_document_storage(self, original_file: str, processed_file: str, metadata: dict, file_id: int = None):
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
@log_task("finalize_storage")
|
||||
def finalize_document_storage(original_file: str, processed_file: str, metadata: dict):
|
||||
"""
|
||||
Final storage step after embedding metadata.
|
||||
We will now call 'send_to_all_destinations' to push the final PDF to Dropbox/Nextcloud/Paperless.
|
||||
After uploading, send a notification about the processed file.
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Finalizing document storage for {processed_file}")
|
||||
task_logger(f"Finalizing document storage for {processed_file}", step_name="finalize_storage")
|
||||
|
||||
# 1. Update Database Status (From Main)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"finalize_document_storage",
|
||||
"in_progress",
|
||||
f"Finalizing: {os.path.basename(processed_file)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
# Enqueue uploads to all destinations (Dropbox, Nextcloud, Paperless)
|
||||
send_task = send_to_all_destinations.delay(processed_file)
|
||||
|
||||
task_logger(f"Triggered send to all destinations with task ID: {send_task.id}",
|
||||
step_name="finalize_storage", status="success")
|
||||
|
||||
# Get file_id from database if not provided (fallback logic from Main)
|
||||
if file_id is None:
|
||||
with SessionLocal() as db:
|
||||
# Only as a last resort, try to find by exact match on local_filename
|
||||
tmp_path = os.path.join(settings.workdir, "tmp", os.path.basename(original_file))
|
||||
file_record = db.query(FileRecord).filter(FileRecord.local_filename == tmp_path).first()
|
||||
if file_record:
|
||||
file_id = file_record.id
|
||||
|
||||
# 2. Determine Configured Destinations (From Copilot)
|
||||
# This is needed for the notification message later
|
||||
configured_destinations = []
|
||||
try:
|
||||
configured_services = get_configured_services_from_validator()
|
||||
# Get list of service names that are configured
|
||||
for service_name, is_configured in configured_services.items():
|
||||
if is_configured:
|
||||
# Format service names for display
|
||||
display_name = service_name.replace("_", " ").title()
|
||||
configured_destinations.append(display_name)
|
||||
except Exception as e:
|
||||
logger.warning(f"[WARNING] Could not determine configured destinations: {e}")
|
||||
configured_destinations = ["configured destinations"]
|
||||
|
||||
# 3. Queue Uploads (Merged)
|
||||
# Uses Main branch signature to ensure file_id is passed, but keeps logic structure
|
||||
logger.info(f"[{task_id}] Queueing uploads to all destinations")
|
||||
log_task_progress(
|
||||
task_id, "finalize_document_storage", "success", "Queuing uploads to destinations", file_id=file_id
|
||||
)
|
||||
|
||||
# Note: send_to_all_destinations is asynchronous and queues upload tasks
|
||||
# We pass 'True' (delete_after) and 'file_id' as per Main branch requirements
|
||||
send_to_all_destinations.delay(processed_file, True, file_id)
|
||||
|
||||
# 4. Send Notification (From Copilot)
|
||||
# Note: This notification is sent after processing is complete but while uploads
|
||||
# are being queued.
|
||||
try:
|
||||
# Get file information
|
||||
file_size = os.path.getsize(processed_file) if os.path.exists(processed_file) else 0
|
||||
filename = os.path.basename(processed_file)
|
||||
|
||||
notify_file_processed(
|
||||
filename=filename, file_size=file_size, metadata=metadata, destinations=configured_destinations
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"[WARNING] Failed to send file processed notification: {e}")
|
||||
|
||||
return {"status": "Completed", "file": processed_file}
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file": processed_file,
|
||||
"send_task_id": send_task.id
|
||||
}
|
||||
|
||||
+65
-60
@@ -1,19 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import json
|
||||
import email
|
||||
import imaplib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import redis
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import redis
|
||||
from celery import shared_task
|
||||
|
||||
from app.config import settings
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
|
||||
from app.tasks.process_document import process_document # Updated import
|
||||
from app.utils.allowed_types import ALLOWED_EXTENSIONS, ALLOWED_MIME_TYPES
|
||||
from app.tasks.convert_to_pdf import convert_to_pdf # new conversion task
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -21,7 +18,7 @@ logger = logging.getLogger(__name__)
|
||||
redis_client = redis.StrictRedis.from_url(settings.redis_url, decode_responses=True)
|
||||
|
||||
LOCK_KEY = "imap_lock" # Unique key for locking
|
||||
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
|
||||
LOCK_EXPIRE = 300 # Lock expires in 5 minutes
|
||||
|
||||
# Local cache file for tracking processed emails
|
||||
CACHE_FILE = os.path.join(settings.workdir, "processed_mails.json")
|
||||
@@ -144,18 +141,20 @@ def check_and_pull_mailbox(
|
||||
)
|
||||
|
||||
|
||||
def pull_inbox(mailbox_key, host, port, username, password, use_ssl, delete_after_process):
|
||||
def pull_inbox(mailbox_key, host, port, username, password, use_ssl,
|
||||
delete_after_process):
|
||||
"""
|
||||
Connects to the IMAP inbox, fetches new unread emails from the last 3 days,
|
||||
and processes attachments while preserving the original unread status.
|
||||
|
||||
|
||||
For Gmail:
|
||||
- Attempts to select the localized All Mail folder.
|
||||
- Runs an X-GM-RAW query: "in:anywhere in:unread newer_than:3d has:attachment".
|
||||
|
||||
|
||||
For non-Gmail mailboxes, it falls back to selecting the INBOX with a SINCE/UNSEEN filter.
|
||||
"""
|
||||
logger.info("Connecting to %s at %s:%s (SSL=%s)", mailbox_key, host, port, use_ssl)
|
||||
logger.info("Connecting to %s at %s:%s (SSL=%s)",
|
||||
mailbox_key, host, port, use_ssl)
|
||||
processed_emails = load_processed_emails()
|
||||
|
||||
try:
|
||||
@@ -178,11 +177,13 @@ def pull_inbox(mailbox_key, host, port, username, password, use_ssl, delete_afte
|
||||
else:
|
||||
# For non-Gmail, select INBOX and use SINCE/UNSEEN query.
|
||||
mail.select("INBOX")
|
||||
since_date = (datetime.now(timezone.utc) - timedelta(days=3)).strftime("%d-%b-%Y")
|
||||
status, search_data = mail.search(None, f"(SINCE {since_date} UNSEEN)")
|
||||
since_date = (datetime.now(timezone.utc) - timedelta(days=3)
|
||||
).strftime("%d-%b-%Y")
|
||||
status, search_data = mail.search(None, f'(SINCE {since_date} UNSEEN)')
|
||||
|
||||
if status != "OK":
|
||||
logger.warning("Search failed on mailbox %s. Status=%s", mailbox_key, status)
|
||||
logger.warning("Search failed on mailbox %s. Status=%s",
|
||||
mailbox_key, status)
|
||||
mail.close()
|
||||
mail.logout()
|
||||
return
|
||||
@@ -193,7 +194,8 @@ def pull_inbox(mailbox_key, host, port, username, password, use_ssl, delete_afte
|
||||
for num in msg_numbers:
|
||||
status, msg_data = mail.fetch(num, "(RFC822)")
|
||||
if status != "OK":
|
||||
logger.warning("Failed to fetch message %s in %s. Status=%s", num, mailbox_key, status)
|
||||
logger.warning("Failed to fetch message %s in %s. Status=%s",
|
||||
num, mailbox_key, status)
|
||||
continue
|
||||
|
||||
raw_email = msg_data[0][1]
|
||||
@@ -211,30 +213,28 @@ def pull_inbox(mailbox_key, host, port, username, password, use_ssl, delete_afte
|
||||
# For Gmail, check if the email already has the "Ingested" label.
|
||||
if is_gmail_host:
|
||||
if email_already_has_label(mail, num, "Ingested"):
|
||||
logger.info("Skipping email %s in %s, already labeled 'Ingested'.", msg_id, mailbox_key)
|
||||
logger.info("Skipping email %s in %s, already labeled 'Ingested'.",
|
||||
msg_id, mailbox_key)
|
||||
continue
|
||||
|
||||
# Process attachments (and convert non-PDF files).
|
||||
# We call the function without assigning its return value since it is not used.
|
||||
fetch_attachments_and_enqueue(email_message)
|
||||
|
||||
if settings.imap_readonly_mode:
|
||||
logger.info("Readonly mode: skipping mailbox modifications for %s in %s", msg_id, mailbox_key)
|
||||
else:
|
||||
if is_gmail_host:
|
||||
mark_as_processed_with_star(mail, num)
|
||||
mark_as_processed_with_label(mail, num, label="Ingested")
|
||||
|
||||
if delete_after_process:
|
||||
logger.info("Deleting message %s from %s", num.decode(), mailbox_key)
|
||||
mail.store(num, "+FLAGS", "\\Deleted")
|
||||
else:
|
||||
mail.store(num, "-FLAGS", "\\Seen")
|
||||
if is_gmail_host:
|
||||
mark_as_processed_with_star(mail, num)
|
||||
mark_as_processed_with_label(mail, num, label="Ingested")
|
||||
|
||||
processed_emails[msg_id] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
save_processed_emails(processed_emails)
|
||||
|
||||
if not settings.imap_readonly_mode and delete_after_process:
|
||||
if delete_after_process:
|
||||
logger.info("Deleting message %s from %s", num.decode(), mailbox_key)
|
||||
mail.store(num, "+FLAGS", "\\Deleted")
|
||||
else:
|
||||
mail.store(num, "-FLAGS", "\\Seen")
|
||||
|
||||
if delete_after_process:
|
||||
mail.expunge()
|
||||
|
||||
mail.close()
|
||||
@@ -248,30 +248,44 @@ def pull_inbox(mailbox_key, host, port, username, password, use_ssl, delete_afte
|
||||
def fetch_attachments_and_enqueue(email_message):
|
||||
"""
|
||||
Extracts attachments from the email and processes only allowed file types.
|
||||
|
||||
Files are accepted if either:
|
||||
1. They have a MIME type from the ALLOWED_MIME_TYPES set, OR
|
||||
2. They have a '.pdf' file extension (regardless of MIME type)
|
||||
|
||||
|
||||
Allowed file types include:
|
||||
- PDF: application/pdf or *.pdf extension
|
||||
- PDF: application/pdf
|
||||
- Microsoft Office files:
|
||||
- Word: application/msword,
|
||||
- Word: application/msword,
|
||||
application/vnd.openxmlformats-officedocument.wordprocessingml.document
|
||||
- Excel: application/vnd.ms-excel,
|
||||
- Excel: application/vnd.ms-excel,
|
||||
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
|
||||
- PowerPoint: application/vnd.ms-powerpoint,
|
||||
- PowerPoint: application/vnd.ms-powerpoint,
|
||||
application/vnd.openxmlformats-officedocument.presentationml.presentation
|
||||
- Other meaningful attachments:
|
||||
- Plain text: text/plain
|
||||
- CSV: text/csv
|
||||
- Rich Text Format: application/rtf, text/rtf
|
||||
|
||||
If the attachment is a PDF (by extension or MIME type), it is enqueued for upload;
|
||||
any other allowed file is enqueued for conversion to PDF.
|
||||
|
||||
|
||||
Attachments not in this list are skipped. Common image MIME types such as
|
||||
image/jpeg, image/png, image/gif, image/bmp, image/tiff, and image/webp are
|
||||
intentionally excluded.
|
||||
|
||||
If the attachment is a PDF, it is enqueued for upload; any other allowed file
|
||||
is enqueued for conversion to PDF.
|
||||
|
||||
Returns True if at least one allowed attachment was processed.
|
||||
"""
|
||||
ALLOWED_MIME_TYPES = {
|
||||
"application/pdf",
|
||||
"application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"text/plain",
|
||||
"text/csv",
|
||||
"application/rtf",
|
||||
"text/rtf",
|
||||
}
|
||||
|
||||
has_attachment = False
|
||||
for part in email_message.walk():
|
||||
if part.get_content_maintype() == "multipart":
|
||||
@@ -281,26 +295,21 @@ def fetch_attachments_and_enqueue(email_message):
|
||||
if not filename:
|
||||
continue
|
||||
|
||||
# Check if it's a PDF file by extension, regardless of MIME type
|
||||
is_pdf_by_extension = filename.lower().endswith(".pdf")
|
||||
|
||||
mime_type = part.get_content_type()
|
||||
file_ext = os.path.splitext(filename)[1].lower()
|
||||
# Accept file if it has an allowed MIME type, an allowed extension, OR is a PDF by extension
|
||||
if mime_type not in ALLOWED_MIME_TYPES and file_ext not in ALLOWED_EXTENSIONS and not is_pdf_by_extension:
|
||||
logger.info("Skipping attachment %s with MIME type %s", filename, mime_type)
|
||||
if mime_type not in ALLOWED_MIME_TYPES:
|
||||
logger.info("Skipping attachment %s with MIME type %s",
|
||||
filename, mime_type)
|
||||
continue
|
||||
|
||||
file_path = os.path.join(settings.workdir, filename)
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(part.get_payload(decode=True))
|
||||
|
||||
# If it's a PDF by MIME type or extension, process it directly
|
||||
if mime_type == "application/pdf" or is_pdf_by_extension:
|
||||
process_document.delay(file_path)
|
||||
logger.info("Enqueued PDF for upload: %s (MIME: %s)", filename, mime_type)
|
||||
if mime_type == "application/pdf":
|
||||
process_document.delay(file_path) # Updated function call
|
||||
logger.info("Enqueued PDF for upload: %s", filename)
|
||||
elif mime_type in ALLOWED_MIME_TYPES:
|
||||
# Other allowed files are sent for conversion
|
||||
# Enqueue conversion to PDF using the Gotenberg service.
|
||||
convert_to_pdf.delay(file_path)
|
||||
logger.info("Enqueued file for conversion to PDF: %s", filename)
|
||||
|
||||
@@ -314,10 +323,6 @@ def email_already_has_label(mail, msg_id, label="Ingested"):
|
||||
Returns True if the label is found, False otherwise.
|
||||
"""
|
||||
try:
|
||||
# Convert msg_id to bytes if it's an integer
|
||||
if isinstance(msg_id, int):
|
||||
msg_id = str(msg_id).encode()
|
||||
|
||||
label_status, label_data = mail.fetch(msg_id, "(X-GM-LABELS)")
|
||||
if label_status == "OK" and label_data and len(label_data) > 0:
|
||||
raw_labels = label_data[0][1].decode("utf-8", errors="ignore")
|
||||
@@ -387,7 +392,7 @@ def find_all_mail_xlist(mail):
|
||||
Returns the folder name if found, otherwise None.
|
||||
"""
|
||||
tag = mail._new_tag().decode("ascii")
|
||||
command_str = f'{tag} XLIST "" "*"'
|
||||
command_str = f"{tag} XLIST \"\" \"*\""
|
||||
mail.send((command_str + "\r\n").encode("utf-8"))
|
||||
|
||||
all_mail_folder = None
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
"""
|
||||
Periodic task to detect and recover from stalled processing steps.
|
||||
|
||||
This task runs periodically (every minute by default) to find any processing steps
|
||||
that have been stuck in "in_progress" state for too long and mark them as failed.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.database import SessionLocal
|
||||
from app.utils.step_timeout import mark_stalled_steps_as_failed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery.task(name="app.tasks.monitor_stalled_steps.monitor_stalled_steps")
|
||||
def monitor_stalled_steps():
|
||||
"""
|
||||
Periodic task to detect and mark stalled processing steps as failed.
|
||||
|
||||
This task:
|
||||
1. Connects to the database
|
||||
2. Finds any in-progress steps that exceeded the timeout
|
||||
3. Marks them as failed with a timeout error message
|
||||
4. Logs the recovery action
|
||||
|
||||
This helps prevent files from getting stuck in "pending" state when
|
||||
processing crashes or hangs without proper error handling.
|
||||
|
||||
Scheduled to run every minute via Celery Beat.
|
||||
"""
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
stalled_count = mark_stalled_steps_as_failed(db)
|
||||
|
||||
if stalled_count > 0:
|
||||
logger.warning(
|
||||
f"[{datetime.now(timezone.utc).isoformat()}] "
|
||||
f"Recovered {stalled_count} stalled step(s). "
|
||||
f"Marked as failed due to timeout."
|
||||
)
|
||||
else:
|
||||
logger.debug(f"[{datetime.now(timezone.utc).isoformat()}] No stalled steps found.")
|
||||
|
||||
return {"recovered": stalled_count}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in monitor_stalled_steps task: {e}", exc_info=True)
|
||||
return {"error": str(e), "recovered": 0}
|
||||
+71
-499
@@ -1,554 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
import shutil
|
||||
import mimetypes
|
||||
import fitz # PyMuPDF for checking embedded text
|
||||
|
||||
import pypdf # Upgraded from PyPDF2 to fix CVE-2023-36464
|
||||
from pypdf.errors import PdfReadError
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.tasks.process_with_textract import process_with_textract
|
||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
||||
from app.celery_app import celery
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
||||
from app.tasks.process_with_ocr import process_with_ocr
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import get_unique_filepath_with_counter, hash_file, log_task_progress
|
||||
from app.utils.step_manager import initialize_file_steps
|
||||
from app.utils.text_quality import check_text_quality, detect_pdf_text_source
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from app.utils import hash_file, task_logger, log_task
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def process_document(
|
||||
self, original_local_file: str, original_filename: str = None, file_id: int = None, force_cloud_ocr: bool = False
|
||||
):
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
@log_task("process_document")
|
||||
def process_document(original_local_file: str):
|
||||
"""
|
||||
Process a document file and trigger appropriate text extraction.
|
||||
|
||||
Args:
|
||||
original_local_file: Path to the file on disk
|
||||
original_filename: Optional original filename (if different from path basename)
|
||||
file_id: Optional existing file record ID. When provided, skips duplicate
|
||||
detection and reuses the existing record (used for reprocessing).
|
||||
force_cloud_ocr: If True, forces Azure Document Intelligence OCR processing
|
||||
regardless of embedded text quality. Used for re-processing.
|
||||
|
||||
Steps:
|
||||
1. Check if we have a FileRecord entry (via SHA-256 hash). If found, skip re-processing.
|
||||
(Skipped when file_id is provided for reprocessing.)
|
||||
2. If not found, insert a new DB row and continue with the pipeline:
|
||||
- Save immutable copy to /workdir/original
|
||||
- Copy file to /workdir/tmp for processing
|
||||
- Copy file to /workdir/tmp
|
||||
- Check for embedded text. If present, run local GPT extraction
|
||||
- Otherwise, queue Azure Document Intelligence processing
|
||||
3. If force_cloud_ocr is True, skip local text extraction and use cloud OCR
|
||||
- Otherwise, queue Textract-based OCR
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting document processing: {original_local_file}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_document",
|
||||
"in_progress",
|
||||
f"Processing file: {original_local_file}",
|
||||
)
|
||||
task_id = process_document.request.id
|
||||
task_logger(f"Processing {original_local_file}", step_name="process_document", task_id=task_id, file_path=original_local_file)
|
||||
|
||||
if not os.path.exists(original_local_file):
|
||||
logger.error(f"[{task_id}] File {original_local_file} not found.")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_document",
|
||||
"failure",
|
||||
"File not found",
|
||||
detail=f"File not found on disk: {original_local_file}",
|
||||
)
|
||||
task_logger(f"File {original_local_file} not found.", level="error", step_name="process_document", task_id=task_id)
|
||||
return {"error": "File not found"}
|
||||
|
||||
# 0. Check for duplicate files (if enabled)
|
||||
if settings.enable_deduplication:
|
||||
logger.info(f"[{task_id}] Computing file hash for deduplication check...")
|
||||
log_task_progress(task_id, "check_for_duplicates", "in_progress", "Computing file hash for deduplication")
|
||||
filehash = hash_file(original_local_file)
|
||||
else:
|
||||
logger.info(f"[{task_id}] Computing file hash (deduplication disabled)...")
|
||||
filehash = hash_file(original_local_file)
|
||||
|
||||
# Use provided original_filename or fall back to basename of path
|
||||
if original_filename is None:
|
||||
original_filename = os.path.basename(original_local_file)
|
||||
# 0. Compute the file hash and check for duplicates
|
||||
task_logger(f"Computing hash for {original_local_file}", step_name="compute_hash", task_id=task_id)
|
||||
filehash = hash_file(original_local_file)
|
||||
original_filename = os.path.basename(original_local_file)
|
||||
file_size = os.path.getsize(original_local_file)
|
||||
mime_type, _ = mimetypes.guess_type(original_local_file)
|
||||
if not mime_type:
|
||||
mime_type = "application/octet-stream"
|
||||
|
||||
logger.info(f"[{task_id}] File hash: {filehash[:10]}..., Size: {file_size} bytes, MIME: {mime_type}")
|
||||
|
||||
# Log deduplication step result (only if enabled)
|
||||
if settings.enable_deduplication:
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_for_duplicates",
|
||||
"in_progress",
|
||||
f"Hash: {filehash[:10]}..., checking for duplicates",
|
||||
)
|
||||
|
||||
# Acquire DB session in the task
|
||||
with SessionLocal() as db:
|
||||
# When file_id is provided, we are reprocessing an existing file.
|
||||
# Skip the duplicate check and reuse the existing record.
|
||||
if file_id is not None:
|
||||
existing_record = db.query(FileRecord).filter_by(id=file_id).one_or_none()
|
||||
if existing_record is None:
|
||||
logger.error(f"[{task_id}] File record with ID {file_id} not found for reprocessing.")
|
||||
log_task_progress(task_id, "process_document", "failure", "File record not found", file_id=file_id)
|
||||
return {"error": "File record not found", "file_id": file_id}
|
||||
logger.info(f"[{task_id}] Reprocessing existing file record ID: {file_id}, skipping duplicate check.")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_document",
|
||||
"in_progress",
|
||||
f"Reprocessing file record ID: {file_id}",
|
||||
file_id=file_id,
|
||||
)
|
||||
new_record = existing_record
|
||||
# Keep all FileRecord operations within this session scope
|
||||
task_logger(f"Checking for duplicate files", step_name="check_duplicates", task_id=task_id)
|
||||
existing_record = db.query(FileRecord).filter(FileRecord.filehash == filehash).one_or_none()
|
||||
if existing_record:
|
||||
task_logger(f"Duplicate file detected (hash={filehash[:10]}...). Skipping processing.",
|
||||
step_name="process_document", task_id=task_id, file_id=existing_record.id, status="success")
|
||||
return {
|
||||
"status": "duplicate_file",
|
||||
"file_id": existing_record.id,
|
||||
"detail": "File already processed."
|
||||
}
|
||||
else:
|
||||
# Check for duplicate only if this is a new file (not reprocessing)
|
||||
# IMPORTANT: Only consider it a duplicate if it matches a DIFFERENT file
|
||||
existing = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.filehash == filehash, FileRecord.is_duplicate.is_(False))
|
||||
.order_by(FileRecord.created_at.asc())
|
||||
.first()
|
||||
)
|
||||
if existing is None:
|
||||
existing = (
|
||||
db.query(FileRecord).filter(FileRecord.filehash == filehash).order_by(FileRecord.id.asc()).first()
|
||||
)
|
||||
|
||||
# A file is only a duplicate if it matches a different file's hash
|
||||
# (not its own hash when reprocessing)
|
||||
if existing and existing.id != file_id and settings.enable_deduplication:
|
||||
logger.info(f"[{task_id}] Duplicate file detected (hash={filehash[:10]}...) Skipping processing.")
|
||||
duplicate_record = FileRecord(
|
||||
filehash=filehash,
|
||||
original_filename=original_filename,
|
||||
local_filename="",
|
||||
file_size=file_size,
|
||||
mime_type=mime_type,
|
||||
is_duplicate=True,
|
||||
duplicate_of_id=existing.id,
|
||||
)
|
||||
db.add(duplicate_record)
|
||||
db.commit()
|
||||
db.refresh(duplicate_record)
|
||||
|
||||
if settings.enable_deduplication and settings.show_deduplication_step:
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_for_duplicates",
|
||||
"success",
|
||||
f"Duplicate detected - matching file ID {existing.id}",
|
||||
file_id=duplicate_record.id,
|
||||
detail=(
|
||||
f"Duplicate file detected.\n"
|
||||
f"File hash: {filehash}\n"
|
||||
f"Original file record ID: {existing.id}\n"
|
||||
f"This file record ID: {duplicate_record.id}\n"
|
||||
f"Original filename: {original_filename}"
|
||||
),
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_document",
|
||||
"success",
|
||||
"Duplicate file detected, skipping",
|
||||
file_id=duplicate_record.id,
|
||||
detail=(
|
||||
f"Duplicate file detected.\n"
|
||||
f"File hash: {filehash}\n"
|
||||
f"Original file record ID: {existing.id}\n"
|
||||
f"Original filename: {original_filename}"
|
||||
),
|
||||
)
|
||||
return {
|
||||
"status": "duplicate_file",
|
||||
"file_id": duplicate_record.id,
|
||||
"original_file_id": existing.id,
|
||||
"detail": "File already processed.",
|
||||
}
|
||||
|
||||
# Not a duplicate (or deduplication disabled) -> insert a new record
|
||||
logger.info(f"[{task_id}] Creating new file record in database")
|
||||
log_task_progress(task_id, "create_file_record", "in_progress", "Creating file record")
|
||||
task_logger(f"Creating file record for {original_local_file}", step_name="create_file_record", task_id=task_id)
|
||||
new_record = FileRecord(
|
||||
filehash=filehash,
|
||||
original_filename=original_filename,
|
||||
local_filename="", # Will fill in after we move it
|
||||
file_size=file_size,
|
||||
mime_type=mime_type,
|
||||
is_duplicate=False,
|
||||
)
|
||||
db.add(new_record)
|
||||
db.commit()
|
||||
db.refresh(new_record)
|
||||
logger.info(f"[{task_id}] File record created with ID: {new_record.id}")
|
||||
# Pre-initialize all expected processing steps as "pending" so that
|
||||
# status tracking reflects the complete pipeline from the start.
|
||||
initialize_file_steps(db, new_record.id)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"create_file_record",
|
||||
"success",
|
||||
f"File record ID: {new_record.id}",
|
||||
file_id=new_record.id,
|
||||
)
|
||||
# Update the check_for_duplicates step now that file_id is available.
|
||||
# This must happen after initialize_file_steps() which creates the
|
||||
# step as "pending". The dedup check already passed at this point.
|
||||
if settings.enable_deduplication:
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_for_duplicates",
|
||||
"success",
|
||||
"New file - no duplicates found",
|
||||
file_id=new_record.id,
|
||||
)
|
||||
|
||||
# 1. Generate a UUID-based filename for storage
|
||||
file_ext = os.path.splitext(original_local_file)[1]
|
||||
file_uuid = str(uuid.uuid4())
|
||||
new_filename = f"{file_uuid}{file_ext}"
|
||||
# 1. Generate a UUID-based filename and place it in /workdir/tmp
|
||||
task_logger(f"Copying to workdir", step_name="copy_to_workdir", task_id=task_id, file_id=new_record.id)
|
||||
file_ext = os.path.splitext(original_local_file)[1]
|
||||
file_uuid = str(uuid.uuid4())
|
||||
new_filename = f"{file_uuid}{file_ext}"
|
||||
|
||||
# 2. Save immutable copy to /workdir/original (only for new files, not reprocessing)
|
||||
# For reprocessing, the original_file_path should already exist in the database
|
||||
if file_id is None: # New file - save original copy
|
||||
# This copy serves as the permanent, untouched reference of the ingested file
|
||||
original_dir = os.path.join(settings.workdir, "original")
|
||||
os.makedirs(original_dir, exist_ok=True)
|
||||
tmp_dir = os.path.join(settings.workdir, "tmp")
|
||||
os.makedirs(tmp_dir, exist_ok=True)
|
||||
new_local_path = os.path.join(tmp_dir, new_filename)
|
||||
|
||||
# Use collision-resistant naming with -0001, -0002 suffixes
|
||||
base_name = os.path.splitext(new_filename)[0]
|
||||
original_file_path = get_unique_filepath_with_counter(original_dir, base_name, file_ext)
|
||||
# Copy the file instead of moving it
|
||||
shutil.copy(original_local_file, new_local_path)
|
||||
|
||||
logger.info(f"[{task_id}] Saving immutable original to: {original_file_path}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"save_original",
|
||||
"in_progress",
|
||||
f"Saving original to {os.path.basename(original_file_path)}",
|
||||
file_id=new_record.id,
|
||||
)
|
||||
shutil.copy(original_local_file, original_file_path)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"save_original",
|
||||
"success",
|
||||
f"Original saved: {os.path.basename(original_file_path)}",
|
||||
file_id=new_record.id,
|
||||
)
|
||||
# Update the DB with final local filename
|
||||
new_record.local_filename = new_local_path
|
||||
db.commit()
|
||||
|
||||
# Update the DB with original_file_path
|
||||
new_record.original_file_path = original_file_path
|
||||
else:
|
||||
# Reprocessing - original should already exist
|
||||
logger.info(f"[{task_id}] Reprocessing: original file already saved at {new_record.original_file_path}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"save_original",
|
||||
"success",
|
||||
"Reprocessing: using existing original",
|
||||
file_id=new_record.id,
|
||||
)
|
||||
|
||||
# 3. Copy to /workdir/tmp for processing
|
||||
tmp_dir = os.path.join(settings.workdir, "tmp")
|
||||
os.makedirs(tmp_dir, exist_ok=True)
|
||||
new_local_path = os.path.join(tmp_dir, new_filename)
|
||||
|
||||
logger.info(f"[{task_id}] Copying file to processing area: {new_local_path}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"copy_file",
|
||||
"in_progress",
|
||||
f"Copying file to {new_filename}",
|
||||
file_id=new_record.id,
|
||||
)
|
||||
# Copy the file instead of moving it
|
||||
shutil.copy(original_local_file, new_local_path)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"copy_file",
|
||||
"success",
|
||||
f"File copied to {new_filename}",
|
||||
file_id=new_record.id,
|
||||
)
|
||||
|
||||
# Update the DB with local_filename
|
||||
new_record.local_filename = new_local_path
|
||||
db.commit()
|
||||
|
||||
# Store file_id before session closes to avoid DetachedInstanceError
|
||||
file_id = new_record.id
|
||||
# Perform all further interactions with existing_record/new_record here
|
||||
|
||||
# 2. Check for embedded text (outside the DB session to avoid long open transactions)
|
||||
# Skip local text extraction if force_cloud_ocr is True
|
||||
if force_cloud_ocr:
|
||||
logger.info(f"[{task_id}] Force Cloud OCR requested, skipping embedded text check")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_text",
|
||||
"success",
|
||||
"Force Cloud OCR requested, queuing OCR",
|
||||
file_id=file_id,
|
||||
)
|
||||
# Mark local text extraction as skipped since force_cloud_ocr was requested
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_text",
|
||||
"skipped",
|
||||
"Force cloud OCR requested, skipping local extraction",
|
||||
file_id=file_id,
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_document",
|
||||
"success",
|
||||
"Queued for forced OCR processing",
|
||||
file_id=file_id,
|
||||
)
|
||||
process_with_ocr.delay(new_filename, file_id)
|
||||
return {"file": new_local_path, "status": "Queued for forced OCR", "file_id": file_id}
|
||||
|
||||
# If the file is not a PDF, skip embedded text check and convert to PDF first
|
||||
is_pdf = mime_type == "application/pdf" or os.path.splitext(new_local_path)[1].lower() == ".pdf"
|
||||
if not is_pdf:
|
||||
logger.info(f"[{task_id}] Non-PDF file detected, queuing PDF conversion before OCR")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_text",
|
||||
"skipped",
|
||||
"Non-PDF file detected, converting to PDF",
|
||||
file_id=file_id,
|
||||
)
|
||||
# Mark local text extraction as skipped since the file needs PDF conversion first
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_text",
|
||||
"skipped",
|
||||
"Non-PDF file, text extraction deferred to OCR after conversion",
|
||||
file_id=file_id,
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_document",
|
||||
"success",
|
||||
"Queued for PDF conversion",
|
||||
file_id=file_id,
|
||||
)
|
||||
celery.send_task(
|
||||
"app.tasks.convert_to_pdf.convert_to_pdf",
|
||||
args=[new_local_path, original_filename],
|
||||
)
|
||||
return {"file": new_local_path, "status": "Queued for PDF conversion", "file_id": file_id}
|
||||
|
||||
logger.info(f"[{task_id}] Checking for embedded text in PDF")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_text",
|
||||
"in_progress",
|
||||
"Checking for embedded text",
|
||||
file_id=file_id,
|
||||
)
|
||||
try:
|
||||
with open(new_local_path, "rb") as file:
|
||||
pdf_reader = pypdf.PdfReader(file)
|
||||
has_text = False
|
||||
for page in pdf_reader.pages:
|
||||
if page.extract_text().strip():
|
||||
has_text = True
|
||||
break
|
||||
except PdfReadError as exc:
|
||||
logger.warning(f"[{task_id}] PDF read error during embedded text check: {exc}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_text",
|
||||
"in_progress",
|
||||
"PDF read error, retrying embedded text check",
|
||||
file_id=file_id,
|
||||
detail=str(exc),
|
||||
)
|
||||
raise self.retry(
|
||||
exc=exc,
|
||||
countdown=10,
|
||||
kwargs={
|
||||
"original_local_file": original_local_file,
|
||||
"original_filename": original_filename,
|
||||
"file_id": file_id,
|
||||
"force_cloud_ocr": force_cloud_ocr,
|
||||
},
|
||||
)
|
||||
task_logger(f"Checking for embedded text", step_name="check_embedded_text", task_id=task_id, file_id=new_record.id)
|
||||
pdf_doc = fitz.open(new_local_path)
|
||||
has_text = any(page.get_text() for page in pdf_doc)
|
||||
pdf_doc.close()
|
||||
|
||||
if has_text:
|
||||
logger.info(f"[{task_id}] PDF {original_local_file} contains embedded text. Processing locally.")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_text",
|
||||
"success",
|
||||
"Embedded text found, extracting locally",
|
||||
file_id=file_id,
|
||||
)
|
||||
task_logger(f"PDF {original_local_file} contains embedded text. Processing locally.",
|
||||
step_name="process_document", task_id=task_id, file_id=new_record.id)
|
||||
|
||||
# Extract text locally
|
||||
logger.info(f"[{task_id}] Extracting text from PDF")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_text",
|
||||
"in_progress",
|
||||
"Extracting text locally",
|
||||
file_id=file_id,
|
||||
)
|
||||
extracted_text = ""
|
||||
with open(new_local_path, "rb") as file:
|
||||
pdf_reader = pypdf.PdfReader(file)
|
||||
for page in pdf_reader.pages:
|
||||
extracted_text += page.extract_text() + "\n"
|
||||
|
||||
logger.info(f"[{task_id}] Extracted {len(extracted_text)} characters")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_text",
|
||||
"success",
|
||||
f"Extracted {len(extracted_text)} characters",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# AI-based text quality check
|
||||
# Digitally-created PDFs are always trusted; OCR-sourced or unknown
|
||||
# PDFs are validated. Poor-quality text triggers automatic re-OCR.
|
||||
# ----------------------------------------------------------------
|
||||
if settings.enable_text_quality_check:
|
||||
text_source = detect_pdf_text_source(new_local_path)
|
||||
logger.info(f"[{task_id}] Detected PDF text source: {text_source.value}")
|
||||
|
||||
quality_result = check_text_quality(extracted_text, text_source)
|
||||
logger.info(
|
||||
f"[{task_id}] Text quality check result: "
|
||||
f"good={quality_result.is_good_quality}, score={quality_result.quality_score}, "
|
||||
f"source={quality_result.text_source.value}, feedback={quality_result.feedback!r}"
|
||||
)
|
||||
|
||||
# Persist the quality score immediately so it's available for filtering
|
||||
# even if the file is later sent to OCR for re-processing.
|
||||
with SessionLocal() as _db:
|
||||
_rec = _db.query(FileRecord).filter_by(id=file_id).first()
|
||||
if _rec:
|
||||
_rec.ocr_quality_score = quality_result.quality_score
|
||||
_db.commit()
|
||||
|
||||
if not quality_result.is_good_quality:
|
||||
# Poor quality: discard embedded text and re-OCR instead.
|
||||
# Pass the original embedded text so the OCR task can compare
|
||||
# its result against the original and keep the better version.
|
||||
issues_str = ", ".join(quality_result.issues) if quality_result.issues else "unspecified"
|
||||
detail_msg = (
|
||||
f"Text quality check FAILED – score={quality_result.quality_score}/100, "
|
||||
f"source={quality_result.text_source.value}, issues=[{issues_str}].\n"
|
||||
f"AI feedback: {quality_result.feedback}\n"
|
||||
f"Embedded text will be compared with fresh OCR output; best version will be used."
|
||||
)
|
||||
logger.warning(f"[{task_id}] {detail_msg}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_text_quality",
|
||||
"failure",
|
||||
f"Poor quality text (score={quality_result.quality_score}/100); queuing OCR for comparison",
|
||||
file_id=file_id,
|
||||
detail=detail_msg,
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_document",
|
||||
"success",
|
||||
"Queued for OCR (text quality too low)",
|
||||
file_id=file_id,
|
||||
)
|
||||
process_with_ocr.delay(new_filename, file_id, extracted_text)
|
||||
return {
|
||||
"file": new_local_path,
|
||||
"status": "Queued for OCR (poor embedded text quality)",
|
||||
"file_id": file_id,
|
||||
}
|
||||
|
||||
# Good quality: record the result and proceed with local extraction.
|
||||
detail_msg = (
|
||||
f"Text quality check PASSED – score={quality_result.quality_score}/100, "
|
||||
f"source={quality_result.text_source.value}.\n"
|
||||
f"AI feedback: {quality_result.feedback}"
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_text_quality",
|
||||
"success",
|
||||
f"Text quality OK (score={quality_result.quality_score}/100)",
|
||||
file_id=file_id,
|
||||
detail=detail_msg,
|
||||
)
|
||||
|
||||
# Mark OCR as skipped since we extracted text locally
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_with_ocr",
|
||||
"skipped",
|
||||
"Local text extraction succeeded, OCR not needed",
|
||||
file_id=file_id,
|
||||
)
|
||||
task_logger(f"Extracting text locally", step_name="extract_text_locally", task_id=task_id, file_id=new_record.id)
|
||||
pdf_doc = fitz.open(new_local_path)
|
||||
for page in pdf_doc:
|
||||
extracted_text += page.get_text("text") + "\n"
|
||||
pdf_doc.close()
|
||||
|
||||
# Call metadata extraction directly
|
||||
logger.info(f"[{task_id}] Queueing metadata extraction")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_document",
|
||||
"success",
|
||||
"Queued for metadata extraction",
|
||||
file_id=file_id,
|
||||
)
|
||||
extract_metadata_with_gpt.delay(new_filename, extracted_text, file_id)
|
||||
return {
|
||||
"file": new_local_path,
|
||||
"status": "Text extracted locally",
|
||||
"file_id": file_id,
|
||||
}
|
||||
task_logger(f"Text extracted locally. Queuing for metadata extraction.",
|
||||
step_name="process_document", task_id=task_id, file_id=new_record.id, status="success")
|
||||
metadata_task = extract_metadata_with_gpt.delay(new_filename, extracted_text)
|
||||
task_logger(f"Triggered metadata extraction task: {metadata_task.id}",
|
||||
step_name="process_document", task_id=task_id)
|
||||
|
||||
return {"file": new_local_path, "status": "Text extracted locally", "file_id": new_record.id}
|
||||
|
||||
# 3. If no embedded text, queue OCR processing
|
||||
logger.info(f"[{task_id}] No embedded text found. Queueing OCR processing")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"check_text",
|
||||
"success",
|
||||
"No embedded text, queuing OCR",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Mark local text extraction as skipped since we're using cloud OCR
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"extract_text",
|
||||
"skipped",
|
||||
"No embedded text, using OCR instead",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_document",
|
||||
"success",
|
||||
"Queued for OCR processing",
|
||||
file_id=file_id,
|
||||
)
|
||||
process_with_ocr.delay(new_filename, file_id)
|
||||
return {"file": new_local_path, "status": "Queued for OCR", "file_id": file_id}
|
||||
# 3. If no embedded text, queue Textract processing
|
||||
task_logger(f"No embedded text found. Queuing for OCR.",
|
||||
step_name="process_document", task_id=task_id, file_id=new_record.id, status="success")
|
||||
ocr_task = process_with_textract.delay(new_filename)
|
||||
task_logger(f"Triggered OCR task: {ocr_task.id}", step_name="process_document", task_id=task_id)
|
||||
|
||||
return {"file": new_local_path, "status": "Queued for OCR", "file_id": new_record.id}
|
||||
|
||||
@@ -1,224 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
import azure.core.exceptions
|
||||
import pypdf # Upgraded from PyPDF2 to fix CVE-2023-36464
|
||||
from azure.ai.documentintelligence import DocumentIntelligenceClient
|
||||
from azure.ai.documentintelligence.models import AnalyzeOutputOption, AnalyzeResult
|
||||
from azure.core.credentials import AzureKeyCredential
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.tasks.rotate_pdf_pages import rotate_pdf_pages
|
||||
from app.utils import log_task_progress
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Initialize Azure Document Intelligence client with error handling
|
||||
try:
|
||||
document_intelligence_client = DocumentIntelligenceClient(
|
||||
endpoint=settings.azure_endpoint, credential=AzureKeyCredential(settings.azure_ai_key)
|
||||
)
|
||||
logger.info("Azure Document Intelligence client initialized successfully")
|
||||
except (ValueError, azure.core.exceptions.ClientAuthenticationError) as e:
|
||||
logger.error(f"Failed to initialize Azure Document Intelligence client: {e}")
|
||||
document_intelligence_client = None
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error initializing Azure Document Intelligence client: {e}")
|
||||
document_intelligence_client = None
|
||||
|
||||
# Azure Document Intelligence service limits for Standard S0 tier
|
||||
AZURE_DOC_INTELLIGENCE_LIMITS = {
|
||||
"max_file_size_bytes": 500 * 1024 * 1024, # 500 MB
|
||||
"max_pages": 2000,
|
||||
}
|
||||
|
||||
|
||||
def get_pdf_page_count(file_path):
|
||||
"""Get the number of pages in a PDF file."""
|
||||
try:
|
||||
with open(file_path, "rb") as file:
|
||||
pdf_reader = pypdf.PdfReader(file)
|
||||
return len(pdf_reader.pages)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting PDF page count: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def check_page_rotation(result, filename, task_id=None):
|
||||
"""
|
||||
Checks if pages in the document are rotated and logs the rotation information.
|
||||
|
||||
Args:
|
||||
result: The AnalyzeResult from Azure Document Intelligence API
|
||||
filename: The name of the file being processed
|
||||
task_id: Optional Celery task ID for log prefixing
|
||||
|
||||
Returns:
|
||||
dict: Dictionary mapping page indices (integers) to rotation angles
|
||||
"""
|
||||
prefix = f"[{task_id}] " if task_id else ""
|
||||
logger.info(f"{prefix}Checking rotation for document: {filename}")
|
||||
rotation_data = {}
|
||||
|
||||
if not hasattr(result, "pages") or not result.pages:
|
||||
logger.warning(f"{prefix}No page information available for rotation check: {filename}")
|
||||
return rotation_data
|
||||
|
||||
for i, page in enumerate(result.pages):
|
||||
if hasattr(page, "angle"):
|
||||
rotation_angle = page.angle
|
||||
if rotation_angle != 0:
|
||||
logger.info(f"{prefix}Page {i + 1} is rotated by {rotation_angle} degrees")
|
||||
# Store page index as integer, not string
|
||||
rotation_data[i] = rotation_angle
|
||||
else:
|
||||
logger.info(f"{prefix}Page {i + 1} has no rotation (0 degrees)")
|
||||
else:
|
||||
logger.info(f"{prefix}Page {i + 1} rotation information not available")
|
||||
|
||||
return rotation_data
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def process_with_azure_document_intelligence(self, filename: str, file_id: int = None):
|
||||
"""
|
||||
Processes a PDF document using Azure Document Intelligence and overlays OCR text onto
|
||||
the local temporary file (stored under <workdir>/tmp).
|
||||
|
||||
Steps:
|
||||
0. Verify the file meets Azure Document Intelligence service limits
|
||||
1. Uploads the document for OCR using Azure Document Intelligence.
|
||||
2. Retrieves the processed PDF with embedded text.
|
||||
3. Saves the OCR-processed PDF locally in the same location as before.
|
||||
4. Checks for page rotation and triggers page rotation if needed.
|
||||
5. Triggers downstream metadata extraction.
|
||||
|
||||
Args:
|
||||
filename: Name of the file to process
|
||||
file_id: Optional file ID to pass through to subsequent tasks
|
||||
"""
|
||||
task_id = self.request.id
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_with_azure_document_intelligence",
|
||||
"in_progress",
|
||||
f"Starting OCR for {filename}",
|
||||
file_id=file_id,
|
||||
)
|
||||
try:
|
||||
tmp_file_path = os.path.join(settings.workdir, "tmp", filename)
|
||||
if not os.path.exists(tmp_file_path):
|
||||
raise FileNotFoundError(f"Local file not found: {tmp_file_path}")
|
||||
|
||||
# Check file size against service limits
|
||||
file_size = os.path.getsize(tmp_file_path)
|
||||
if file_size > AZURE_DOC_INTELLIGENCE_LIMITS["max_file_size_bytes"]:
|
||||
error_msg = (
|
||||
f"File size ({file_size / (1024 * 1024):.2f} MB) exceeds Azure Document Intelligence limit of 500 MB"
|
||||
)
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"validate_file",
|
||||
"failure",
|
||||
f"File too large: {filename}",
|
||||
file_id=file_id,
|
||||
detail=error_msg,
|
||||
)
|
||||
return {"error": error_msg, "file": filename, "status": "Failed - Size limit exceeded"}
|
||||
|
||||
# For PDF files, check page count against service limits
|
||||
# "Fail open" approach: only reject if we're sure it exceeds the limit
|
||||
if filename.lower().endswith(".pdf"):
|
||||
page_count = get_pdf_page_count(tmp_file_path)
|
||||
if page_count is not None and page_count > AZURE_DOC_INTELLIGENCE_LIMITS["max_pages"]:
|
||||
error_msg = f"PDF page count ({page_count}) exceeds Azure Document Intelligence limit of 2000 pages"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"validate_file",
|
||||
"failure",
|
||||
f"Too many pages: {filename}",
|
||||
file_id=file_id,
|
||||
detail=error_msg,
|
||||
)
|
||||
return {"error": error_msg, "file": filename, "status": "Failed - Page limit exceeded"}
|
||||
if page_count is None:
|
||||
logger.warning(
|
||||
f"[{task_id}] Could not determine page count for {filename}, proceeding with processing anyway"
|
||||
)
|
||||
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"validate_file",
|
||||
"success",
|
||||
f"File validation passed for {filename}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
logger.info(f"[{task_id}] Processing {filename} with Azure Document Intelligence OCR.")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"call_azure_ocr",
|
||||
"in_progress",
|
||||
f"Sending {filename} to Azure Document Intelligence",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Open and send the document for processing
|
||||
with open(tmp_file_path, "rb") as f:
|
||||
poller = document_intelligence_client.begin_analyze_document(
|
||||
"prebuilt-read", body=f, output=[AnalyzeOutputOption.PDF]
|
||||
)
|
||||
result: AnalyzeResult = poller.result()
|
||||
operation_id = poller.details["operation_id"]
|
||||
|
||||
# Check and log page rotation information
|
||||
rotation_data = check_page_rotation(result, filename, task_id=task_id)
|
||||
|
||||
# Retrieve the processed searchable PDF
|
||||
response = document_intelligence_client.get_analyze_result_pdf(model_id=result.model_id, result_id=operation_id)
|
||||
searchable_pdf_path = tmp_file_path # Overwrite the original PDF location
|
||||
with open(searchable_pdf_path, "wb") as writer:
|
||||
writer.writelines(response)
|
||||
logger.info(f"[{task_id}] Searchable PDF saved at: {searchable_pdf_path}")
|
||||
|
||||
# Extract raw text content from the result
|
||||
extracted_text = result.content if result.content else ""
|
||||
logger.info(f"[{task_id}] Extracted text for {filename}: {len(extracted_text)} characters")
|
||||
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"call_azure_ocr",
|
||||
"success",
|
||||
f"Azure OCR completed for {filename}",
|
||||
file_id=file_id,
|
||||
detail=f"Extracted {len(extracted_text)} characters, {len(rotation_data)} rotated pages detected",
|
||||
)
|
||||
|
||||
# Trigger page rotation task if rotation is detected, otherwise proceed to metadata extraction
|
||||
rotate_pdf_pages.delay(filename, extracted_text, rotation_data, file_id)
|
||||
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_with_azure_document_intelligence",
|
||||
"success",
|
||||
f"OCR processing complete for {filename}",
|
||||
file_id=file_id,
|
||||
detail=f"Searchable PDF saved, {len(extracted_text)} characters extracted",
|
||||
)
|
||||
|
||||
return {"file": filename, "searchable_pdf": searchable_pdf_path, "cleaned_text": extracted_text}
|
||||
except Exception as e:
|
||||
logger.error(f"[{task_id}] Error processing {filename} with Azure Document Intelligence: {e}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_with_azure_document_intelligence",
|
||||
"failure",
|
||||
f"OCR failed for {filename}",
|
||||
file_id=file_id,
|
||||
detail=str(e),
|
||||
)
|
||||
raise
|
||||
@@ -1,289 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Unified OCR processing task for DocuElevate.
|
||||
|
||||
This task replaces the single-provider ``process_with_azure_document_intelligence``
|
||||
task with a multi-engine OCR pipeline that:
|
||||
|
||||
1. Runs every OCR provider listed in ``OCR_PROVIDERS`` (default: ``azure``).
|
||||
2. Merges/cross-checks the results using the configured AI model when more
|
||||
than one provider is active (see ``OCR_MERGE_STRATEGY``).
|
||||
3. Writes the best searchable PDF back to the working directory.
|
||||
4. Optionally compares the OCR output against the original embedded text
|
||||
(passed as *original_text*) using a head-to-head AI review and keeps the
|
||||
higher-quality text for downstream processing.
|
||||
5. Hands off to the page-rotation and metadata-extraction pipeline exactly as
|
||||
the legacy Azure task did.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.tasks.rotate_pdf_pages import rotate_pdf_pages
|
||||
from app.utils import log_task_progress
|
||||
from app.utils.ocr_provider import OCRResult, embed_text_layer, get_ocr_providers, merge_ocr_results
|
||||
from app.utils.text_quality import TextSource, check_text_quality, compare_text_quality
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def process_with_ocr(self, filename: str, file_id: Optional[int] = None, original_text: Optional[str] = None):
|
||||
"""Run the configured OCR providers on *filename* and continue the pipeline.
|
||||
|
||||
When multiple OCR providers are configured the results are merged using the
|
||||
AI model (or a simpler strategy controlled by ``OCR_MERGE_STRATEGY``).
|
||||
|
||||
If *original_text* is provided (the original embedded text that failed the
|
||||
quality check), the OCR result is compared against it using a head-to-head
|
||||
AI review. The higher-quality text is passed to downstream tasks.
|
||||
|
||||
Args:
|
||||
filename: Base name of the file inside ``<workdir>/tmp/``.
|
||||
file_id: Optional database record ID passed through to downstream tasks.
|
||||
original_text: Optional original embedded text for head-to-head comparison.
|
||||
"""
|
||||
task_id = self.request.id
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_with_ocr",
|
||||
"in_progress",
|
||||
f"Starting OCR for {filename}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
try:
|
||||
tmp_file_path = os.path.join(settings.workdir, "tmp", filename)
|
||||
if not os.path.exists(tmp_file_path):
|
||||
raise FileNotFoundError(f"Local file not found: {tmp_file_path}")
|
||||
|
||||
providers = get_ocr_providers()
|
||||
provider_names = [p.name for p in providers]
|
||||
logger.info(f"[{task_id}] Running {len(providers)} OCR provider(s): {provider_names}")
|
||||
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"run_ocr_providers",
|
||||
"in_progress",
|
||||
f"Running OCR providers: {', '.join(provider_names)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
results = []
|
||||
errors = []
|
||||
for provider in providers:
|
||||
pname = provider.__class__.__name__
|
||||
try:
|
||||
logger.info(f"[{task_id}] Running {pname} on {filename}")
|
||||
result: OCRResult = provider.process(tmp_file_path)
|
||||
results.append(result)
|
||||
logger.info(f"[{task_id}] {pname} extracted {len(result.text)} chars")
|
||||
except Exception as exc:
|
||||
logger.error(f"[{task_id}] {pname} failed for {filename}: {exc}")
|
||||
errors.append(f"{pname}: {exc}")
|
||||
|
||||
if not results:
|
||||
error_summary = "; ".join(errors)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"run_ocr_providers",
|
||||
"failure",
|
||||
"All OCR providers failed",
|
||||
file_id=file_id,
|
||||
detail=error_summary,
|
||||
)
|
||||
raise RuntimeError(f"All OCR providers failed for {filename}: {error_summary}")
|
||||
|
||||
if errors:
|
||||
logger.warning(f"[{task_id}] Some OCR providers failed: {'; '.join(errors)}")
|
||||
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"run_ocr_providers",
|
||||
"success",
|
||||
f"{len(results)} of {len(providers)} OCR provider(s) succeeded",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Merge results (no-op when only one provider succeeded)
|
||||
extracted_text, searchable_pdf_path, rotation_data = merge_ocr_results(results, filename)
|
||||
logger.info(
|
||||
f"[{task_id}] Merged OCR text: {len(extracted_text)} chars, "
|
||||
f"pdf={'yes' if searchable_pdf_path else 'no'}, "
|
||||
f"rotations={len(rotation_data)}"
|
||||
)
|
||||
|
||||
# If no provider produced a searchable PDF, post-process the original
|
||||
# PDF with ocrmypdf to embed an invisible text layer so the output is
|
||||
# selectable/searchable in PDF viewers.
|
||||
if searchable_pdf_path is None:
|
||||
lang = getattr(settings, "tesseract_language", None) or "eng"
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"embed_text_layer",
|
||||
"in_progress",
|
||||
"Embedding searchable text layer into PDF",
|
||||
file_id=file_id,
|
||||
)
|
||||
embedded = embed_text_layer(tmp_file_path, tmp_file_path, language=lang)
|
||||
if embedded:
|
||||
searchable_pdf_path = tmp_file_path
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"embed_text_layer",
|
||||
"success",
|
||||
"Searchable text layer embedded via ocrmypdf",
|
||||
file_id=file_id,
|
||||
)
|
||||
else:
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"embed_text_layer",
|
||||
"skipped",
|
||||
"ocrmypdf unavailable – PDF will not have a searchable text layer",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Head-to-head comparison with original embedded text (if provided)
|
||||
# ----------------------------------------------------------------
|
||||
final_text = extracted_text
|
||||
if original_text and original_text.strip() and extracted_text.strip():
|
||||
logger.info(f"[{task_id}] Original embedded text provided; running head-to-head quality comparison")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"compare_ocr_quality",
|
||||
"in_progress",
|
||||
"Comparing OCR result against original embedded text",
|
||||
file_id=file_id,
|
||||
)
|
||||
try:
|
||||
comparison = compare_text_quality(original_text, extracted_text)
|
||||
comparison_detail = (
|
||||
f"Original score: {comparison.original_score}/100, "
|
||||
f"OCR score: {comparison.ocr_score}/100, "
|
||||
f"Preferred: {comparison.preferred}\n"
|
||||
f"AI explanation: {comparison.explanation}"
|
||||
)
|
||||
logger.info(f"[{task_id}] OCR comparison – {comparison_detail}")
|
||||
|
||||
if comparison.preferred == "original":
|
||||
# Original text is actually better – use it instead of OCR.
|
||||
final_text = original_text
|
||||
logger.info(
|
||||
f"[{task_id}] Original embedded text selected "
|
||||
f"(original={comparison.original_score} > ocr={comparison.ocr_score})"
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"compare_ocr_quality",
|
||||
"success",
|
||||
f"Original text preferred (original={comparison.original_score}/100 vs "
|
||||
f"ocr={comparison.ocr_score}/100)",
|
||||
file_id=file_id,
|
||||
detail=comparison_detail,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"[{task_id}] OCR text selected "
|
||||
f"(preferred={comparison.preferred!r}, "
|
||||
f"ocr={comparison.ocr_score}, original={comparison.original_score})"
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"compare_ocr_quality",
|
||||
"success",
|
||||
f"OCR text preferred (ocr={comparison.ocr_score}/100 vs "
|
||||
f"original={comparison.original_score}/100)",
|
||||
file_id=file_id,
|
||||
detail=comparison_detail,
|
||||
)
|
||||
except Exception as cmp_exc:
|
||||
logger.warning(f"[{task_id}] Head-to-head comparison failed ({cmp_exc}); keeping OCR text")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"compare_ocr_quality",
|
||||
"skipped",
|
||||
f"Comparison failed ({cmp_exc}); keeping OCR output",
|
||||
file_id=file_id,
|
||||
)
|
||||
elif original_text is not None:
|
||||
# original_text was provided but one side is empty – pick whichever has content.
|
||||
if not extracted_text.strip() and original_text.strip():
|
||||
final_text = original_text
|
||||
logger.info(f"[{task_id}] OCR returned empty text; falling back to original embedded text")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"compare_ocr_quality",
|
||||
"success",
|
||||
"OCR empty – using original embedded text",
|
||||
file_id=file_id,
|
||||
)
|
||||
else:
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"compare_ocr_quality",
|
||||
"skipped",
|
||||
"No original text to compare; using OCR output",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_with_ocr",
|
||||
"success",
|
||||
f"OCR complete for {filename}",
|
||||
file_id=file_id,
|
||||
detail=f"Extracted {len(extracted_text)} chars using {len(results)} provider(s); "
|
||||
f"final text length: {len(final_text)} chars",
|
||||
)
|
||||
|
||||
# Score the final embedded text — the text that will land in ocr_text.
|
||||
# We always call check_text_quality() on final_text because:
|
||||
# - merge_ocr_results() may have AI-merged output from several engines
|
||||
# - compare_text_quality() scores are relative (not the same scale)
|
||||
# - The original may have been preferred, reversing the OCR output
|
||||
# OCR-produced (or AI-merged) text is treated as TextSource.OCR_PREVIOUS
|
||||
# so the quality AI call is always made.
|
||||
if file_id is not None:
|
||||
try:
|
||||
quality_result = check_text_quality(final_text, TextSource.OCR_PREVIOUS)
|
||||
logger.info(
|
||||
f"[{task_id}] Final text quality: score={quality_result.quality_score}/100, "
|
||||
f"good={quality_result.is_good_quality}, feedback={quality_result.feedback!r}"
|
||||
)
|
||||
with SessionLocal() as _db:
|
||||
_rec = _db.query(FileRecord).filter_by(id=file_id).first()
|
||||
if _rec:
|
||||
_rec.ocr_quality_score = quality_result.quality_score
|
||||
_db.commit()
|
||||
logger.info(f"[{task_id}] Saved ocr_quality_score={quality_result.quality_score} for file_id={file_id}")
|
||||
except Exception as _score_exc:
|
||||
logger.warning(f"[{task_id}] Could not persist ocr_quality_score: {_score_exc}")
|
||||
|
||||
# Continue pipeline: rotate pages (if needed), then extract metadata
|
||||
rotate_pdf_pages.delay(filename, final_text, rotation_data, file_id)
|
||||
|
||||
return {
|
||||
"file": filename,
|
||||
"searchable_pdf": searchable_pdf_path or tmp_file_path,
|
||||
"cleaned_text": final_text,
|
||||
"providers_used": [r.provider for r in results],
|
||||
}
|
||||
|
||||
except Exception as exc:
|
||||
logger.error(f"[{task_id}] OCR failed for {filename}: {exc}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"process_with_ocr",
|
||||
"failure",
|
||||
f"OCR failed for {filename}",
|
||||
file_id=file_id,
|
||||
detail=str(exc),
|
||||
)
|
||||
raise
|
||||
@@ -0,0 +1,101 @@
|
||||
import os
|
||||
import logging
|
||||
from azure.core.credentials import AzureKeyCredential
|
||||
from azure.ai.documentintelligence import DocumentIntelligenceClient
|
||||
from azure.ai.documentintelligence.models import AnalyzeOutputOption, AnalyzeResult
|
||||
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
||||
from app.celery_app import celery
|
||||
from app.utils import task_logger, log_task
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Initialize Azure Document Intelligence client
|
||||
document_intelligence_client = DocumentIntelligenceClient(
|
||||
endpoint=settings.azure_endpoint,
|
||||
credential=AzureKeyCredential(settings.azure_ai_key)
|
||||
)
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
@log_task("process_with_textract")
|
||||
def process_with_textract(s3_filename: str):
|
||||
"""
|
||||
Processes a PDF document using Azure Document Intelligence and overlays OCR text onto
|
||||
the local temporary file (stored under <workdir>/tmp).
|
||||
|
||||
Steps:
|
||||
1. Uploads the document for OCR using Azure Document Intelligence.
|
||||
2. Retrieves the processed PDF with embedded text.
|
||||
3. Saves the OCR-processed PDF locally in the same location as before.
|
||||
4. Extracts the text content for metadata processing.
|
||||
5. Triggers downstream metadata extraction by calling extract_metadata_with_gpt.
|
||||
"""
|
||||
task_id = process_with_textract.request.id
|
||||
tmp_file_path = os.path.join(settings.workdir, "tmp", s3_filename)
|
||||
|
||||
# Get the file_id from the database
|
||||
file_id = None
|
||||
with SessionLocal() as db:
|
||||
file_record = db.query(FileRecord).filter(
|
||||
FileRecord.local_filename == tmp_file_path
|
||||
).first()
|
||||
if file_record:
|
||||
file_id = file_record.id
|
||||
|
||||
task_logger(f"Starting OCR for {s3_filename}", step_name="process_with_textract",
|
||||
task_id=task_id, file_id=file_id, file_path=tmp_file_path)
|
||||
|
||||
if not os.path.exists(tmp_file_path):
|
||||
task_logger(f"Local file not found: {tmp_file_path}", level="error",
|
||||
step_name="process_with_textract", task_id=task_id,
|
||||
file_id=file_id, file_path=tmp_file_path)
|
||||
raise FileNotFoundError(f"Local file not found: {tmp_file_path}")
|
||||
|
||||
try:
|
||||
task_logger(f"Sending document to Azure Document Intelligence",
|
||||
step_name="azure_document_intelligence", task_id=task_id,
|
||||
file_id=file_id, file_path=tmp_file_path)
|
||||
|
||||
# Open and send the document for processing
|
||||
with open(tmp_file_path, "rb") as f:
|
||||
poller = document_intelligence_client.begin_analyze_document(
|
||||
"prebuilt-read", body=f, output=[AnalyzeOutputOption.PDF]
|
||||
)
|
||||
result: AnalyzeResult = poller.result()
|
||||
operation_id = poller.details["operation_id"]
|
||||
|
||||
task_logger(f"Azure Document Intelligence processing complete, operation ID: {operation_id}",
|
||||
step_name="azure_document_intelligence", task_id=task_id)
|
||||
|
||||
# Retrieve the processed searchable PDF
|
||||
task_logger(f"Retrieving searchable PDF", step_name="retrieve_pdf", task_id=task_id)
|
||||
response = document_intelligence_client.get_analyze_result_pdf(
|
||||
model_id=result.model_id, result_id=operation_id
|
||||
)
|
||||
searchable_pdf_path = tmp_file_path # Overwrite the original PDF location
|
||||
with open(searchable_pdf_path, "wb") as writer:
|
||||
writer.writelines(response)
|
||||
|
||||
# Extract raw text content from the result
|
||||
extracted_text = result.content if result.content else ""
|
||||
text_length = len(extracted_text)
|
||||
task_logger(f"Extracted {text_length} characters of text",
|
||||
step_name="extract_text", task_id=task_id)
|
||||
|
||||
# Trigger downstream metadata extraction
|
||||
task_logger(f"OCR completed. Queueing metadata extraction for {s3_filename}",
|
||||
step_name="process_with_textract", task_id=task_id, status="success")
|
||||
|
||||
metadata_task = extract_metadata_with_gpt.delay(s3_filename, extracted_text)
|
||||
task_logger(f"Triggered metadata extraction task: {metadata_task.id}",
|
||||
step_name="process_with_textract", task_id=task_id)
|
||||
|
||||
return {"file": s3_filename, "searchable_pdf": searchable_pdf_path, "text_length": text_length}
|
||||
except Exception as e:
|
||||
task_logger(f"Error processing with Azure Document Intelligence: {e}",
|
||||
level="error", step_name="process_with_textract", task_id=task_id)
|
||||
raise
|
||||
@@ -1,68 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import logging
|
||||
from app.config import settings
|
||||
import openai
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import task_logger, log_task
|
||||
|
||||
# Import the shared Celery instance
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import log_task_progress
|
||||
from app.utils.ai_provider import get_ai_provider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
# Initialize OpenAI client dynamically
|
||||
client = openai.OpenAI(
|
||||
api_key=settings.openai_api_key,
|
||||
base_url=settings.openai_base_url
|
||||
)
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
@log_task("refine_text")
|
||||
def refine_text_with_gpt(s3_filename: str, raw_text: str):
|
||||
"""Uses OpenAI to clean and refine OCR text."""
|
||||
task_id = refine_text_with_gpt.request.id
|
||||
task_logger(f"Starting text refinement for {s3_filename}", step_name="refine_text", task_id=task_id)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=settings.openai_model,
|
||||
messages=[
|
||||
{"role": "system", "content": "Clean and format the following text. The idea is that the text you see comes from an OCR system and your task is to eliminate OCR errors. Keep the original language when doing so."},
|
||||
{"role": "user", "content": raw_text}
|
||||
]
|
||||
)
|
||||
|
||||
cleaned_text = response.choices[0].message.content
|
||||
task_logger(f"Text refinement completed for {s3_filename}", step_name="refine_text", task_id=task_id)
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def refine_text_with_gpt(self, filename: str, raw_text: str):
|
||||
"""Uses the configured AI provider to clean and refine OCR text."""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting OCR text refinement for: {filename}")
|
||||
log_task_progress(task_id, "refine_text_with_gpt", "in_progress", f"Refining OCR text for {filename}")
|
||||
# Trigger next task (import locally if needed to avoid circular imports)
|
||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
||||
metadata_task = extract_metadata_with_gpt.delay(s3_filename, cleaned_text)
|
||||
task_logger(f"Triggered metadata extraction task: {metadata_task.id}", step_name="refine_text", task_id=task_id)
|
||||
|
||||
try:
|
||||
log_task_progress(task_id, "call_ai_provider", "in_progress", "Calling AI provider for text refinement")
|
||||
return {"file": s3_filename, "cleaned_text": cleaned_text}
|
||||
|
||||
provider = get_ai_provider()
|
||||
model = settings.ai_model or settings.openai_model
|
||||
cleaned_text = provider.chat_completion(
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"Clean and format the following text. The idea is that the text you see comes from an OCR "
|
||||
"system and your task is to eliminate OCR errors. Keep the original language when doing so."
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": raw_text},
|
||||
],
|
||||
model=model,
|
||||
)
|
||||
|
||||
logger.info(f"[{task_id}] Text refinement complete for {filename}: {len(cleaned_text)} characters")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"call_ai_provider",
|
||||
"success",
|
||||
"Received refined text from AI provider",
|
||||
detail=f"Input: {len(raw_text)} chars → Output: {len(cleaned_text)} chars",
|
||||
)
|
||||
|
||||
# Trigger next task (import locally if needed to avoid circular imports)
|
||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
||||
|
||||
extract_metadata_with_gpt.delay(filename, cleaned_text)
|
||||
|
||||
logger.info(f"[{task_id}] Queueing metadata extraction for {filename}")
|
||||
log_task_progress(task_id, "refine_text_with_gpt", "success", "Text refined, queuing metadata extraction")
|
||||
|
||||
return {"filename": filename, "cleaned_text": cleaned_text}
|
||||
except Exception as e:
|
||||
logger.exception(f"[{task_id}] Text refinement failed for {filename}: {e}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"refine_text_with_gpt",
|
||||
"failure",
|
||||
f"Exception: {str(e)}",
|
||||
detail=f"Text refinement failed for {filename}.\nException: {str(e)}",
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
from celery import Task
|
||||
|
||||
|
||||
class BaseTaskWithRetry(Task):
|
||||
autoretry_for = (Exception,)
|
||||
retry_kwargs = {"max_retries": 3, "countdown": 10} # 3 retries, 10s delay
|
||||
retry_backoff = True # Exponential backoff
|
||||
|
||||
|
||||
@@ -1,196 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
import pypdf # Upgraded from PyPDF2 to fix CVE-2023-36464
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.tasks.extract_metadata_with_gpt import extract_metadata_with_gpt
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import log_task_progress
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def determine_rotation_angle(detected_angle):
|
||||
"""
|
||||
Determine the optimal rotation angle based on detected angle.
|
||||
|
||||
Args:
|
||||
detected_angle: The angle detected by Azure Document Intelligence
|
||||
|
||||
Returns:
|
||||
int: The angle to rotate the page in pypdf (must be multiple of 90 degrees)
|
||||
"""
|
||||
# Normalize angle to be between 0 and 360
|
||||
normalized_angle = detected_angle % 360
|
||||
if normalized_angle < 0:
|
||||
normalized_angle += 360
|
||||
|
||||
# If angle is very small (< 1 degree), don't rotate
|
||||
if abs(normalized_angle) < 1 or abs(normalized_angle - 360) < 1:
|
||||
return 0
|
||||
|
||||
# For angles close to 90, 180, or 270 degrees (±5°), round to nearest 90° increment
|
||||
for target in [90, 180, 270]:
|
||||
if abs(normalized_angle - target) < 5:
|
||||
# pypdf uses clockwise rotation, so we need to use the complementary angle
|
||||
rotation_value = (360 - target) % 360
|
||||
logger.info(f"Detected angle {detected_angle}° is close to {target}°, will rotate by {rotation_value}°")
|
||||
return rotation_value
|
||||
|
||||
# For other significant angles, round to nearest 90° increment
|
||||
# (pypdf only supports rotations in 90-degree increments)
|
||||
closest_90_multiple = round(normalized_angle / 90) * 90
|
||||
# Convert to pypdf rotation value (clockwise)
|
||||
rotation_value = (360 - closest_90_multiple) % 360
|
||||
logger.info(f"Detected angle {detected_angle}° rounded to {closest_90_multiple}°, will rotate by {rotation_value}°")
|
||||
return rotation_value
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def rotate_pdf_pages(self, filename: str, extracted_text: str, rotation_data=None, file_id: int = None):
|
||||
"""
|
||||
Rotates pages in a PDF document based on detected rotation angles.
|
||||
|
||||
Args:
|
||||
filename: The name of the file to rotate
|
||||
extracted_text: The extracted text from the document
|
||||
rotation_data: Optional rotation data dictionary {page_index: angle}
|
||||
file_id: Optional file ID to pass through to subsequent tasks
|
||||
"""
|
||||
try:
|
||||
task_id = self.request.id
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"rotate_pdf_pages",
|
||||
"in_progress",
|
||||
f"Checking page rotation for {filename}",
|
||||
file_id=file_id,
|
||||
)
|
||||
pdf_path = os.path.join(settings.workdir, "tmp", filename)
|
||||
if not os.path.exists(pdf_path):
|
||||
raise FileNotFoundError(f"PDF file not found: {pdf_path}")
|
||||
|
||||
# Skip rotation if no rotation data provided
|
||||
if not rotation_data:
|
||||
logger.info(f"[{task_id}] No rotation data provided for {filename}, proceeding with metadata extraction")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"rotate_pdf_pages",
|
||||
"success",
|
||||
"No rotation needed, proceeding to metadata extraction",
|
||||
file_id=file_id,
|
||||
)
|
||||
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
|
||||
return {"file": filename, "status": "no_rotation_needed"}
|
||||
|
||||
# Standardize rotation_data keys to integers
|
||||
normalized_rotation_data = {}
|
||||
for key, value in rotation_data.items():
|
||||
try:
|
||||
normalized_rotation_data[int(key)] = float(value)
|
||||
except (ValueError, TypeError):
|
||||
logger.warning(f"[{task_id}] Invalid rotation data key-value: {key}:{value}")
|
||||
|
||||
if not any(abs(angle) > 0 for angle in normalized_rotation_data.values()):
|
||||
logger.info(
|
||||
f"[{task_id}] No significant rotations detected in {filename}, proceeding with metadata extraction"
|
||||
)
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"rotate_pdf_pages",
|
||||
"success",
|
||||
"No rotation needed, proceeding to metadata extraction",
|
||||
file_id=file_id,
|
||||
)
|
||||
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
|
||||
return {"file": filename, "status": "no_rotation_needed"}
|
||||
|
||||
logger.info(f"[{task_id}] Rotating {len(normalized_rotation_data)} pages in {filename}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"apply_rotation",
|
||||
"in_progress",
|
||||
f"Rotating {len(normalized_rotation_data)} pages",
|
||||
file_id=file_id,
|
||||
)
|
||||
applied_rotations = {}
|
||||
|
||||
# Load the PDF
|
||||
with open(pdf_path, "rb") as file:
|
||||
pdf_reader = pypdf.PdfReader(file)
|
||||
pdf_writer = pypdf.PdfWriter()
|
||||
|
||||
# Process each page
|
||||
for page_idx in range(len(pdf_reader.pages)):
|
||||
page = pdf_reader.pages[page_idx]
|
||||
|
||||
# Apply rotation if this page has rotation data
|
||||
if page_idx in normalized_rotation_data and abs(normalized_rotation_data[page_idx]) > 0:
|
||||
detected_angle = normalized_rotation_data[page_idx]
|
||||
rotation_angle = determine_rotation_angle(detected_angle)
|
||||
|
||||
if rotation_angle > 0:
|
||||
# pypdf uses clockwise rotation in 90-degree increments
|
||||
page.rotate(rotation_angle)
|
||||
logger.info(
|
||||
f"[{task_id}] Page {page_idx + 1} rotated by {rotation_angle}° "
|
||||
f"(from detected {detected_angle}°)"
|
||||
)
|
||||
applied_rotations[str(page_idx)] = rotation_angle
|
||||
else:
|
||||
logger.info(
|
||||
f"[{task_id}] Page {page_idx + 1} had detected angle {detected_angle}° "
|
||||
"but determined it doesn't need rotation"
|
||||
)
|
||||
|
||||
pdf_writer.add_page(page)
|
||||
|
||||
# Save the rotated PDF
|
||||
with open(pdf_path, "wb") as output_file:
|
||||
pdf_writer.write(output_file)
|
||||
|
||||
if applied_rotations:
|
||||
logger.info(
|
||||
f"[{task_id}] Successfully rotated PDF: {filename} with rotations: {json.dumps(applied_rotations)}"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"[{task_id}] Detected rotations in {filename} but no rotations were actually applied "
|
||||
"(angles too small or not multiples of 90°)"
|
||||
)
|
||||
|
||||
# Continue with metadata extraction
|
||||
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
|
||||
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"rotate_pdf_pages",
|
||||
"success",
|
||||
f"Rotation complete for {filename}",
|
||||
file_id=file_id,
|
||||
detail=json.dumps({"applied_rotations": applied_rotations}),
|
||||
)
|
||||
|
||||
return {
|
||||
"file": filename,
|
||||
"status": "rotated" if applied_rotations else "no_rotation_needed",
|
||||
"detected_rotations": rotation_data,
|
||||
"applied_rotations": applied_rotations,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[{task_id}] Error rotating PDF {filename}: {e}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"rotate_pdf_pages",
|
||||
"failure",
|
||||
f"Rotation failed: {str(e)}",
|
||||
file_id=file_id,
|
||||
detail=json.dumps({"error": str(e), "filename": filename}),
|
||||
)
|
||||
# Continue with metadata extraction despite rotation failure
|
||||
extract_metadata_with_gpt.delay(filename, extracted_text, file_id)
|
||||
return {"file": filename, "status": "rotation_failed", "error": str(e)}
|
||||
+24
-243
@@ -1,253 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import logging
|
||||
import os
|
||||
# app/tasks/send_to_all.py
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal
|
||||
from app.models import FileRecord
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.tasks.upload_to_dropbox import upload_to_dropbox
|
||||
from app.tasks.upload_to_email import upload_to_email
|
||||
from app.tasks.upload_to_ftp import upload_to_ftp
|
||||
from app.tasks.upload_to_google_drive import upload_to_google_drive
|
||||
from app.tasks.upload_to_nextcloud import upload_to_nextcloud
|
||||
from app.tasks.upload_to_onedrive import upload_to_onedrive
|
||||
from app.tasks.upload_to_paperless import upload_to_paperless
|
||||
from app.tasks.upload_to_s3 import upload_to_s3
|
||||
from app.tasks.upload_to_sftp import upload_to_sftp
|
||||
from app.tasks.upload_to_webdav import upload_to_webdav
|
||||
from app.utils.config_validator import get_provider_status
|
||||
from app.utils.logging import log_task_progress
|
||||
from app.utils import task_logger, log_task
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _should_upload_to_dropbox():
|
||||
return bool(settings.dropbox_app_key and settings.dropbox_app_secret and settings.dropbox_refresh_token)
|
||||
|
||||
|
||||
def _should_upload_to_nextcloud():
|
||||
return bool(settings.nextcloud_upload_url and settings.nextcloud_username and settings.nextcloud_password)
|
||||
|
||||
|
||||
def _should_upload_to_paperless():
|
||||
return bool(settings.paperless_ngx_api_token and settings.paperless_host)
|
||||
|
||||
|
||||
def _should_upload_to_google_drive():
|
||||
# Check for OAuth configuration
|
||||
if getattr(settings, "google_drive_use_oauth", False):
|
||||
return bool(
|
||||
settings.google_drive_client_id
|
||||
and settings.google_drive_client_secret
|
||||
and settings.google_drive_refresh_token
|
||||
and settings.google_drive_folder_id
|
||||
)
|
||||
# Or check for service account configuration
|
||||
else:
|
||||
return bool(settings.google_drive_credentials_json and settings.google_drive_folder_id)
|
||||
|
||||
|
||||
def _should_upload_to_webdav():
|
||||
return bool(settings.webdav_url and settings.webdav_username and settings.webdav_password)
|
||||
|
||||
|
||||
def _should_upload_to_ftp():
|
||||
return bool(settings.ftp_host and settings.ftp_username and settings.ftp_password)
|
||||
|
||||
|
||||
def _should_upload_to_sftp():
|
||||
return bool(settings.sftp_host and settings.sftp_username and (settings.sftp_password or settings.sftp_private_key))
|
||||
|
||||
|
||||
def _should_upload_to_email():
|
||||
return bool(
|
||||
settings.email_host and settings.email_username and settings.email_password and settings.email_default_recipient
|
||||
)
|
||||
|
||||
|
||||
def _should_upload_to_onedrive():
|
||||
return bool(settings.onedrive_client_id and settings.onedrive_client_secret and settings.onedrive_refresh_token)
|
||||
|
||||
|
||||
def _should_upload_to_s3():
|
||||
return bool(settings.s3_bucket_name and settings.aws_access_key_id and settings.aws_secret_access_key)
|
||||
|
||||
|
||||
def get_configured_services_from_validator():
|
||||
@celery.task
|
||||
@log_task("send_to_all_destinations")
|
||||
def send_to_all_destinations(file_path: str):
|
||||
"""
|
||||
Use the config validator to determine which services are configured properly.
|
||||
Returns a dictionary with service names as keys and boolean values indicating
|
||||
whether they're properly configured.
|
||||
Fires off tasks to upload a single file to Dropbox, Nextcloud, and Paperless.
|
||||
These tasks run in parallel (Celery returns immediately from each .delay()).
|
||||
"""
|
||||
providers = get_provider_status()
|
||||
task_logger(f"Sending {file_path} to all destinations", step_name="send_to_all")
|
||||
|
||||
dropbox_task = upload_to_dropbox.delay(file_path)
|
||||
nextcloud_task = upload_to_nextcloud.delay(file_path)
|
||||
paperless_task = upload_to_paperless.delay(file_path)
|
||||
|
||||
service_map = {
|
||||
"Dropbox": "dropbox",
|
||||
"NextCloud": "nextcloud",
|
||||
"Paperless-ngx": "paperless",
|
||||
"Google Drive": "google_drive",
|
||||
"WebDAV": "webdav",
|
||||
"FTP Storage": "ftp",
|
||||
"SFTP Storage": "sftp",
|
||||
"Email": "email",
|
||||
"OneDrive": "onedrive",
|
||||
"S3 Storage": "s3",
|
||||
task_logger(f"Enqueued file for all destinations: Dropbox (task: {dropbox_task.id}), "
|
||||
f"Nextcloud (task: {nextcloud_task.id}), Paperless (task: {paperless_task.id})",
|
||||
step_name="send_to_all", status="success")
|
||||
|
||||
return {
|
||||
"status": "All upload tasks enqueued",
|
||||
"file_path": file_path,
|
||||
"task_ids": {
|
||||
"dropbox": dropbox_task.id,
|
||||
"nextcloud": nextcloud_task.id,
|
||||
"paperless": paperless_task.id
|
||||
}
|
||||
}
|
||||
|
||||
result = {}
|
||||
for provider_name, internal_name in service_map.items():
|
||||
if provider_name in providers:
|
||||
result[internal_name] = providers[provider_name].get("configured", False)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def send_to_all_destinations(self, file_path: str, use_validator=True, file_id: int = None):
|
||||
"""
|
||||
Distribute a file to all configured storage destinations.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to distribute
|
||||
use_validator: Whether to use the config validator to determine enabled services
|
||||
(if False, falls back to individual checks)
|
||||
file_id: Optional file ID to associate with logs
|
||||
"""
|
||||
task_id = self.request.id
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
logger.error(f"[{task_id}] File not found: {file_path}")
|
||||
log_task_progress(task_id, "send_to_all_destinations", "failure", "File not found", file_id=file_id)
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
logger.info(f"[{task_id}] Sending {file_path} to all configured destinations")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"send_to_all_destinations",
|
||||
"in_progress",
|
||||
f"Distributing: {os.path.basename(file_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
|
||||
# Get file_id from database if not provided (fallback only, prefer passing file_id explicitly)
|
||||
if file_id is None:
|
||||
with SessionLocal() as db:
|
||||
# Only as a last resort, try to find by basename match
|
||||
# This should not be needed if file_id is passed correctly through the chain
|
||||
file_record = (
|
||||
db.query(FileRecord)
|
||||
.filter(FileRecord.local_filename == os.path.join(settings.workdir, "tmp", os.path.basename(file_path)))
|
||||
.first()
|
||||
)
|
||||
if file_record:
|
||||
file_id = file_record.id
|
||||
|
||||
results = {}
|
||||
|
||||
# Define service configurations
|
||||
services = [
|
||||
{
|
||||
"name": "dropbox",
|
||||
"should_upload": _should_upload_to_dropbox,
|
||||
"upload_func": upload_to_dropbox,
|
||||
},
|
||||
{
|
||||
"name": "nextcloud",
|
||||
"should_upload": _should_upload_to_nextcloud,
|
||||
"upload_func": upload_to_nextcloud,
|
||||
},
|
||||
{
|
||||
"name": "paperless",
|
||||
"should_upload": _should_upload_to_paperless,
|
||||
"upload_func": upload_to_paperless,
|
||||
},
|
||||
{
|
||||
"name": "google_drive",
|
||||
"should_upload": _should_upload_to_google_drive,
|
||||
"upload_func": upload_to_google_drive,
|
||||
},
|
||||
{
|
||||
"name": "webdav",
|
||||
"should_upload": _should_upload_to_webdav,
|
||||
"upload_func": upload_to_webdav,
|
||||
},
|
||||
{
|
||||
"name": "ftp",
|
||||
"should_upload": _should_upload_to_ftp,
|
||||
"upload_func": upload_to_ftp,
|
||||
},
|
||||
{
|
||||
"name": "sftp",
|
||||
"should_upload": _should_upload_to_sftp,
|
||||
"upload_func": upload_to_sftp,
|
||||
},
|
||||
{
|
||||
"name": "email",
|
||||
"should_upload": _should_upload_to_email,
|
||||
"upload_func": upload_to_email,
|
||||
},
|
||||
{
|
||||
"name": "onedrive",
|
||||
"should_upload": _should_upload_to_onedrive,
|
||||
"upload_func": upload_to_onedrive,
|
||||
},
|
||||
{
|
||||
"name": "s3",
|
||||
"should_upload": _should_upload_to_s3,
|
||||
"upload_func": upload_to_s3,
|
||||
},
|
||||
]
|
||||
|
||||
# Optionally get configuration status from validator
|
||||
configured_services = {}
|
||||
if use_validator:
|
||||
try:
|
||||
configured_services = get_configured_services_from_validator()
|
||||
logger.info(f"[{task_id}] Configured services according to validator: {configured_services}")
|
||||
except Exception as e:
|
||||
logger.warning(f"[{task_id}] Failed to get configuration from validator: {str(e)}")
|
||||
use_validator = False
|
||||
|
||||
# Process each service
|
||||
queued_count = 0
|
||||
for service in services:
|
||||
service_name = service["name"]
|
||||
|
||||
# Determine if service is configured
|
||||
is_configured = False
|
||||
if use_validator and service_name in configured_services:
|
||||
is_configured = configured_services[service_name]
|
||||
logger.debug(f"[{task_id}] {service_name} configuration from validator: {is_configured}")
|
||||
else:
|
||||
try:
|
||||
is_configured = service["should_upload"]()
|
||||
logger.debug(f"[{task_id}] {service_name} configuration from function: {is_configured}")
|
||||
except Exception as e:
|
||||
logger.error(f"[{task_id}] Error checking configuration for {service_name}: {str(e)}")
|
||||
is_configured = False
|
||||
|
||||
# Queue the upload task if service is configured
|
||||
if is_configured:
|
||||
logger.info(f"[{task_id}] Queueing {file_path} for {service_name} upload")
|
||||
log_task_progress(
|
||||
task_id, f"queue_{service_name}", "in_progress", f"Queueing upload to {service_name}", file_id=file_id
|
||||
)
|
||||
try:
|
||||
task = service["upload_func"].delay(file_path, file_id=file_id)
|
||||
results[f"{service_name}_task_id"] = task.id
|
||||
queued_count += 1
|
||||
log_task_progress(
|
||||
task_id, f"queue_{service_name}", "success", f"Queued for {service_name}", file_id=file_id
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"[{task_id}] Failed to queue {service_name} task: {str(e)}")
|
||||
results[f"{service_name}_error"] = str(e)
|
||||
log_task_progress(task_id, f"queue_{service_name}", "failure", f"Failed: {str(e)}", file_id=file_id)
|
||||
|
||||
logger.info(f"[{task_id}] Queued {queued_count} upload tasks")
|
||||
log_task_progress(task_id, "send_to_all_destinations", "success", f"Queued {queued_count} uploads", file_id=file_id)
|
||||
|
||||
return {"status": "Queued", "file_path": file_path, "tasks": results}
|
||||
|
||||
+37
-179
@@ -1,223 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
import dropbox
|
||||
import requests
|
||||
from dropbox.exceptions import ApiError, AuthError
|
||||
|
||||
from app.celery_app import celery
|
||||
import dropbox
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import log_task_progress
|
||||
from app.utils.filename_utils import extract_remote_path, get_unique_filename
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _validate_dropbox_settings():
|
||||
"""Validate that all required Dropbox settings are available."""
|
||||
missing = []
|
||||
|
||||
if not hasattr(settings, "dropbox_refresh_token") or not settings.dropbox_refresh_token:
|
||||
missing.append("refresh token")
|
||||
|
||||
if not hasattr(settings, "dropbox_app_key") or not settings.dropbox_app_key:
|
||||
missing.append("app key")
|
||||
|
||||
if not hasattr(settings, "dropbox_app_secret") or not settings.dropbox_app_secret:
|
||||
missing.append("app secret")
|
||||
|
||||
if missing:
|
||||
logger.error(f"Cannot refresh Dropbox token: Missing {', '.join(missing)}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.utils import task_logger, log_task
|
||||
|
||||
def get_dropbox_access_token():
|
||||
"""Refresh the Dropbox access token using the stored refresh token from ENV."""
|
||||
|
||||
# Check if needed settings are available
|
||||
if not _validate_dropbox_settings():
|
||||
return None
|
||||
|
||||
token_url = "https://api.dropbox.com/oauth2/token"
|
||||
headers = {"Content-Type": "application/x-www-form-urlencoded"}
|
||||
data = {
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": settings.dropbox_refresh_token,
|
||||
"refresh_token": settings.dropbox_refresh_token, # Now using ENV
|
||||
"client_id": settings.dropbox_app_key,
|
||||
"client_secret": settings.dropbox_app_secret,
|
||||
}
|
||||
|
||||
response = requests.post(token_url, headers=headers, data=data, timeout=settings.http_request_timeout)
|
||||
response = requests.post(token_url, headers=headers, data=data)
|
||||
|
||||
if response.status_code == 200:
|
||||
return response.json()["access_token"]
|
||||
else:
|
||||
error_msg = f"Failed to refresh Dropbox token: {response.status_code} - {response.text}"
|
||||
logger.error(error_msg)
|
||||
task_logger(error_msg, level="error", step_name="dropbox_auth")
|
||||
raise Exception(error_msg)
|
||||
|
||||
|
||||
def get_dropbox_client():
|
||||
"""
|
||||
Create and return an authenticated Dropbox client using the configured refresh token.
|
||||
|
||||
Returns:
|
||||
dropbox.Dropbox: Authenticated Dropbox client instance
|
||||
|
||||
Raises:
|
||||
ValueError: If required Dropbox configuration is missing
|
||||
AuthError: If authentication with Dropbox fails
|
||||
"""
|
||||
app_key = settings.dropbox_app_key
|
||||
app_secret = settings.dropbox_app_secret
|
||||
refresh_token = settings.dropbox_refresh_token
|
||||
|
||||
# Validate configuration
|
||||
if not app_key or not app_secret:
|
||||
raise ValueError("Dropbox app key or app secret is not configured")
|
||||
|
||||
if not refresh_token:
|
||||
raise ValueError("Dropbox refresh token is not configured")
|
||||
|
||||
# Create a Dropbox client with refresh token
|
||||
try:
|
||||
dbx = dropbox.Dropbox(app_key=app_key, app_secret=app_secret, oauth2_refresh_token=refresh_token)
|
||||
|
||||
# Test the connection
|
||||
dbx.users_get_current_account()
|
||||
logger.info("Successfully authenticated with Dropbox")
|
||||
return dbx
|
||||
|
||||
except AuthError as auth_error:
|
||||
logger.error(f"Dropbox authentication failed: {str(auth_error)}")
|
||||
raise
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating Dropbox client: {str(e)}")
|
||||
raise
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def upload_to_dropbox(self, file_path: str, file_id: int = None):
|
||||
"""
|
||||
Upload a file to Dropbox.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to upload
|
||||
file_id: Optional file ID to associate with logs
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting Dropbox upload: {file_path}")
|
||||
log_task_progress(
|
||||
task_id,
|
||||
"upload_to_dropbox",
|
||||
"in_progress",
|
||||
f"Uploading to Dropbox: {os.path.basename(file_path)}",
|
||||
file_id=file_id,
|
||||
)
|
||||
@celery.task(base=BaseTaskWithRetry)
|
||||
@log_task("upload_to_dropbox")
|
||||
def upload_to_dropbox(file_path: str):
|
||||
"""Uploads a file to Dropbox using the API."""
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
error_msg = f"File not found: {file_path}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_dropbox", "failure", error_msg, file_id=file_id)
|
||||
raise FileNotFoundError(error_msg)
|
||||
|
||||
# Check if Dropbox is properly configured
|
||||
if not (
|
||||
hasattr(settings, "dropbox_app_key")
|
||||
and settings.dropbox_app_key
|
||||
and hasattr(settings, "dropbox_app_secret")
|
||||
and settings.dropbox_app_secret
|
||||
and hasattr(settings, "dropbox_refresh_token")
|
||||
and settings.dropbox_refresh_token
|
||||
):
|
||||
logger.info(f"[{task_id}] Dropbox upload skipped: Missing configuration")
|
||||
log_task_progress(task_id, "upload_to_dropbox", "success", "Skipped: Not configured", file_id=file_id)
|
||||
return {"status": "Skipped", "reason": "Dropbox settings not configured"}
|
||||
task_logger(f"File not found: {file_path}", level="error", step_name="dropbox_upload")
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
|
||||
# Extract filename and set target path
|
||||
filename = os.path.basename(file_path)
|
||||
dropbox_path = f"{settings.dropbox_folder}/{filename}"
|
||||
|
||||
try:
|
||||
# Get the Dropbox client
|
||||
dbx = get_dropbox_client()
|
||||
# Get fresh access token
|
||||
task_logger(f"Getting Dropbox access token", step_name="dropbox_auth")
|
||||
access_token = get_dropbox_access_token()
|
||||
dbx = dropbox.Dropbox(access_token)
|
||||
|
||||
# Calculate remote path based on local file structure
|
||||
remote_base = settings.dropbox_folder or ""
|
||||
remote_path = extract_remote_path(file_path, settings.workdir, remote_base)
|
||||
file_size = os.path.getsize(file_path)
|
||||
chunk_size = 4 * 1024 * 1024 # 4MB chunk size
|
||||
|
||||
# Function to check if file exists in Dropbox
|
||||
def check_exists_in_dropbox(path):
|
||||
try:
|
||||
dbx.files_get_metadata(path)
|
||||
return True
|
||||
except ApiError as e:
|
||||
if e.error.is_path() and e.error.get_path().is_not_found():
|
||||
return False
|
||||
raise
|
||||
|
||||
# Get a unique path in case of collision
|
||||
remote_full_path = f"/{remote_path}" # Dropbox paths should start with /
|
||||
remote_full_path = remote_full_path.replace("//", "/") # Clean double slashes
|
||||
|
||||
# Check for potential file collision and get a unique name if needed
|
||||
dropbox_path = get_unique_filename(remote_full_path, check_exists_in_dropbox)
|
||||
|
||||
# Upload the file
|
||||
logger.info(f"[{task_id}] Uploading {filename} to Dropbox at {dropbox_path}")
|
||||
log_task_progress(task_id, "upload_file", "in_progress", f"Uploading to {dropbox_path}", file_id=file_id)
|
||||
task_logger(f"Starting upload of {filename} ({file_size} bytes) to Dropbox", step_name="dropbox_upload")
|
||||
|
||||
with open(file_path, "rb") as file_data:
|
||||
# Use files_upload_session for large files to avoid timeouts
|
||||
file_size = os.path.getsize(file_path)
|
||||
if file_size > 10 * 1024 * 1024: # 10 MB threshold for chunked upload
|
||||
cursor = None
|
||||
chunk_size = 4 * 1024 * 1024 # 4 MB chunks
|
||||
file_data.seek(0)
|
||||
if file_size <= chunk_size:
|
||||
dbx.files_upload(file_data.read(), dropbox_path)
|
||||
else:
|
||||
task_logger(f"Using chunked upload for {filename}", step_name="dropbox_upload")
|
||||
upload_session_start_result = dbx.files_upload_session_start(file_data.read(chunk_size))
|
||||
cursor = dropbox.files.UploadSessionCursor(
|
||||
session_id=upload_session_start_result.session_id,
|
||||
offset=file_data.tell(),
|
||||
)
|
||||
commit = dropbox.files.CommitInfo(path=dropbox_path)
|
||||
|
||||
# Start upload session
|
||||
session_start = dbx.files_upload_session_start(file_data.read(chunk_size))
|
||||
cursor = dropbox.files.UploadSessionCursor(session_start.session_id, file_data.tell())
|
||||
|
||||
# Upload chunks until we reach the end
|
||||
while file_data.tell() < file_size:
|
||||
if (file_size - file_data.tell()) <= chunk_size:
|
||||
# Last chunk
|
||||
dbx.files_upload_session_finish(
|
||||
file_data.read(chunk_size),
|
||||
cursor,
|
||||
dropbox.files.CommitInfo(path=dropbox_path, mode=dropbox.files.WriteMode.overwrite),
|
||||
)
|
||||
dbx.files_upload_session_finish(file_data.read(chunk_size), cursor, commit)
|
||||
else:
|
||||
# More chunks to upload
|
||||
dbx.files_upload_session_append_v2(file_data.read(chunk_size), cursor)
|
||||
cursor.offset = file_data.tell()
|
||||
else:
|
||||
# Small file, direct upload
|
||||
file_data.seek(0)
|
||||
dbx.files_upload(file_data.read(), dropbox_path, mode=dropbox.files.WriteMode.overwrite)
|
||||
|
||||
logger.info(f"[{task_id}] Successfully uploaded {filename} to Dropbox at {dropbox_path}")
|
||||
log_task_progress(
|
||||
task_id, "upload_to_dropbox", "success", f"Uploaded to Dropbox: {dropbox_path}", file_id=file_id
|
||||
)
|
||||
return {"status": "Completed", "file_path": file_path, "dropbox_path": dropbox_path}
|
||||
task_logger(f"Successfully uploaded {filename} to Dropbox at {dropbox_path}", step_name="dropbox_upload", status="success")
|
||||
return {"status": "Completed", "file": file_path}
|
||||
|
||||
except AuthError:
|
||||
error_msg = f"Authentication failed while uploading {filename} to Dropbox. Check token."
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_dropbox", "failure", error_msg, file_id=file_id)
|
||||
raise Exception(error_msg)
|
||||
except ApiError as e:
|
||||
error_msg = f"Failed to upload {filename} to Dropbox: {e}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_dropbox", "failure", error_msg, file_id=file_id)
|
||||
raise Exception(error_msg)
|
||||
except Exception as e:
|
||||
error_msg = f"Unexpected error uploading {filename} to Dropbox: {e}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_dropbox", "failure", error_msg, file_id=file_id)
|
||||
error_msg = f"Failed to upload {filename} to Dropbox: {str(e)}"
|
||||
task_logger(error_msg, level="error", step_name="dropbox_upload", status="failure")
|
||||
raise Exception(error_msg)
|
||||
|
||||
@@ -1,299 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import smtplib
|
||||
import socket
|
||||
from datetime import datetime
|
||||
from email.mime.application import MIMEApplication
|
||||
from email.mime.image import MIMEImage
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import log_task_progress
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Constants
|
||||
_LOGO_FILENAME = "logo.png"
|
||||
|
||||
|
||||
def get_email_template(template_name="default.html"):
|
||||
"""
|
||||
Load email template from one of these locations in order of precedence:
|
||||
1. Custom template from workdir/templates/email/
|
||||
2. Default template from app/templates/email/
|
||||
"""
|
||||
# First try to load from workdir (user customizable location)
|
||||
try:
|
||||
workdir_template_path = os.path.join(settings.workdir, "templates", "email")
|
||||
if os.path.exists(workdir_template_path):
|
||||
env = Environment(
|
||||
loader=FileSystemLoader(workdir_template_path), autoescape=select_autoescape(["html", "xml"])
|
||||
)
|
||||
env.globals["now"] = datetime.now # Add the now function to the Jinja environment
|
||||
template = env.get_template(template_name)
|
||||
logger.info(f"Using custom email template from workdir: {template_name}")
|
||||
return template
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load custom email template: {str(e)}")
|
||||
|
||||
# Fallback to built-in template
|
||||
try:
|
||||
# Get the app directory path (where this file is)
|
||||
current_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
app_template_path = os.path.join(current_dir, "templates", "email")
|
||||
env = Environment(loader=FileSystemLoader(app_template_path), autoescape=select_autoescape(["html", "xml"]))
|
||||
env.globals["now"] = datetime.now # Add the now function to the Jinja environment
|
||||
template = env.get_template(template_name)
|
||||
logger.info(f"Using built-in email template: {template_name}")
|
||||
return template
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load built-in email template: {str(e)}")
|
||||
raise ValueError(f"Could not find any valid email template: {str(e)}")
|
||||
|
||||
|
||||
def extract_metadata_from_file(file_path):
|
||||
"""
|
||||
Try to extract metadata from a file using several methods:
|
||||
1. Check for a .json metadata file with the same name
|
||||
2. Extract metadata from PDF if it's embedded
|
||||
|
||||
Returns a dictionary of metadata or None if not found
|
||||
"""
|
||||
metadata = {}
|
||||
|
||||
# Check for separate metadata JSON file
|
||||
metadata_path = os.path.splitext(file_path)[0] + ".json"
|
||||
if os.path.exists(metadata_path):
|
||||
try:
|
||||
with open(metadata_path, "r", encoding="utf-8") as f:
|
||||
metadata = json.load(f)
|
||||
logger.info(f"Loaded metadata from external JSON file: {metadata_path}")
|
||||
return metadata
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load metadata from JSON file: {str(e)}")
|
||||
|
||||
# TODO: For PDF files, try to extract embedded metadata using PyPDF2
|
||||
# This would require additional dependencies, so for now we'll just check for external JSON
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
def attach_logo(msg):
|
||||
"""Attach the DocuElevate logo to the email with proper Content-ID."""
|
||||
try:
|
||||
# Try to find logo in workdir first (for customization)
|
||||
custom_logo_path = os.path.join(settings.workdir, "templates", "email", _LOGO_FILENAME)
|
||||
if os.path.exists(custom_logo_path):
|
||||
logo_path = custom_logo_path
|
||||
else:
|
||||
# Use built-in logo
|
||||
app_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
logo_path = os.path.join(app_dir, "static", _LOGO_FILENAME)
|
||||
# Fallback to logo in frontend/static if app/static doesn't exist
|
||||
if not os.path.exists(logo_path):
|
||||
logo_path = os.path.join(app_dir, "..", "frontend", "static", _LOGO_FILENAME)
|
||||
|
||||
if os.path.exists(logo_path):
|
||||
with open(logo_path, "rb") as img:
|
||||
logo_data = img.read()
|
||||
|
||||
# Determine image MIME type based on extension
|
||||
mimetype = "image/svg+xml" if logo_path.endswith(".svg") else "image/png"
|
||||
logo_attach = MIMEImage(logo_data, mimetype)
|
||||
logo_attach.add_header("Content-ID", "<logo>")
|
||||
logo_attach.add_header("Content-Disposition", "inline", filename=_LOGO_FILENAME)
|
||||
msg.attach(logo_attach)
|
||||
logger.info(f"Logo attached from {logo_path}")
|
||||
return True
|
||||
else:
|
||||
logger.warning("Could not find logo file")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Error attaching logo: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
def _prepare_recipients(recipients):
|
||||
"""Helper function to prepare email recipients list."""
|
||||
if not recipients:
|
||||
if not settings.email_default_recipient:
|
||||
error_msg = "No recipients specified and no default recipient configured"
|
||||
logger.error(error_msg)
|
||||
return None, error_msg
|
||||
return [settings.email_default_recipient], None
|
||||
elif isinstance(recipients, str):
|
||||
return [recipients], None # Convert single email to list
|
||||
return recipients, None
|
||||
|
||||
|
||||
def _send_email_with_smtp(msg, filename, recipients):
|
||||
"""Helper function to handle SMTP connection and sending."""
|
||||
try:
|
||||
# First try to resolve the hostname
|
||||
socket.gethostbyname(settings.email_host)
|
||||
|
||||
# Connect to the SMTP server
|
||||
with smtplib.SMTP(settings.email_host, settings.email_port, timeout=30) as server:
|
||||
# Use TLS if specified
|
||||
if settings.email_use_tls:
|
||||
server.starttls()
|
||||
|
||||
# Login if credentials are provided
|
||||
if settings.email_username and settings.email_password:
|
||||
server.login(settings.email_username, settings.email_password)
|
||||
|
||||
# Send the email
|
||||
server.send_message(msg)
|
||||
|
||||
logger.info(f"Successfully sent {filename} via email to {', '.join(recipients)}")
|
||||
return None
|
||||
except socket.gaierror as e:
|
||||
error_msg = f"Failed to resolve email host: {settings.email_host} - {str(e)}"
|
||||
logger.error(error_msg)
|
||||
return {"status": "Failed", "reason": error_msg, "error": str(e)}
|
||||
except (ConnectionRefusedError, TimeoutError) as e:
|
||||
error_msg = f"Connection error to SMTP server {settings.email_host}:{settings.email_port} - {str(e)}"
|
||||
logger.error(error_msg)
|
||||
return {"status": "Failed", "reason": error_msg, "error": str(e)}
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def upload_to_email(
|
||||
self,
|
||||
file_path: str,
|
||||
recipients=None,
|
||||
subject=None,
|
||||
message=None,
|
||||
template_name="default.html",
|
||||
include_metadata=True,
|
||||
file_id: int = None,
|
||||
):
|
||||
"""
|
||||
Sends a file via email to the specified recipients.
|
||||
If recipients is None, uses the configured default email recipient.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to send
|
||||
recipients: Optional list of recipient email addresses
|
||||
subject: Optional email subject
|
||||
message: Optional custom message
|
||||
template_name: Email template to use
|
||||
include_metadata: Whether to include metadata in the email
|
||||
file_id: Optional file ID to associate with logs
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting email send: {file_path}")
|
||||
log_task_progress(
|
||||
task_id, "upload_to_email", "in_progress", f"Sending via email: {os.path.basename(file_path)}", file_id=file_id
|
||||
)
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
error_msg = f"File not found: {file_path}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_email", "failure", error_msg, file_id=file_id)
|
||||
raise FileNotFoundError(error_msg)
|
||||
|
||||
# Extract filename
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
# Check if email settings are configured
|
||||
if not settings.email_host:
|
||||
error_msg = "Email host is not configured"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_email", "skipped", error_msg, file_id=file_id)
|
||||
return {"status": "Skipped", "reason": error_msg}
|
||||
|
||||
# Log email configuration for debugging
|
||||
logger.debug(
|
||||
f"[{task_id}] Email config - Host: {settings.email_host}, Port: {settings.email_port}, "
|
||||
f"Username: {settings.email_username}, TLS: {settings.email_use_tls}"
|
||||
)
|
||||
|
||||
# Process recipients
|
||||
recipients, error = _prepare_recipients(recipients)
|
||||
if error:
|
||||
logger.error(f"[{task_id}] {error}")
|
||||
log_task_progress(task_id, "upload_to_email", "skipped", error, file_id=file_id)
|
||||
return {"status": "Skipped", "reason": error}
|
||||
|
||||
# Use provided subject or create default
|
||||
subject = subject or f"DocuElevate Document: {filename}"
|
||||
|
||||
# Extract document metadata if available
|
||||
metadata = {}
|
||||
if include_metadata:
|
||||
metadata = extract_metadata_from_file(file_path)
|
||||
|
||||
try:
|
||||
# Create the email
|
||||
msg = MIMEMultipart("related")
|
||||
msg["From"] = settings.email_sender or settings.email_username
|
||||
msg["To"] = ", ".join(recipients)
|
||||
msg["Subject"] = subject
|
||||
|
||||
# Create alternative part for HTML content
|
||||
alt_part = MIMEMultipart("alternative")
|
||||
msg.attach(alt_part)
|
||||
|
||||
# Attach logo to the email
|
||||
has_logo = attach_logo(msg)
|
||||
|
||||
# Load and render template
|
||||
template = get_email_template(template_name)
|
||||
|
||||
# Context data for the template
|
||||
context = {
|
||||
"filename": filename,
|
||||
"message": message or f"Attached is the document: {filename}",
|
||||
"app_name": "DocuElevate",
|
||||
"app_url": f"https://{settings.external_hostname}" if settings.external_hostname else None,
|
||||
"custom_message": message,
|
||||
"metadata": metadata,
|
||||
"has_metadata": bool(metadata),
|
||||
"has_logo": has_logo,
|
||||
"current_year": datetime.now().year,
|
||||
}
|
||||
|
||||
# Render HTML body
|
||||
html_content = template.render(**context)
|
||||
alt_part.attach(MIMEText(html_content, "html"))
|
||||
|
||||
# Attach the file
|
||||
with open(file_path, "rb") as file:
|
||||
attachment = MIMEApplication(file.read(), _subtype="pdf")
|
||||
attachment.add_header("Content-Disposition", f'attachment; filename="{filename}"')
|
||||
msg.attach(attachment)
|
||||
|
||||
# Send the email through SMTP
|
||||
error_result = _send_email_with_smtp(msg, filename, recipients)
|
||||
if error_result:
|
||||
logger.error(f"[{task_id}] Failed to send email: {error_result.get('reason')}")
|
||||
log_task_progress(task_id, "upload_to_email", "failure", error_result.get("reason"), file_id=file_id)
|
||||
return error_result
|
||||
|
||||
logger.info(f"[{task_id}] Successfully sent {filename} via email to {len(recipients)} recipients")
|
||||
log_task_progress(task_id, "upload_to_email", "success", f"Sent via email: {filename}", file_id=file_id)
|
||||
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file": file_path,
|
||||
"recipients": recipients,
|
||||
"subject": subject,
|
||||
"metadata_included": bool(metadata),
|
||||
"logo_included": has_logo,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to send {filename} via email: {str(e)}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_email", "failure", error_msg, file_id=file_id)
|
||||
raise Exception(error_msg)
|
||||
@@ -1,149 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Security Warning: FTP is an insecure protocol. FTPS (FTP_TLS) is strongly recommended.
|
||||
# This module attempts to use FTPS by default and falls back to plaintext FTP only if configured.
|
||||
import ftplib # nosec B402 - FTP usage is intentional for legacy server support
|
||||
import logging
|
||||
import os
|
||||
|
||||
from app.celery_app import celery
|
||||
from app.config import settings
|
||||
from app.tasks.retry_config import BaseTaskWithRetry
|
||||
from app.utils import log_task_progress
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery.task(base=BaseTaskWithRetry, bind=True)
|
||||
def upload_to_ftp(self, file_path: str, file_id: int = None):
|
||||
"""
|
||||
Uploads a file to an FTP server in the configured folder.
|
||||
|
||||
Security Note: This function prefers FTPS (FTP with TLS) for secure connections.
|
||||
Plaintext FTP is only used if FTPS fails and ftp_allow_plaintext=True (default).
|
||||
For security-critical environments, set ftp_allow_plaintext=False and ftp_use_tls=True.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to upload
|
||||
file_id: Optional file ID to associate with logs
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[{task_id}] Starting FTP upload: {file_path}")
|
||||
log_task_progress(
|
||||
task_id, "upload_to_ftp", "in_progress", f"Uploading to FTP: {os.path.basename(file_path)}", file_id=file_id
|
||||
)
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
error_msg = f"File not found: {file_path}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_ftp", "failure", error_msg, file_id=file_id)
|
||||
raise FileNotFoundError(error_msg)
|
||||
|
||||
# Extract filename
|
||||
filename = os.path.basename(file_path)
|
||||
|
||||
# Check if FTP settings are configured
|
||||
if not settings.ftp_host:
|
||||
error_msg = "FTP host is not configured"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_ftp", "failure", error_msg, file_id=file_id)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
try:
|
||||
# First attempt FTPS (FTP with TLS)
|
||||
use_tls = getattr(settings, "ftp_use_tls", True) # Default to try TLS
|
||||
allow_plaintext = getattr(settings, "ftp_allow_plaintext", True) # Default to allow plaintext fallback
|
||||
used_tls = False # Track whether we successfully used TLS
|
||||
|
||||
if use_tls:
|
||||
try:
|
||||
logger.info(f"Attempting FTPS connection to {settings.ftp_host}")
|
||||
ftp = ftplib.FTP_TLS() # noqa: S321
|
||||
ftp.connect(host=settings.ftp_host, port=settings.ftp_port or 21)
|
||||
|
||||
# Login with credentials
|
||||
ftp.login(user=settings.ftp_username, passwd=settings.ftp_password)
|
||||
|
||||
# Enable data protection - encrypt the data channel
|
||||
ftp.prot_p()
|
||||
logger.info("Successfully established FTPS connection with TLS")
|
||||
used_tls = True
|
||||
except Exception as e:
|
||||
if not allow_plaintext:
|
||||
error_msg = f"FTPS connection failed and plaintext FTP is forbidden: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
else:
|
||||
logger.warning(f"FTPS connection failed, falling back to regular FTP: {str(e)}")
|
||||
# Fall back to regular FTP - only if explicitly allowed by configuration
|
||||
ftp = ftplib.FTP() # nosec B321 - Fallback to FTP intentional when configured # noqa: S321
|
||||
ftp.connect(host=settings.ftp_host, port=settings.ftp_port or 21)
|
||||
|
||||
# Login with credentials
|
||||
ftp.login(user=settings.ftp_username, passwd=settings.ftp_password)
|
||||
else:
|
||||
# Check if plaintext is allowed when TLS is explicitly disabled
|
||||
if not allow_plaintext:
|
||||
error_msg = "Plaintext FTP is forbidden by configuration"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
# Directly use regular FTP if TLS is explicitly disabled
|
||||
logger.warning("Using plaintext FTP - connection is NOT encrypted!")
|
||||
ftp = ftplib.FTP() # nosec B321 - Plaintext FTP intentional when explicitly configured # noqa: S321
|
||||
ftp.connect(host=settings.ftp_host, port=settings.ftp_port or 21)
|
||||
|
||||
# Login with credentials
|
||||
ftp.login(user=settings.ftp_username, passwd=settings.ftp_password)
|
||||
|
||||
# Change to target directory if specified
|
||||
if settings.ftp_folder:
|
||||
try:
|
||||
# Try to navigate to the directory, create if it doesn't exist
|
||||
ftp_folder = settings.ftp_folder
|
||||
# Remove leading slash if present
|
||||
if ftp_folder.startswith("/"):
|
||||
ftp_folder = ftp_folder[1:]
|
||||
|
||||
# Try to change to the directory
|
||||
try:
|
||||
ftp.cwd(ftp_folder)
|
||||
except ftplib.error_perm:
|
||||
# Create directory structure if it doesn't exist
|
||||
folders = ftp_folder.split("/")
|
||||
current_dir = ""
|
||||
for folder in folders:
|
||||
if folder:
|
||||
current_dir += f"/{folder}"
|
||||
try:
|
||||
ftp.cwd(current_dir)
|
||||
except ftplib.error_perm:
|
||||
ftp.mkd(current_dir)
|
||||
ftp.cwd(current_dir)
|
||||
except ftplib.Error as e:
|
||||
error_msg = f"Failed to change/create directory on FTP server: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
# Upload the file
|
||||
with open(file_path, "rb") as file_data:
|
||||
ftp.storbinary(f"STOR {filename}", file_data)
|
||||
|
||||
# Close FTP connection
|
||||
ftp.quit()
|
||||
|
||||
logger.info(f"[{task_id}] Successfully uploaded {filename} to FTP server at {settings.ftp_host}")
|
||||
log_task_progress(task_id, "upload_to_ftp", "success", f"Uploaded to FTP: {filename}", file_id=file_id)
|
||||
return {
|
||||
"status": "Completed",
|
||||
"file": file_path,
|
||||
"ftp_host": settings.ftp_host,
|
||||
"ftp_path": f"{settings.ftp_folder}/{filename}" if settings.ftp_folder else filename,
|
||||
"used_tls": used_tls,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to upload {filename} to FTP server: {str(e)}"
|
||||
logger.error(f"[{task_id}] {error_msg}")
|
||||
log_task_progress(task_id, "upload_to_ftp", "failure", error_msg, file_id=file_id)
|
||||
raise Exception(error_msg)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user