copilot-swe-agent[bot]
eea99eb01d
- Add ENABLE_PDFA_CONVERSION, PDFA_FORMAT, PDFA_UPLOAD_TO_PROVIDERS config settings - Add original_pdfa_path and processed_pdfa_path columns to FileRecord model - Create Alembic migration 011_add_pdfa_paths - Create app/tasks/convert_to_pdfa.py Celery task using ocrmypdf + Ghostscript - Integrate PDF/A conversion into finalize_document_storage pipeline - Add comprehensive unit tests (15 tests) - Update .env.demo and docs/ConfigurationGuide.md Co-authored-by: christianlouis <361235+christianlouis@users.noreply.github.com>
362 lines
16 KiB
Bash
362 lines
16 KiB
Bash
# **Core Settings**
|
||
WORKDIR=/workdir
|
||
DATABASE_URL=sqlite:///./app/database.db
|
||
REDIS_URL=redis://redis:6379/0
|
||
EXTERNAL_HOSTNAME=docuelevate.example.com
|
||
GOTENBERG_URL=http://gotenberg:3000
|
||
ALLOW_FILE_DELETE=true # Allow deletion of file records
|
||
|
||
# **UI / Appearance**
|
||
# Default colour scheme: system (follow OS), light, or dark
|
||
# Individual users can always override with the navbar dark-mode toggle.
|
||
# UI_DEFAULT_COLOR_SCHEME=system
|
||
|
||
# **Batch Processing Settings**
|
||
# Control throttling behavior for the /processall endpoint to prevent overwhelming downstream APIs
|
||
PROCESSALL_THROTTLE_THRESHOLD=20 # Number of files above which throttling is applied (default: 20)
|
||
PROCESSALL_THROTTLE_DELAY=3 # Delay in seconds between each task submission when throttling (default: 3)
|
||
|
||
# **Task Retry Settings**
|
||
# Failed tasks are automatically retried with exponential backoff and jitter.
|
||
# TASK_RETRY_MAX_RETRIES=3 # Max retry attempts per task (default: 3)
|
||
# TASK_RETRY_DELAYS=60,300,900 # Countdown (seconds) before each retry; 1 min, 5 min, 15 min
|
||
# TASK_RETRY_JITTER=true # Add ±20% random jitter to prevent thundering-herd (default: true)
|
||
|
||
# **Client-Side Upload Throttling**
|
||
# Controls pacing when the browser uploads files (especially large directory drops).
|
||
# The browser auto-detects rate-limit (HTTP 429) responses and backs off accordingly.
|
||
UPLOAD_CONCURRENCY=3 # Max simultaneous uploads from the browser (default: 3)
|
||
UPLOAD_QUEUE_DELAY_MS=500 # Delay (ms) between starting each upload slot (default: 500)
|
||
|
||
# **File Upload Size Limits** (Security - see SECURITY_AUDIT.md)
|
||
# Maximum file upload size in bytes. Default: 1GB (1073741824 bytes)
|
||
# Prevents resource exhaustion attacks. Adjust based on your server capacity.
|
||
MAX_UPLOAD_SIZE=1073741824
|
||
|
||
# Maximum size for a single file chunk in bytes (optional)
|
||
# If set and a file exceeds this size, it will be split into smaller chunks for processing
|
||
# Default: None (no splitting). Example: 104857600 for 100MB chunks
|
||
# MAX_SINGLE_FILE_SIZE=104857600
|
||
|
||
# **Request Body Size Limit** (Security - see SECURITY_AUDIT.md)
|
||
# Maximum request body size in bytes for non-file-upload requests (JSON, form data, etc.).
|
||
# Default: 1MB (1048576 bytes). File uploads are governed by MAX_UPLOAD_SIZE above.
|
||
# Prevents memory exhaustion from oversized JSON/form payloads.
|
||
# MAX_REQUEST_BODY_SIZE=1048576
|
||
|
||
# **Security Headers** (see SECURITY_AUDIT.md and docs/DeploymentGuide.md)
|
||
# Disabled by default since most deployments use a reverse proxy (Traefik, Nginx, etc.)
|
||
# that already adds these headers. Set to true only if deploying directly without a reverse proxy.
|
||
# SECURITY_HEADERS_ENABLED=false
|
||
|
||
# If you enable security headers, you can also configure individual headers:
|
||
# Strict-Transport-Security (HSTS) - Forces HTTPS connections
|
||
# Only effective when served over HTTPS. Disable if not using HTTPS or if proxy adds this header
|
||
# SECURITY_HEADER_HSTS_ENABLED=true
|
||
# SECURITY_HEADER_HSTS_VALUE="max-age=31536000; includeSubDomains"
|
||
|
||
# Content-Security-Policy (CSP) - Controls resource loading
|
||
# Customize based on your application's resource loading needs
|
||
# Default allows self-hosted resources, inline scripts/styles, and external images
|
||
# SECURITY_HEADER_CSP_ENABLED=true
|
||
# SECURITY_HEADER_CSP_VALUE="default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:;"
|
||
|
||
# X-Frame-Options - Prevents clickjacking attacks
|
||
# Options: DENY (no framing), SAMEORIGIN (same origin framing only), ALLOW-FROM uri
|
||
# SECURITY_HEADER_X_FRAME_OPTIONS_ENABLED=true
|
||
# SECURITY_HEADER_X_FRAME_OPTIONS_VALUE="DENY"
|
||
|
||
# X-Content-Type-Options - Prevents MIME sniffing
|
||
# Always set to 'nosniff' when enabled
|
||
# SECURITY_HEADER_X_CONTENT_TYPE_OPTIONS_ENABLED=true
|
||
|
||
# **CORS (Cross-Origin Resource Sharing)** (see SECURITY_AUDIT.md – Infrastructure Security)
|
||
# Disabled by default: most deployments rely on a reverse proxy (Traefik, Nginx, etc.) to inject
|
||
# CORS headers. Set CORS_ENABLED=true only if DocuElevate is exposed directly without a proxy,
|
||
# or if your proxy does not handle CORS. When enabled, only list the exact origins that need access.
|
||
#
|
||
# Rationale for reverse-proxy-first approach:
|
||
# Traefik/Nginx already set Access-Control-Allow-Origin (and related headers) for every response,
|
||
# so adding the middleware here would duplicate headers. When this flag is False the application
|
||
# trusts the proxy layer to enforce CORS policy; set it to True for standalone / direct-access
|
||
# deployments only.
|
||
#
|
||
# CORS_ENABLED=false
|
||
#
|
||
# Comma-separated list of allowed origins (use * to allow all - not recommended with credentials)
|
||
# CORS_ALLOWED_ORIGINS=https://app.example.com,https://admin.example.com
|
||
#
|
||
# Allow cookies / Authorization headers in cross-origin requests
|
||
# Must be False when CORS_ALLOWED_ORIGINS=* (browser security requirement)
|
||
# CORS_ALLOW_CREDENTIALS=false
|
||
#
|
||
# Allowed HTTP methods (comma-separated)
|
||
# CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,OPTIONS,PATCH
|
||
#
|
||
# Allowed request headers (use * to allow all)
|
||
# CORS_ALLOWED_HEADERS=*
|
||
|
||
# **Rate Limiting** (see SECURITY_AUDIT.md and docs/API.md)
|
||
# Protects against DoS attacks and API abuse by limiting request rates per IP/user
|
||
# Enabled by default - highly recommended for production
|
||
RATE_LIMITING_ENABLED=true
|
||
|
||
# Default rate limit for all API endpoints (format: count/period)
|
||
# Periods can be: second, minute, hour, day
|
||
# Default: 100 requests per minute per IP/user
|
||
RATE_LIMIT_DEFAULT=100/minute
|
||
|
||
# Rate limit for file upload endpoints
|
||
# Allows faster uploads while still preventing abuse
|
||
# Default: 600 uploads per minute per IP/user
|
||
RATE_LIMIT_UPLOAD=600/minute
|
||
|
||
# Rate limit for authentication endpoints
|
||
# Strict limit to prevent brute force attacks
|
||
# Default: 10 attempts per minute per IP
|
||
RATE_LIMIT_AUTH=10/minute
|
||
|
||
# Note: Processing endpoints (OCR, metadata extraction) use built-in queue throttling
|
||
# via Celery task queue to control processing rates and prevent upstream API overloads.
|
||
# No additional API-level rate limit is needed for processing endpoints.
|
||
|
||
# **Authentication**
|
||
AUTH_ENABLED=true
|
||
# Generate a secure random string, for example:
|
||
# python -c "import secrets; print(secrets.token_hex(32))"
|
||
SESSION_SECRET=b39fd43f68d0491ca942f28a16e484b1e763fe9accf4445ca2669a5f3b179eb4
|
||
ADMIN_USERNAME=admin
|
||
ADMIN_PASSWORD=your_secure_password
|
||
ADMIN_GROUP_NAME=admin
|
||
|
||
# **OpenID Connect/Authentik Settings**
|
||
AUTHENTIK_CLIENT_ID=<yourAuthentikAppClientID>
|
||
AUTHENTIK_CLIENT_SECRET=<yourAuthentikClientSecret>
|
||
AUTHENTIK_CONFIG_URL=<ConfigUrlOfYourApp, e.g. https://authentik.example.com/application/o/docuelevate/.well-known/openid-configuration>
|
||
OAUTH_PROVIDER_NAME="Authentik SSO"
|
||
|
||
# **AI/ML Services**
|
||
# Select your AI provider: openai | azure | anthropic | gemini | ollama | openrouter | portkey | litellm
|
||
AI_PROVIDER=openai
|
||
# Model override (optional – falls back to OPENAI_MODEL when not set)
|
||
# AI_MODEL=gpt-4o-mini
|
||
|
||
# --- OpenAI (AI_PROVIDER=openai) ---
|
||
OPENAI_API_KEY="<OPENAI_API_KEY>"
|
||
OPENAI_BASE_URL=https://api.openai.com/v1
|
||
OPENAI_MODEL=gpt-4o-mini
|
||
|
||
# --- Anthropic Claude (AI_PROVIDER=anthropic) ---
|
||
# ANTHROPIC_API_KEY=sk-ant-...
|
||
# AI_MODEL=claude-3-5-sonnet-20241022
|
||
|
||
# --- Google Gemini (AI_PROVIDER=gemini) ---
|
||
# GEMINI_API_KEY=AIza...
|
||
# AI_MODEL=gemini-1.5-pro
|
||
|
||
# --- Ollama local LLMs (AI_PROVIDER=ollama) ---
|
||
# OLLAMA_BASE_URL=http://localhost:11434
|
||
# AI_MODEL=llama3.2
|
||
|
||
# --- OpenRouter (AI_PROVIDER=openrouter) ---
|
||
# OPENROUTER_API_KEY=sk-or-...
|
||
# AI_MODEL=anthropic/claude-3.5-sonnet
|
||
|
||
# --- Portkey AI Gateway (AI_PROVIDER=portkey) ---
|
||
# PORTKEY_API_KEY=pk-...
|
||
# PORTKEY_VIRTUAL_KEY=vk-... # optional – routes to provider credentials in Portkey vault
|
||
# PORTKEY_CONFIG=pc-... # optional – saved Config ID for fallbacks / load balancing
|
||
|
||
# --- Azure OpenAI (AI_PROVIDER=azure) ---
|
||
# OPENAI_API_KEY=<azure-key>
|
||
# OPENAI_BASE_URL=https://my-resource.openai.azure.com
|
||
# AZURE_OPENAI_API_VERSION=2024-02-01
|
||
# AI_MODEL=gpt-4o # deployment name in Azure
|
||
|
||
# Azure Document Intelligence (OCR – separate from AI provider above)
|
||
# **Email Settings**
|
||
EMAIL_HOST=smtp.example.com
|
||
EMAIL_PORT=587
|
||
EMAIL_USERNAME=docuelevate@example.com
|
||
EMAIL_PASSWORD=your_secure_email_password
|
||
EMAIL_USE_TLS=True
|
||
EMAIL_SENDER=DocuElevate System <docuelevate@example.com>
|
||
EMAIL_DEFAULT_RECIPIENT=recipient@example.com
|
||
|
||
# **IMAP Settings**
|
||
IMAP1_HOST=mail.example.com
|
||
IMAP1_PORT=993
|
||
IMAP1_USERNAME=<IMAP1_USERNAME>
|
||
IMAP1_PASSWORD=<IMAP1_PASSWORD>
|
||
IMAP1_SSL=true
|
||
IMAP1_POLL_INTERVAL_MINUTES=5
|
||
IMAP1_DELETE_AFTER_PROCESS=false
|
||
|
||
IMAP2_HOST=imap.gmail.com
|
||
IMAP2_PORT=993
|
||
IMAP2_USERNAME=<IMAP2_USERNAME>
|
||
IMAP2_PASSWORD=<IMAP2_PASSWORD>
|
||
IMAP2_SSL=true
|
||
IMAP2_POLL_INTERVAL_MINUTES=10
|
||
IMAP2_DELETE_AFTER_PROCESS=false
|
||
|
||
# IMAP Readonly Mode (Feature Flag)
|
||
# When true, IMAP processing will fetch and process attachments but will NOT modify
|
||
# the mailbox state (no starring, labeling, deleting, or flag changes).
|
||
# Use for pre-production instances that share a mailbox with production.
|
||
IMAP_READONLY_MODE=false
|
||
|
||
# **Storage/Document Services**
|
||
# Amazon S3
|
||
AWS_REGION=us-east-1
|
||
AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
|
||
AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
|
||
S3_BUCKET_NAME=my-document-bucket
|
||
S3_FOLDER_PREFIX=documents/uploads/2023/ # Organizes files in this subfolder
|
||
S3_STORAGE_CLASS=STANDARD
|
||
S3_ACL=private
|
||
|
||
# NextCloud
|
||
NEXTCLOUD_UPLOAD_URL=https://nextcloud.example.com/remote.php/dav/files/<USERNAME>
|
||
NEXTCLOUD_FOLDER="<NEXTCLOUD_FOLDER_PATH>"
|
||
NEXTCLOUD_USERNAME=<NEXTCLOUD_USERNAME>
|
||
NEXTCLOUD_PASSWORD=<NEXTCLOUD_PASSWORD>
|
||
|
||
# Paperless-ngx
|
||
PAPERLESS_HOST=https://paperless.example.com
|
||
PAPERLESS_NGX_API_TOKEN=<PAPERLESS_API_TOKEN>
|
||
# Optional: Name of the custom field in Paperless-ngx to store the "absender" (sender) value
|
||
# If set, the extracted sender information will be automatically set as a custom field in Paperless
|
||
# Example: PAPERLESS_CUSTOM_FIELD_ABSENDER=Absender
|
||
# PAPERLESS_CUSTOM_FIELD_ABSENDER=
|
||
|
||
# Optional: JSON mapping of metadata fields to Paperless custom field names
|
||
# This allows you to map multiple extracted metadata fields to custom fields in Paperless
|
||
# The mapping format is: {"metadata_field_name": "PaperlessCustomFieldName", ...}
|
||
# Available metadata fields: absender, empfaenger, correspondent, document_type, language,
|
||
# kommunikationsart, kommunikationskategorie, reference_number, etc.
|
||
# Example: PAPERLESS_CUSTOM_FIELDS_MAPPING={"absender": "Sender", "empfaenger": "Recipient", "language": "Language", "correspondent": "Correspondent"}
|
||
# PAPERLESS_CUSTOM_FIELDS_MAPPING=
|
||
|
||
# Dropbox
|
||
DROPBOX_APP_KEY=<DROPBOX_APP_KEY>
|
||
DROPBOX_APP_SECRET=<DROPBOX_APP_SECRET>
|
||
DROPBOX_REFRESH_TOKEN=<DROPBOX_REFRESH_TOKEN>
|
||
DROPBOX_FOLDER="/Documents/Uploads"
|
||
|
||
# Google Drive
|
||
# Service Account Method:
|
||
GOOGLE_DRIVE_CREDENTIALS_JSON={"type":"service_account","project_id":"your-project","private_key_id":"key-id","private_key":"-----BEGIN PRIVATE KEY-----\nYOUR_PRIVATE_KEY\n-----END PRIVATE KEY-----\n","client_email":"service-account@project.iam.gserviceaccount.com","client_id":"client-id","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/service-account%40project.iam.gserviceaccount.com"}
|
||
GOOGLE_DRIVE_FOLDER_ID=<YOUR_FOLDER_ID>
|
||
GOOGLE_DRIVE_DELEGATE_TO=<OPTIONAL_USER_EMAIL>
|
||
|
||
# OAuth Method (Alternative):
|
||
GOOGLE_DRIVE_USE_OAUTH=false # Set to true to use OAuth instead of service account
|
||
GOOGLE_DRIVE_CLIENT_ID=your-oauth-client-id # Required for OAuth method
|
||
GOOGLE_DRIVE_CLIENT_SECRET=your-oauth-client-secret # Required for OAuth method
|
||
GOOGLE_DRIVE_REFRESH_TOKEN=your-oauth-refresh-token # Required for OAuth method
|
||
|
||
# OneDrive
|
||
ONEDRIVE_CLIENT_ID=your-client-id
|
||
ONEDRIVE_CLIENT_SECRET=your-client-secret
|
||
ONEDRIVE_TENANT_ID=common
|
||
ONEDRIVE_REFRESH_TOKEN=your-refresh-token
|
||
ONEDRIVE_FOLDER_PATH=Documents/Uploads
|
||
|
||
# WebDAV
|
||
WEBDAV_URL=https://webdav.example.com/path
|
||
WEBDAV_USERNAME=webdav_user
|
||
WEBDAV_PASSWORD=your_secure_webdav_password
|
||
WEBDAV_FOLDER=/Documents/Uploads
|
||
WEBDAV_VERIFY_SSL=True
|
||
|
||
# FTP
|
||
# Security Note: FTP_USE_TLS=True is strongly recommended for secure connections
|
||
# Set FTP_ALLOW_PLAINTEXT=False in production to prevent unencrypted FTP
|
||
FTP_HOST=ftp.example.com
|
||
FTP_PORT=21
|
||
FTP_USERNAME=ftp_user
|
||
FTP_PASSWORD=your_secure_ftp_password
|
||
FTP_FOLDER=/Documents/Uploads
|
||
FTP_USE_TLS=True
|
||
FTP_ALLOW_PLAINTEXT=True
|
||
|
||
# SFTP
|
||
# Security Note: Host key verification is enabled by default (False)
|
||
# Only set to True in development/testing environments if needed
|
||
# When false, configure SSH known_hosts for proper host key verification
|
||
SFTP_HOST=sftp.example.com
|
||
SFTP_PORT=22
|
||
SFTP_USERNAME=sftp_user
|
||
SFTP_PASSWORD=your_secure_sftp_password
|
||
# SFTP_PRIVATE_KEY=/path/to/private_key.pem
|
||
# SFTP_PRIVATE_KEY_PASSPHRASE=optional_passphrase
|
||
SFTP_FOLDER=/Documents/Uploads
|
||
SFTP_DISABLE_HOST_KEY_VERIFICATION=False # Default is False (secure); set to True only for testing
|
||
|
||
# **HTTP Request Settings**
|
||
# Timeout for HTTP requests - set higher to handle large PDF files (up to 1GB)
|
||
HTTP_REQUEST_TIMEOUT=120 # Timeout in seconds (default: 120 for large file operations)
|
||
|
||
# **Notification Settings**
|
||
# Configure notification services using Apprise URL format
|
||
# See https://github.com/caronc/apprise#supported-notifications
|
||
# Examples:
|
||
# - Discord: discord://webhook_id/webhook_token
|
||
# - Telegram: tgram://bot_token/chat_id
|
||
# - Email: mailto://user:pass@example.com
|
||
# - Pushover: pover://user_key/app_token
|
||
# - Slack: slack://tokenA/tokenB/tokenC
|
||
# - Matrix: matrix://username:password@domain/#room
|
||
|
||
# You can specify multiple notification URLs by separating them with commas
|
||
NOTIFICATION_URLS=discord://webhook_id/webhook_token,mailto://user:pass@gmail.com,tgram://bot_token/chat_id
|
||
|
||
# Control when notifications are sent
|
||
NOTIFY_ON_TASK_FAILURE=True
|
||
NOTIFY_ON_CREDENTIAL_FAILURE=True
|
||
NOTIFY_ON_STARTUP=True
|
||
NOTIFY_ON_SHUTDOWN=False
|
||
NOTIFY_ON_FILE_PROCESSED=True
|
||
|
||
# Webhooks – Notify external systems via HTTP POST on document events.
|
||
# Individual webhooks (URL, events, secret) are managed via /api/webhooks/.
|
||
WEBHOOK_ENABLED=True
|
||
|
||
# Uptime Kuma
|
||
UPTIME_KUMA_URL=https://status.example.com/api/push/abcdef123456?status=up
|
||
UPTIME_KUMA_PING_INTERVAL=5
|
||
|
||
# **Full-Text Search (Meilisearch)**
|
||
# URL for the Meilisearch instance.
|
||
# Default is "http://meilisearch:7700" — the Docker Compose / K8s service name —
|
||
# so container-to-container networking works without extra configuration.
|
||
# Override to "http://localhost:7700" only when running the API process outside Docker.
|
||
MEILISEARCH_URL=http://meilisearch:7700
|
||
# Optional master/API key for secured Meilisearch instances
|
||
# MEILISEARCH_API_KEY=your_master_key_here
|
||
MEILISEARCH_INDEX_NAME=documents
|
||
ENABLE_SEARCH=True
|
||
|
||
# **Duplicate Detection**
|
||
# Exact duplicate detection (SHA-256) is always on during document processing.
|
||
# The settings below control near-duplicate detection (same scanned content,
|
||
# different hash) and the visibility of deduplication steps.
|
||
ENABLE_DEDUPLICATION=True
|
||
SHOW_DEDUPLICATION_STEP=True
|
||
# Minimum cosine similarity score (0–1) for two documents to be flagged as
|
||
# near-duplicates. 0.85 means 85 % semantic overlap. Lower = more matches.
|
||
NEAR_DUPLICATE_THRESHOLD=0.85
|
||
|
||
# **PDF/A Archival Conversion**
|
||
# When enabled, PDF/A copies of both the original ingested file and the processed
|
||
# file are created and saved alongside the standard copies. This may double or
|
||
# triple storage but provides better legal coverage with time-stamped archival copies.
|
||
# Uses ocrmypdf with Ghostscript for the conversion.
|
||
ENABLE_PDFA_CONVERSION=false
|
||
# PDF/A format variant: 1 = PDF/A-1b, 2 = PDF/A-2b (default), 3 = PDF/A-3b
|
||
PDFA_FORMAT=2
|
||
# Also upload the processed PDF/A variant to all configured storage providers
|
||
# (files are uploaded with a '-PDFA' suffix in their filename)
|
||
PDFA_UPLOAD_TO_PROVIDERS=false
|